{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsend an html message to one more email addresses.", "response": "def send_html_email(\n subject, # single line with no line-breaks\n html_content,\n to_emails,\n from_email=DEFAULT_FROM_EMAIL,\n fail_silently=True\n ):\n \"\"\" \n Send a message to one more email address(s).\n With html content as primary.\n \"\"\"\n messenger = EmailMessage(subject, html_content, from_email, to_emails)\n messenger.content_subtype = \"html\" # Main content is now text/html\n try:\n messenger.send()\n except Exception as e:\n if not fail_silently:\n raise"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef trim_form(parser, token):\n try:\n trim_form, orig_form, opcode, fields, as_, new_form = token.split_contents()\n except ValueError:\n raise template.TemplateSyntaxError('Invalid arguments for %r' % token.split_contents()[0])\n\n return FieldSetNode(opcode, fields.split(','), orig_form, new_form)", "response": "Returns a form that only contains a subset of the original fields."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nturning a command - line argument into a list.", "response": "def unshell_list(s):\n \"\"\"Turn a command-line argument into a list.\"\"\"\n if not s:\n return None\n if sys.platform == 'win32':\n # When running coverage as coverage.exe, some of the behavior\n # of the shell is emulated: wildcards are expanded into a list of\n # filenames. So you have to single-quote patterns on the command\n # line, but (not) helpfully, the single quotes are included in the\n # argument, so we have to strip them off here.\n s = s.strip(\"'\")\n return s.split(',')"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef parse_args(self, args=None, options=None):\n try:\n options, args = \\\n super(CoverageOptionParser, self).parse_args(args, options)\n except self.OptionParserError:\n return False, None, None\n return True, options, args", "response": "Call optparse. parse_args but return a triple"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_action(self, dash, dashdash, action_code):\n option = self.add_option(dash, dashdash, action='callback',\n callback=self._append_action\n )\n option.action_code = action_code", "response": "Add a specialized option that is the action to execute."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _append_action(self, option, opt_unused, value_unused, parser):\n parser.values.actions.append(option.action_code)", "response": "Callback for an option that adds to the actions list."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef command_line(self, argv):\n # Collect the command-line options.\n if not argv:\n self.help_fn(topic='minimum_help')\n return OK\n\n # The command syntax we parse depends on the first argument. Classic\n # syntax always starts with an option.\n self.classic = argv[0].startswith('-')\n if self.classic:\n parser = ClassicOptionParser()\n else:\n parser = CMDS.get(argv[0])\n if not parser:\n self.help_fn(\"Unknown command: '%s'\" % argv[0])\n return ERR\n argv = argv[1:]\n\n parser.help_fn = self.help_fn\n ok, options, args = parser.parse_args(argv)\n if not ok:\n return ERR\n\n # Handle help and version.\n if self.do_help(options, args, parser):\n return OK\n\n # Check for conflicts and problems in the options.\n if not self.args_ok(options, args):\n return ERR\n\n # Listify the list options.\n source = unshell_list(options.source)\n omit = unshell_list(options.omit)\n include = unshell_list(options.include)\n debug = unshell_list(options.debug)\n\n # Do something.\n self.coverage = self.covpkg.coverage(\n data_suffix = options.parallel_mode,\n cover_pylib = options.pylib,\n timid = options.timid,\n branch = options.branch,\n config_file = options.rcfile,\n source = source,\n omit = omit,\n include = include,\n debug = debug,\n )\n\n if 'debug' in options.actions:\n return self.do_debug(args)\n\n if 'erase' in options.actions or options.erase_first:\n self.coverage.erase()\n else:\n self.coverage.load()\n\n if 'execute' in options.actions:\n self.do_execute(options, args)\n\n if 'combine' in options.actions:\n self.coverage.combine()\n self.coverage.save()\n\n # Remaining actions are reporting, with some common options.\n report_args = dict(\n morfs = args,\n ignore_errors = options.ignore_errors,\n omit = omit,\n include = include,\n )\n\n if 'report' in options.actions:\n total = self.coverage.report(\n show_missing=options.show_missing, **report_args)\n if 'annotate' in options.actions:\n self.coverage.annotate(\n directory=options.directory, **report_args)\n if 'html' in options.actions:\n total = self.coverage.html_report(\n directory=options.directory, title=options.title,\n **report_args)\n if 'xml' in options.actions:\n outfile = options.outfile\n total = self.coverage.xml_report(outfile=outfile, **report_args)\n\n if options.fail_under is not None:\n if total >= options.fail_under:\n return OK\n else:\n return FAIL_UNDER\n else:\n return OK", "response": "This function is called by the command - line interface to Coverage. It will parse the command - line arguments and then call the coverage command."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef help(self, error=None, topic=None, parser=None):\n assert error or topic or parser\n if error:\n print(error)\n print(\"Use 'coverage help' for help.\")\n elif parser:\n print(parser.format_help().strip())\n else:\n help_msg = HELP_TOPICS.get(topic, '').strip()\n if help_msg:\n print(help_msg % self.covpkg.__dict__)\n else:\n print(\"Don't know topic %r\" % topic)", "response": "Display an error message or the named topic."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks if all of the arguments are ok.", "response": "def args_ok(self, options, args):\n \"\"\"Check for conflicts and problems in the options.\n\n Returns True if everything is ok, or False if not.\n\n \"\"\"\n for i in ['erase', 'execute']:\n for j in ['annotate', 'html', 'report', 'combine']:\n if (i in options.actions) and (j in options.actions):\n self.help_fn(\"You can't specify the '%s' and '%s' \"\n \"options at the same time.\" % (i, j))\n return False\n\n if not options.actions:\n self.help_fn(\n \"You must specify at least one of -e, -x, -c, -r, -a, or -b.\"\n )\n return False\n args_allowed = (\n 'execute' in options.actions or\n 'annotate' in options.actions or\n 'html' in options.actions or\n 'debug' in options.actions or\n 'report' in options.actions or\n 'xml' in options.actions\n )\n if not args_allowed and args:\n self.help_fn(\"Unexpected arguments: %s\" % \" \".join(args))\n return False\n\n if 'execute' in options.actions and not args:\n self.help_fn(\"Nothing to do.\")\n return False\n\n return True"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef do_debug(self, args):\n\n if not args:\n self.help_fn(\"What information would you like: data, sys?\")\n return ERR\n for info in args:\n if info == 'sys':\n print(\"-- sys ----------------------------------------\")\n for line in info_formatter(self.coverage.sysinfo()):\n print(\" %s\" % line)\n elif info == 'data':\n print(\"-- data ---------------------------------------\")\n self.coverage.load()\n print(\"path: %s\" % self.coverage.data.filename)\n print(\"has_arcs: %r\" % self.coverage.data.has_arcs())\n summary = self.coverage.data.summary(fullpath=True)\n if summary:\n filenames = sorted(summary.keys())\n print(\"\\n%d files:\" % len(filenames))\n for f in filenames:\n print(\"%s: %d lines\" % (f, summary[f]))\n else:\n print(\"No data collected\")\n else:\n self.help_fn(\"Don't know what you mean by %r\" % info)\n return ERR\n return OK", "response": "Implementation of coverage debug"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef serialize_object(obj, threshold=64e-6):\n databuffers = []\n if isinstance(obj, (list, tuple)):\n clist = canSequence(obj)\n slist = map(serialize, clist)\n for s in slist:\n if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:\n databuffers.append(s.getData())\n s.data = None\n return pickle.dumps(slist,-1), databuffers\n elif isinstance(obj, dict):\n sobj = {}\n for k in sorted(obj.iterkeys()):\n s = serialize(can(obj[k]))\n if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:\n databuffers.append(s.getData())\n s.data = None\n sobj[k] = s\n return pickle.dumps(sobj,-1),databuffers\n else:\n s = serialize(can(obj))\n if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:\n databuffers.append(s.getData())\n s.data = None\n return pickle.dumps(s,-1),databuffers", "response": "Serialize an object into a list of sendable buffers."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreconstructs an object serialized by serialize_object from data buffers.", "response": "def unserialize_object(bufs):\n \"\"\"reconstruct an object serialized by serialize_object from data buffers.\"\"\"\n bufs = list(bufs)\n sobj = pickle.loads(bufs.pop(0))\n if isinstance(sobj, (list, tuple)):\n for s in sobj:\n if s.data is None:\n s.data = bufs.pop(0)\n return uncanSequence(map(unserialize, sobj)), bufs\n elif isinstance(sobj, dict):\n newobj = {}\n for k in sorted(sobj.iterkeys()):\n s = sobj[k]\n if s.data is None:\n s.data = bufs.pop(0)\n newobj[k] = uncan(unserialize(s))\n return newobj, bufs\n else:\n if sobj.data is None:\n sobj.data = bufs.pop(0)\n return uncan(unserialize(sobj)), bufs"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pack_apply_message(f, args, kwargs, threshold=64e-6):\n msg = [pickle.dumps(can(f),-1)]\n databuffers = [] # for large objects\n sargs, bufs = serialize_object(args,threshold)\n msg.append(sargs)\n databuffers.extend(bufs)\n skwargs, bufs = serialize_object(kwargs,threshold)\n msg.append(skwargs)\n databuffers.extend(bufs)\n msg.extend(databuffers)\n return msg", "response": "pack up a function args and kwargs to be sent over the wire\n as a series of buffers"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef unpack_apply_message(bufs, g=None, copy=True):\n bufs = list(bufs) # allow us to pop\n assert len(bufs) >= 3, \"not enough buffers!\"\n if not copy:\n for i in range(3):\n bufs[i] = bufs[i].bytes\n cf = pickle.loads(bufs.pop(0))\n sargs = list(pickle.loads(bufs.pop(0)))\n skwargs = dict(pickle.loads(bufs.pop(0)))\n # print sargs, skwargs\n f = uncan(cf, g)\n for sa in sargs:\n if sa.data is None:\n m = bufs.pop(0)\n if sa.getTypeDescriptor() in ('buffer', 'ndarray'):\n # always use a buffer, until memoryviews get sorted out\n sa.data = buffer(m)\n # disable memoryview support\n # if copy:\n # sa.data = buffer(m)\n # else:\n # sa.data = m.buffer\n else:\n if copy:\n sa.data = m\n else:\n sa.data = m.bytes\n \n args = uncanSequence(map(unserialize, sargs), g)\n kwargs = {}\n for k in sorted(skwargs.iterkeys()):\n sa = skwargs[k]\n if sa.data is None:\n m = bufs.pop(0)\n if sa.getTypeDescriptor() in ('buffer', 'ndarray'):\n # always use a buffer, until memoryviews get sorted out\n sa.data = buffer(m)\n # disable memoryview support\n # if copy:\n # sa.data = buffer(m)\n # else:\n # sa.data = m.buffer\n else:\n if copy:\n sa.data = m\n else:\n sa.data = m.bytes\n\n kwargs[k] = uncan(unserialize(sa), g)\n \n return f,args,kwargs", "response": "unpacks f args kwargs from buffers packed by pack_apply_message"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef is_url(url):\n if '://' not in url:\n return False\n proto, addr = url.split('://', 1)\n if proto.lower() not in ['tcp','pgm','epgm','ipc','inproc']:\n return False\n return True", "response": "boolean check for whether a string is a zmq url"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef validate_url(url):\n if not isinstance(url, basestring):\n raise TypeError(\"url must be a string, not %r\"%type(url))\n url = url.lower()\n \n proto_addr = url.split('://')\n assert len(proto_addr) == 2, 'Invalid url: %r'%url\n proto, addr = proto_addr\n assert proto in ['tcp','pgm','epgm','ipc','inproc'], \"Invalid protocol: %r\"%proto\n \n # domain pattern adapted from http://www.regexlib.com/REDetails.aspx?regexp_id=391\n # author: Remi Sabourin\n pat = re.compile(r'^([\\w\\d]([\\w\\d\\-]{0,61}[\\w\\d])?\\.)*[\\w\\d]([\\w\\d\\-]{0,61}[\\w\\d])?$')\n \n if proto == 'tcp':\n lis = addr.split(':')\n assert len(lis) == 2, 'Invalid url: %r'%url\n addr,s_port = lis\n try:\n port = int(s_port)\n except ValueError:\n raise AssertionError(\"Invalid port %r in url: %r\"%(port, url))\n \n assert addr == '*' or pat.match(addr) is not None, 'Invalid url: %r'%url\n \n else:\n # only validate tcp urls currently\n pass\n \n return True", "response": "validate a url for zeromq"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validate_url_container(container):\n if isinstance(container, basestring):\n url = container\n return validate_url(url)\n elif isinstance(container, dict):\n container = container.itervalues()\n \n for element in container:\n validate_url_container(element)", "response": "validate a potentially nested collection of urls."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef split_url(url):\n proto_addr = url.split('://')\n assert len(proto_addr) == 2, 'Invalid url: %r'%url\n proto, addr = proto_addr\n lis = addr.split(':')\n assert len(lis) == 2, 'Invalid url: %r'%url\n addr,s_port = lis\n return proto,addr,s_port", "response": "split a zmq url into proto addr s_port"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndisambiguates an IP address into a multi - ip interface 0. 0 and '*' into connectable Surfaces based on the location.", "response": "def disambiguate_ip_address(ip, location=None):\n \"\"\"turn multi-ip interfaces '0.0.0.0' and '*' into connectable\n ones, based on the location (default interpretation of location is localhost).\"\"\"\n if ip in ('0.0.0.0', '*'):\n try:\n external_ips = socket.gethostbyname_ex(socket.gethostname())[2]\n except (socket.gaierror, IndexError):\n # couldn't identify this machine, assume localhost\n external_ips = []\n if location is None or location in external_ips or not external_ips:\n # If location is unspecified or cannot be determined, assume local\n ip='127.0.0.1'\n elif location:\n return location\n return ip"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef disambiguate_url(url, location=None):\n try:\n proto,ip,port = split_url(url)\n except AssertionError:\n # probably not tcp url; could be ipc, etc.\n return url\n \n ip = disambiguate_ip_address(ip,location)\n \n return \"%s://%s:%s\"%(proto,ip,port)", "response": "disambiguate a url into a multi - ip interface 0. 0 and '*' into connectable\n ."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _pull(keys):\n user_ns = globals()\n if isinstance(keys, (list,tuple, set)):\n for key in keys:\n if not user_ns.has_key(key):\n raise NameError(\"name '%s' is not defined\"%key)\n return map(user_ns.get, keys)\n else:\n if not user_ns.has_key(keys):\n raise NameError(\"name '%s' is not defined\"%keys)\n return user_ns.get(keys)", "response": "helper method for implementing client. pull via client. apply"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nselecting and return n random ports that are available.", "response": "def select_random_ports(n):\n \"\"\"Selects and return n random ports that are available.\"\"\"\n ports = []\n for i in xrange(n):\n sock = socket.socket()\n sock.bind(('', 0))\n while sock.getsockname()[1] in _random_ports:\n sock.close()\n sock = socket.socket()\n sock.bind(('', 0))\n ports.append(sock)\n for i, sock in enumerate(ports):\n port = sock.getsockname()[1]\n sock.close()\n ports[i] = port\n _random_ports.add(port)\n return ports"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrelaying interupt and term signals to children.", "response": "def signal_children(children):\n \"\"\"Relay interupt/term signals to children, for more solid process cleanup.\"\"\"\n def terminate_children(sig, frame):\n log = Application.instance().log\n log.critical(\"Got signal %i, terminating children...\"%sig)\n for child in children:\n child.terminate()\n \n sys.exit(sig != SIGINT)\n # sys.exit(sig)\n for sig in (SIGINT, SIGABRT, SIGTERM):\n signal(sig, terminate_children)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef remote(view, block=None, **flags):\n\n def remote_function(f):\n return RemoteFunction(view, f, block=block, **flags)\n return remote_function", "response": "Turn a function into a remote function."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef parallel(view, dist='b', block=None, ordered=True, **flags):\n\n def parallel_function(f):\n return ParallelFunction(view, f, dist=dist, block=block, ordered=ordered, **flags)\n return parallel_function", "response": "Turn a function into a parallel remote function."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall a function on each element of a sequence remotely.", "response": "def map(self, *sequences):\n \"\"\"call a function on each element of a sequence remotely.\n This should behave very much like the builtin map, but return an AsyncMapResult\n if self.block is False.\n \"\"\"\n # set _map as a flag for use inside self.__call__\n self._map = True\n try:\n ret = self.__call__(*sequences)\n finally:\n del self._map\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the last n items in readline history.", "response": "def get_readline_tail(self, n=10):\n \"\"\"Get the last n items in readline history.\"\"\"\n end = self.shell.readline.get_current_history_length() + 1\n start = max(end-n, 1)\n ghi = self.shell.readline.get_history_item\n return [ghi(x) for x in range(start, end)]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_autoindent(self,value=None):\n\n if value != 0 and not self.has_readline:\n if os.name == 'posix':\n warn(\"The auto-indent feature requires the readline library\")\n self.autoindent = 0\n return\n if value is None:\n self.autoindent = not self.autoindent\n else:\n self.autoindent = value", "response": "Set the autoindent flag checking for readline support."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes logging in case it was requested at the command line.", "response": "def init_logstart(self):\n \"\"\"Initialize logging in case it was requested at the command line.\n \"\"\"\n if self.logappend:\n self.magic('logstart %s append' % self.logappend)\n elif self.logfile:\n self.magic('logstart %s' % self.logfile)\n elif self.logstart:\n self.magic('logstart')"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef init_virtualenv(self):\n if 'VIRTUAL_ENV' not in os.environ:\n # Not in a virtualenv\n return\n \n if sys.executable.startswith(os.environ['VIRTUAL_ENV']):\n # Running properly in the virtualenv, don't need to do anything\n return\n \n warn(\"Attempting to work in a virtualenv. If you encounter problems, please \"\n \"install IPython inside the virtualenv.\\n\")\n if sys.platform == \"win32\":\n virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages') \n else:\n virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',\n 'python%d.%d' % sys.version_info[:2], 'site-packages')\n \n import site\n sys.path.insert(0, virtual_env)\n site.addsitedir(virtual_env)", "response": "Add a virtualenv to sys. path so the user can import modules from it."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_sys_module_state(self):\n self._orig_sys_module_state = {}\n self._orig_sys_module_state['stdin'] = sys.stdin\n self._orig_sys_module_state['stdout'] = sys.stdout\n self._orig_sys_module_state['stderr'] = sys.stderr\n self._orig_sys_module_state['excepthook'] = sys.excepthook\n self._orig_sys_modules_main_name = self.user_module.__name__\n self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)", "response": "Save the state of sys. modules."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef restore_sys_module_state(self):\n try:\n for k, v in self._orig_sys_module_state.iteritems():\n setattr(sys, k, v)\n except AttributeError:\n pass\n # Reset what what done in self.init_sys_modules\n if self._orig_sys_modules_main_mod is not None:\n sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod", "response": "Restore the state of the sys module."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):\n\n # At some point in the future, this should validate the hook before it\n # accepts it. Probably at least check that the hook takes the number\n # of args it's supposed to.\n\n f = types.MethodType(hook,self)\n\n # check if the hook is for strdispatcher first\n if str_key is not None:\n sdp = self.strdispatchers.get(name, StrDispatch())\n sdp.add_s(str_key, f, priority )\n self.strdispatchers[name] = sdp\n return\n if re_key is not None:\n sdp = self.strdispatchers.get(name, StrDispatch())\n sdp.add_re(re.compile(re_key), f, priority )\n self.strdispatchers[name] = sdp\n return\n\n dp = getattr(self.hooks, name, None)\n if name not in IPython.core.hooks.__all__:\n print \"Warning! Hook '%s' is not one of %s\" % \\\n (name, IPython.core.hooks.__all__ )\n if not dp:\n dp = IPython.core.hooks.CommandChainDispatcher()\n\n try:\n dp.add(f,priority)\n except AttributeError:\n # it was not commandchain, plain old func - replace\n dp = f\n\n setattr(self.hooks,name, dp)", "response": "Set an internal IPython hook."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef register_post_execute(self, func):\n if not callable(func):\n raise ValueError('argument %s must be callable' % func)\n self._post_execute[func] = True", "response": "Register a function for calling after code execution."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a new main module object for user code execution.", "response": "def new_main_mod(self,ns=None):\n \"\"\"Return a new 'main' module object for user code execution.\n \"\"\"\n main_mod = self._user_main_module\n init_fakemod_dict(main_mod,ns)\n return main_mod"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cache_main_mod(self,ns,fname):\n self._main_ns_cache[os.path.abspath(fname)] = ns.copy()", "response": "Cache a main module s namespace."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall the pydb/pdb debugger. Keywords: - force(False): by default, this routine checks the instance call_pdb flag and does not actually invoke the debugger if the flag is false. The 'force' option forces the debugger to activate even if the flag is false.", "response": "def debugger(self,force=False):\n \"\"\"Call the pydb/pdb debugger.\n\n Keywords:\n\n - force(False): by default, this routine checks the instance call_pdb\n flag and does not actually invoke the debugger if the flag is false.\n The 'force' option forces the debugger to activate even if the flag\n is false.\n \"\"\"\n\n if not (force or self.call_pdb):\n return\n\n if not hasattr(sys,'last_traceback'):\n error('No traceback has been produced, nothing to debug.')\n return\n\n # use pydb if available\n if debugger.has_pydb:\n from pydb import pm\n else:\n # fallback to our internal debugger\n pm = lambda : self.InteractiveTB.debugger(force=True)\n\n with self.readline_no_record:\n pm()"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef prepare_user_module(self, user_module=None, user_ns=None):\n if user_module is None and user_ns is not None:\n user_ns.setdefault(\"__name__\", \"__main__\")\n class DummyMod(object):\n \"A dummy module used for IPython's interactive namespace.\"\n pass\n user_module = DummyMod()\n user_module.__dict__ = user_ns\n \n if user_module is None:\n user_module = types.ModuleType(\"__main__\",\n doc=\"Automatically created module for IPython interactive environment\")\n \n # We must ensure that __builtin__ (without the final 's') is always\n # available and pointing to the __builtin__ *module*. For more details:\n # http://mail.python.org/pipermail/python-dev/2001-April/014068.html\n user_module.__dict__.setdefault('__builtin__', builtin_mod)\n user_module.__dict__.setdefault('__builtins__', builtin_mod)\n \n if user_ns is None:\n user_ns = user_module.__dict__\n\n return user_module, user_ns", "response": "Prepare the user module and namespace in which user code will be run."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize all user - visible user - visible user - namespace variables to their minimum defaults.", "response": "def init_user_ns(self):\n \"\"\"Initialize all user-visible namespaces to their minimum defaults.\n\n Certain history lists are also initialized here, as they effectively\n act as user namespaces.\n\n Notes\n -----\n All data structures here are only filled in, they are NOT reset by this\n method. If they were not empty before, data will simply be added to\n therm.\n \"\"\"\n # This function works in two parts: first we put a few things in\n # user_ns, and we sync that contents into user_ns_hidden so that these\n # initial variables aren't shown by %who. After the sync, we add the\n # rest of what we *do* want the user to see with %who even on a new\n # session (probably nothing, so theye really only see their own stuff)\n\n # The user dict must *always* have a __builtin__ reference to the\n # Python standard __builtin__ namespace, which must be imported.\n # This is so that certain operations in prompt evaluation can be\n # reliably executed with builtins. Note that we can NOT use\n # __builtins__ (note the 's'), because that can either be a dict or a\n # module, and can even mutate at runtime, depending on the context\n # (Python makes no guarantees on it). In contrast, __builtin__ is\n # always a module object, though it must be explicitly imported.\n\n # For more details:\n # http://mail.python.org/pipermail/python-dev/2001-April/014068.html\n ns = dict()\n \n # Put 'help' in the user namespace\n try:\n from site import _Helper\n ns['help'] = _Helper()\n except ImportError:\n warn('help() not available - check site.py')\n\n # make global variables for user access to the histories\n ns['_ih'] = self.history_manager.input_hist_parsed\n ns['_oh'] = self.history_manager.output_hist\n ns['_dh'] = self.history_manager.dir_hist\n\n ns['_sh'] = shadowns\n\n # user aliases to input and output histories. These shouldn't show up\n # in %who, as they can have very large reprs.\n ns['In'] = self.history_manager.input_hist_parsed\n ns['Out'] = self.history_manager.output_hist\n\n # Store myself as the public api!!!\n ns['get_ipython'] = self.get_ipython\n \n ns['exit'] = self.exiter\n ns['quit'] = self.exiter\n\n # Sync what we've added so far to user_ns_hidden so these aren't seen\n # by %who\n self.user_ns_hidden.update(ns)\n\n # Anything put into ns now would show up in %who. Think twice before\n # putting anything here, as we really want %who to show the user their\n # stuff, not our variables.\n\n # Finally, update the real user's namespace\n self.user_ns.update(ns)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget a list of all the namespace dictionaries in which IPython might store a user - created object.", "response": "def all_ns_refs(self):\n \"\"\"Get a list of references to all the namespace dictionaries in which\n IPython might store a user-created object.\n \n Note that this does not include the displayhook, which also caches\n objects from the output.\"\"\"\n return [self.user_ns, self.user_global_ns,\n self._user_main_module.__dict__] + self._main_ns_cache.values()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reset(self, new_session=True):\n # Clear histories\n self.history_manager.reset(new_session)\n # Reset counter used to index all histories\n if new_session:\n self.execution_count = 1\n\n # Flush cached output items\n if self.displayhook.do_full_cache:\n self.displayhook.flush()\n\n # The main execution namespaces must be cleared very carefully,\n # skipping the deletion of the builtin-related keys, because doing so\n # would cause errors in many object's __del__ methods.\n if self.user_ns is not self.user_global_ns:\n self.user_ns.clear()\n ns = self.user_global_ns\n drop_keys = set(ns.keys())\n drop_keys.discard('__builtin__')\n drop_keys.discard('__builtins__')\n drop_keys.discard('__name__')\n for k in drop_keys:\n del ns[k]\n \n self.user_ns_hidden.clear()\n \n # Restore the user namespaces to minimal usability\n self.init_user_ns()\n\n # Restore the default and user aliases\n self.alias_manager.clear_aliases()\n self.alias_manager.init_aliases()\n\n # Flush the private list of module references kept for script\n # execution protection\n self.clear_main_mod_cache()\n\n # Clear out the namespace from the last %run\n self.new_main_mod()", "response": "Reset all internal namespaces and aliases and aliases to their default values."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef del_var(self, varname, by_name=False):\n if varname in ('__builtin__', '__builtins__'):\n raise ValueError(\"Refusing to delete %s\" % varname)\n\n ns_refs = self.all_ns_refs\n \n if by_name: # Delete by name\n for ns in ns_refs:\n try:\n del ns[varname]\n except KeyError:\n pass\n else: # Delete by object\n try:\n obj = self.user_ns[varname]\n except KeyError:\n raise NameError(\"name '%s' is not defined\" % varname)\n # Also check in output history\n ns_refs.append(self.history_manager.output_hist)\n for ns in ns_refs:\n to_delete = [n for n, o in ns.iteritems() if o is obj]\n for name in to_delete:\n del ns[name]\n\n # displayhook keeps extra references, but not in a dictionary\n for name in ('_', '__', '___'):\n if getattr(self.displayhook, name) is obj:\n setattr(self.displayhook, name, None)", "response": "Delete a variable from the various namespaces so that we don t keep any hidden references to it."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nclear selective variables from internal namespaces based on a regular expression pattern.", "response": "def reset_selective(self, regex=None):\n \"\"\"Clear selective variables from internal namespaces based on a\n specified regular expression.\n\n Parameters\n ----------\n regex : string or compiled pattern, optional\n A regular expression pattern that will be used in searching\n variable names in the users namespaces.\n \"\"\"\n if regex is not None:\n try:\n m = re.compile(regex)\n except TypeError:\n raise TypeError('regex must be a string or compiled pattern')\n # Search for keys in each namespace that match the given regex\n # If a match is found, delete the key/value pair.\n for ns in self.all_ns_refs:\n for var in ns:\n if m.search(var):\n del ns[var]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef push(self, variables, interactive=True):\n vdict = None\n\n # We need a dict of name/value pairs to do namespace updates.\n if isinstance(variables, dict):\n vdict = variables\n elif isinstance(variables, (basestring, list, tuple)):\n if isinstance(variables, basestring):\n vlist = variables.split()\n else:\n vlist = variables\n vdict = {}\n cf = sys._getframe(1)\n for name in vlist:\n try:\n vdict[name] = eval(name, cf.f_globals, cf.f_locals)\n except:\n print ('Could not get variable %s from %s' %\n (name,cf.f_code.co_name))\n else:\n raise ValueError('variables must be a dict/str/list/tuple')\n\n # Propagate variables to user namespace\n self.user_ns.update(vdict)\n\n # And configure interactive visibility\n user_ns_hidden = self.user_ns_hidden\n if interactive:\n user_ns_hidden.difference_update(vdict)\n else:\n user_ns_hidden.update(vdict)", "response": "Injects a group of variables into the IPython user namespace."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving a dict of variables from the user namespace if they are the same as the values in the dictionary.", "response": "def drop_by_id(self, variables):\n \"\"\"Remove a dict of variables from the user namespace, if they are the\n same as the values in the dictionary.\n \n This is intended for use by extensions: variables that they've added can\n be taken back out if they are unloaded, without removing any that the\n user has overwritten.\n \n Parameters\n ----------\n variables : dict\n A dictionary mapping object names (as strings) to the objects.\n \"\"\"\n for name, obj in variables.iteritems():\n if name in self.user_ns and self.user_ns[name] is obj:\n del self.user_ns[name]\n self.user_ns_hidden.discard(name)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind an object in the available namespaces.", "response": "def _ofind(self, oname, namespaces=None):\n \"\"\"Find an object in the available namespaces.\n\n self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic\n\n Has special code to detect magic functions.\n \"\"\"\n oname = oname.strip()\n #print '1- oname: <%r>' % oname # dbg\n if not oname.startswith(ESC_MAGIC) and \\\n not oname.startswith(ESC_MAGIC2) and \\\n not py3compat.isidentifier(oname, dotted=True):\n return dict(found=False)\n\n alias_ns = None\n if namespaces is None:\n # Namespaces to search in:\n # Put them in a list. The order is important so that we\n # find things in the same order that Python finds them.\n namespaces = [ ('Interactive', self.user_ns),\n ('Interactive (global)', self.user_global_ns),\n ('Python builtin', builtin_mod.__dict__),\n ('Alias', self.alias_manager.alias_table),\n ]\n alias_ns = self.alias_manager.alias_table\n\n # initialize results to 'null'\n found = False; obj = None; ospace = None; ds = None;\n ismagic = False; isalias = False; parent = None\n\n # We need to special-case 'print', which as of python2.6 registers as a\n # function but should only be treated as one if print_function was\n # loaded with a future import. In this case, just bail.\n if (oname == 'print' and not py3compat.PY3 and not \\\n (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):\n return {'found':found, 'obj':obj, 'namespace':ospace,\n 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}\n\n # Look for the given name by splitting it in parts. If the head is\n # found, then we look for all the remaining parts as members, and only\n # declare success if we can find them all.\n oname_parts = oname.split('.')\n oname_head, oname_rest = oname_parts[0],oname_parts[1:]\n for nsname,ns in namespaces:\n try:\n obj = ns[oname_head]\n except KeyError:\n continue\n else:\n #print 'oname_rest:', oname_rest # dbg\n for part in oname_rest:\n try:\n parent = obj\n obj = getattr(obj,part)\n except:\n # Blanket except b/c some badly implemented objects\n # allow __getattr__ to raise exceptions other than\n # AttributeError, which then crashes IPython.\n break\n else:\n # If we finish the for loop (no break), we got all members\n found = True\n ospace = nsname\n if ns == alias_ns:\n isalias = True\n break # namespace loop\n\n # Try to see if it's magic\n if not found:\n obj = None\n if oname.startswith(ESC_MAGIC2):\n oname = oname.lstrip(ESC_MAGIC2)\n obj = self.find_cell_magic(oname)\n elif oname.startswith(ESC_MAGIC):\n oname = oname.lstrip(ESC_MAGIC)\n obj = self.find_line_magic(oname)\n else:\n # search without prefix, so run? will find %run?\n obj = self.find_line_magic(oname)\n if obj is None:\n obj = self.find_cell_magic(oname)\n if obj is not None:\n found = True\n ospace = 'IPython internal'\n ismagic = True\n\n # Last try: special-case some literals like '', [], {}, etc:\n if not found and oname_head in [\"''\",'\"\"','[]','{}','()']:\n obj = eval(oname_head)\n found = True\n ospace = 'Interactive'\n\n return {'found':found, 'obj':obj, 'namespace':ospace,\n 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nseconding part of object finding, to look for property details.", "response": "def _ofind_property(self, oname, info):\n \"\"\"Second part of object finding, to look for property details.\"\"\"\n if info.found:\n # Get the docstring of the class property if it exists.\n path = oname.split('.')\n root = '.'.join(path[:-1])\n if info.parent is not None:\n try:\n target = getattr(info.parent, '__class__')\n # The object belongs to a class instance.\n try:\n target = getattr(target, path[-1])\n # The class defines the object.\n if isinstance(target, property):\n oname = root + '.__class__.' + path[-1]\n info = Struct(self._ofind(oname))\n except AttributeError: pass\n except AttributeError: pass\n\n # We return either the new info or the unmodified input if the object\n # hadn't been found\n return info"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _object_find(self, oname, namespaces=None):\n inf = Struct(self._ofind(oname, namespaces))\n return Struct(self._ofind_property(oname, inf))", "response": "Find an object and return a struct with info about it."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset up the command history and starts regular autosaves.", "response": "def init_history(self):\n \"\"\"Sets up the command history, and starts regular autosaves.\"\"\"\n self.history_manager = HistoryManager(shell=self, config=self.config)\n self.configurables.append(self.history_manager)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting a custom exception handler for the specified exception classes.", "response": "def set_custom_exc(self, exc_tuple, handler):\n \"\"\"set_custom_exc(exc_tuple,handler)\n\n Set a custom exception handler, which will be called if any of the\n exceptions in exc_tuple occur in the mainloop (specifically, in the\n run_code() method).\n\n Parameters\n ----------\n\n exc_tuple : tuple of exception classes\n A *tuple* of exception classes, for which to call the defined\n handler. It is very important that you use a tuple, and NOT A\n LIST here, because of the way Python's except statement works. If\n you only want to trap a single exception, use a singleton tuple::\n\n exc_tuple == (MyCustomException,)\n\n handler : callable\n handler must have the following signature::\n\n def my_handler(self, etype, value, tb, tb_offset=None):\n ...\n return structured_traceback\n\n Your handler must return a structured traceback (a list of strings),\n or None.\n\n This will be made into an instance method (via types.MethodType)\n of IPython itself, and it will be called if any of the exceptions\n listed in the exc_tuple are caught. If the handler is None, an\n internal basic one is used, which just prints basic info.\n\n To protect IPython from crashes, if your handler ever raises an\n exception or returns an invalid result, it will be immediately\n disabled.\n\n WARNING: by putting in your own exception handler into IPython's main\n execution loop, you run a very good chance of nasty crashes. This\n facility should only be used if you really know what you are doing.\"\"\"\n\n assert type(exc_tuple)==type(()) , \\\n \"The custom exceptions must be given AS A TUPLE.\"\n\n def dummy_handler(self,etype,value,tb,tb_offset=None):\n print '*** Simple custom exception handler ***'\n print 'Exception type :',etype\n print 'Exception value:',value\n print 'Traceback :',tb\n #print 'Source code :','\\n'.join(self.buffer)\n \n def validate_stb(stb):\n \"\"\"validate structured traceback return type\n \n return type of CustomTB *should* be a list of strings, but allow\n single strings or None, which are harmless.\n \n This function will *always* return a list of strings,\n and will raise a TypeError if stb is inappropriate.\n \"\"\"\n msg = \"CustomTB must return list of strings, not %r\" % stb\n if stb is None:\n return []\n elif isinstance(stb, basestring):\n return [stb]\n elif not isinstance(stb, list):\n raise TypeError(msg)\n # it's a list\n for line in stb:\n # check every element\n if not isinstance(line, basestring):\n raise TypeError(msg)\n return stb\n\n if handler is None:\n wrapped = dummy_handler\n else:\n def wrapped(self,etype,value,tb,tb_offset=None):\n \"\"\"wrap CustomTB handler, to protect IPython from user code\n \n This makes it harder (but not impossible) for custom exception\n handlers to crash IPython.\n \"\"\"\n try:\n stb = handler(self,etype,value,tb,tb_offset=tb_offset)\n return validate_stb(stb)\n except:\n # clear custom handler immediately\n self.set_custom_exc((), None)\n print >> io.stderr, \"Custom TB Handler failed, unregistering\"\n # show the exception in handler first\n stb = self.InteractiveTB.structured_traceback(*sys.exc_info())\n print >> io.stdout, self.InteractiveTB.stb2text(stb)\n print >> io.stdout, \"The original exception:\"\n stb = self.InteractiveTB.structured_traceback(\n (etype,value,tb), tb_offset=tb_offset\n )\n return stb\n\n self.CustomTB = types.MethodType(wrapped,self)\n self.custom_exceptions = exc_tuple"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_exc_info(self, exc_tuple=None):\n if exc_tuple is None:\n etype, value, tb = sys.exc_info()\n else:\n etype, value, tb = exc_tuple\n\n if etype is None:\n if hasattr(sys, 'last_type'):\n etype, value, tb = sys.last_type, sys.last_value, \\\n sys.last_traceback\n \n if etype is None:\n raise ValueError(\"No exception to find\")\n \n # Now store the exception info in sys.last_type etc.\n # WARNING: these variables are somewhat deprecated and not\n # necessarily safe to use in a threaded environment, but tools\n # like pdb depend on their existence, so let's set them. If we\n # find problems in the field, we'll need to revisit their use.\n sys.last_type = etype\n sys.last_value = value\n sys.last_traceback = tb\n \n return etype, value, tb", "response": "get exc_info from a given tuple sys. exc_info sys. last_type sys. last_value sys. last_traceback"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndisplays the traceback of the exception that just occurred.", "response": "def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,\n exception_only=False):\n \"\"\"Display the exception that just occurred.\n\n If nothing is known about the exception, this is the method which\n should be used throughout the code for presenting user tracebacks,\n rather than directly invoking the InteractiveTB object.\n\n A specific showsyntaxerror() also exists, but this method can take\n care of calling it if needed, so unless you are explicitly catching a\n SyntaxError exception, don't try to analyze the stack manually and\n simply call this method.\"\"\"\n\n try:\n try:\n etype, value, tb = self._get_exc_info(exc_tuple)\n except ValueError:\n self.write_err('No traceback available to show.\\n')\n return\n \n if etype is SyntaxError:\n # Though this won't be called by syntax errors in the input\n # line, there may be SyntaxError cases with imported code.\n self.showsyntaxerror(filename)\n elif etype is UsageError:\n self.write_err(\"UsageError: %s\" % value)\n else:\n if exception_only:\n stb = ['An exception has occurred, use %tb to see '\n 'the full traceback.\\n']\n stb.extend(self.InteractiveTB.get_exception_only(etype,\n value))\n else:\n try:\n # Exception classes can customise their traceback - we\n # use this in IPython.parallel for exceptions occurring\n # in the engines. This should return a list of strings.\n stb = value._render_traceback_()\n except Exception:\n stb = self.InteractiveTB.structured_traceback(etype,\n value, tb, tb_offset=tb_offset)\n\n self._showtraceback(etype, value, stb)\n if self.call_pdb:\n # drop into debugger\n self.debugger(force=True)\n return\n\n # Actually show the traceback\n self._showtraceback(etype, value, stb)\n\n except KeyboardInterrupt:\n self.write_err(\"\\nKeyboardInterrupt\\n\")"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndisplays the syntax error that just occurred.", "response": "def showsyntaxerror(self, filename=None):\n \"\"\"Display the syntax error that just occurred.\n\n This doesn't display a stack trace because there isn't one.\n\n If a filename is given, it is stuffed in the exception instead\n of what was there before (because Python's parser always uses\n \"\" when reading from a string).\n \"\"\"\n etype, value, last_traceback = self._get_exc_info()\n\n if filename and etype is SyntaxError:\n try:\n value.filename = filename\n except:\n # Not the format we expect; leave it alone\n pass\n \n stb = self.SyntaxTB.structured_traceback(etype, value, [])\n self._showtraceback(etype, value, stb)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef init_readline(self):\n\n if self.readline_use:\n import IPython.utils.rlineimpl as readline\n\n self.rl_next_input = None\n self.rl_do_indent = False\n\n if not self.readline_use or not readline.have_readline:\n self.has_readline = False\n self.readline = None\n # Set a number of methods that depend on readline to be no-op\n self.readline_no_record = no_op_context\n self.set_readline_completer = no_op\n self.set_custom_completer = no_op\n if self.readline_use:\n warn('Readline services not available or not loaded.')\n else:\n self.has_readline = True\n self.readline = readline\n sys.modules['readline'] = readline\n\n # Platform-specific configuration\n if os.name == 'nt':\n # FIXME - check with Frederick to see if we can harmonize\n # naming conventions with pyreadline to avoid this\n # platform-dependent check\n self.readline_startup_hook = readline.set_pre_input_hook\n else:\n self.readline_startup_hook = readline.set_startup_hook\n\n # Load user's initrc file (readline config)\n # Or if libedit is used, load editrc.\n inputrc_name = os.environ.get('INPUTRC')\n if inputrc_name is None:\n inputrc_name = '.inputrc'\n if readline.uses_libedit:\n inputrc_name = '.editrc'\n inputrc_name = os.path.join(self.home_dir, inputrc_name)\n if os.path.isfile(inputrc_name):\n try:\n readline.read_init_file(inputrc_name)\n except:\n warn('Problems reading readline initialization file <%s>'\n % inputrc_name)\n\n # Configure readline according to user's prefs\n # This is only done if GNU readline is being used. If libedit\n # is being used (as on Leopard) the readline config is\n # not run as the syntax for libedit is different.\n if not readline.uses_libedit:\n for rlcommand in self.readline_parse_and_bind:\n #print \"loading rl:\",rlcommand # dbg\n readline.parse_and_bind(rlcommand)\n\n # Remove some chars from the delimiters list. If we encounter\n # unicode chars, discard them.\n delims = readline.get_completer_delims()\n if not py3compat.PY3:\n delims = delims.encode(\"ascii\", \"ignore\")\n for d in self.readline_remove_delims:\n delims = delims.replace(d, \"\")\n delims = delims.replace(ESC_MAGIC, '')\n readline.set_completer_delims(delims)\n # otherwise we end up with a monster history after a while:\n readline.set_history_length(self.history_length)\n\n self.refill_readline_hist()\n self.readline_no_record = ReadlineNoRecord(self)\n\n # Configure auto-indent for all platforms\n self.set_autoindent(self.autoindent)", "response": "Initialize the readline module."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nhook to be used at the start of each line. Currently it handles auto - indent only.", "response": "def pre_readline(self):\n \"\"\"readline hook to be used at the start of each line.\n\n Currently it handles auto-indent only.\"\"\"\n\n if self.rl_do_indent:\n self.readline.insert_text(self._indent_current_str())\n if self.rl_next_input is not None:\n self.readline.insert_text(self.rl_next_input)\n self.rl_next_input = None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing the completion machinery.", "response": "def init_completer(self):\n \"\"\"Initialize the completion machinery.\n\n This creates completion machinery that can be used by client code,\n either interactively in-process (typically triggered by the readline\n library), programatically (such as in test suites) or out-of-prcess\n (typically over the network by remote frontends).\n \"\"\"\n from IPython.core.completer import IPCompleter\n from IPython.core.completerlib import (module_completer,\n magic_run_completer, cd_completer, reset_completer)\n\n self.Completer = IPCompleter(shell=self,\n namespace=self.user_ns,\n global_namespace=self.user_global_ns,\n alias_table=self.alias_manager.alias_table,\n use_readline=self.has_readline,\n config=self.config,\n )\n self.configurables.append(self.Completer)\n\n # Add custom completers to the basic ones built into IPCompleter\n sdisp = self.strdispatchers.get('complete_command', StrDispatch())\n self.strdispatchers['complete_command'] = sdisp\n self.Completer.custom_completers = sdisp\n\n self.set_hook('complete_command', module_completer, str_key = 'import')\n self.set_hook('complete_command', module_completer, str_key = 'from')\n self.set_hook('complete_command', magic_run_completer, str_key = '%run')\n self.set_hook('complete_command', cd_completer, str_key = '%cd')\n self.set_hook('complete_command', reset_completer, str_key = '%reset')\n\n # Only configure readline if we truly are using readline. IPython can\n # do tab-completion over the network, in GUIs, etc, where readline\n # itself may be absent\n if self.has_readline:\n self.set_readline_completer()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef complete(self, text, line=None, cursor_pos=None):\n\n # Inject names into __builtin__ so we can complete on the added names.\n with self.builtin_trap:\n return self.Completer.complete(text, line, cursor_pos)", "response": "Return the completed text and a list of possible completions."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef set_custom_completer(self, completer, pos=0):\n\n newcomp = types.MethodType(completer,self.Completer)\n self.Completer.matchers.insert(pos,newcomp)", "response": "Adds a new custom completer function."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_completer_frame(self, frame=None):\n if frame:\n self.Completer.namespace = frame.f_locals\n self.Completer.global_namespace = frame.f_globals\n else:\n self.Completer.namespace = self.user_ns\n self.Completer.global_namespace = self.user_global_ns", "response": "Set the frame of the completer."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute the given line magic.", "response": "def run_line_magic(self, magic_name, line):\n \"\"\"Execute the given line magic.\n\n Parameters\n ----------\n magic_name : str\n Name of the desired magic function, without '%' prefix.\n\n line : str\n The rest of the input line as a single string.\n \"\"\"\n fn = self.find_line_magic(magic_name)\n if fn is None:\n cm = self.find_cell_magic(magic_name)\n etpl = \"Line magic function `%%%s` not found%s.\"\n extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '\n 'did you mean that instead?)' % magic_name )\n error(etpl % (magic_name, extra))\n else:\n # Note: this is the distance in the stack to the user's frame.\n # This will need to be updated if the internal calling logic gets\n # refactored, or else we'll be expanding the wrong variables.\n stack_depth = 2\n magic_arg_s = self.var_expand(line, stack_depth)\n # Put magic args in a list so we can call with f(*a) syntax\n args = [magic_arg_s]\n # Grab local namespace if we need it:\n if getattr(fn, \"needs_local_scope\", False):\n args.append(sys._getframe(stack_depth).f_locals)\n with self.builtin_trap:\n result = fn(*args)\n return result"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexecute the given cell magic.", "response": "def run_cell_magic(self, magic_name, line, cell):\n \"\"\"Execute the given cell magic.\n \n Parameters\n ----------\n magic_name : str\n Name of the desired magic function, without '%' prefix.\n\n line : str\n The rest of the first input line as a single string.\n\n cell : str\n The body of the cell as a (possibly multiline) string.\n \"\"\"\n fn = self.find_cell_magic(magic_name)\n if fn is None:\n lm = self.find_line_magic(magic_name)\n etpl = \"Cell magic function `%%%%%s` not found%s.\"\n extra = '' if lm is None else (' (But line magic `%%%s` exists, '\n 'did you mean that instead?)' % magic_name )\n error(etpl % (magic_name, extra))\n else:\n # Note: this is the distance in the stack to the user's frame.\n # This will need to be updated if the internal calling logic gets\n # refactored, or else we'll be expanding the wrong variables.\n stack_depth = 2\n magic_arg_s = self.var_expand(line, stack_depth)\n with self.builtin_trap:\n result = fn(line, cell)\n return result"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef find_magic(self, magic_name, magic_kind='line'):\n return self.magics_manager.magics[magic_kind].get(magic_name)", "response": "Find and return a magic of the given type by name."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef magic(self, arg_s):\n # TODO: should we issue a loud deprecation warning here?\n magic_name, _, magic_arg_s = arg_s.partition(' ')\n magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)\n return self.run_line_magic(magic_name, magic_arg_s)", "response": "Deprecated. Use run_line_magic to call a magic function by name."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndefine a new macro in the user namespace.", "response": "def define_macro(self, name, themacro):\n \"\"\"Define a new macro\n\n Parameters\n ----------\n name : str\n The name of the macro.\n themacro : str or Macro\n The action to do upon invoking the macro. If a string, a new\n Macro object is created by passing the string to it.\n \"\"\"\n\n from IPython.core import macro\n\n if isinstance(themacro, basestring):\n themacro = macro.Macro(themacro)\n if not isinstance(themacro, macro.Macro):\n raise ValueError('A macro must be a string or a Macro instance.')\n self.user_ns[name] = themacro"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalling the given cmd in a subprocess using os. system ArcGIS", "response": "def system_raw(self, cmd):\n \"\"\"Call the given cmd in a subprocess using os.system\n\n Parameters\n ----------\n cmd : str\n Command to execute.\n \"\"\"\n cmd = self.var_expand(cmd, depth=1)\n # protect os.system from UNC paths on Windows, which it can't handle:\n if sys.platform == 'win32':\n from IPython.utils._process_win32 import AvoidUNCPath\n with AvoidUNCPath() as path:\n if path is not None:\n cmd = '\"pushd %s &&\"%s' % (path, cmd)\n cmd = py3compat.unicode_to_str(cmd)\n ec = os.system(cmd)\n else:\n cmd = py3compat.unicode_to_str(cmd)\n ec = os.system(cmd)\n \n # We explicitly do NOT return the subprocess status code, because\n # a non-None value would trigger :func:`sys.displayhook` calls.\n # Instead, we store the exit_code in user_ns.\n self.user_ns['_exit_code'] = ec"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget output from a subprocess.", "response": "def getoutput(self, cmd, split=True, depth=0):\n \"\"\"Get output (possibly including stderr) from a subprocess.\n\n Parameters\n ----------\n cmd : str\n Command to execute (can not end in '&', as background processes are\n not supported.\n split : bool, optional\n If True, split the output into an IPython SList. Otherwise, an\n IPython LSString is returned. These are objects similar to normal\n lists and strings, with a few convenience attributes for easier\n manipulation of line-based output. You can use '?' on them for\n details.\n depth : int, optional\n How many frames above the caller are the local variables which should\n be expanded in the command string? The default (0) assumes that the\n expansion variables are in the stack frame calling this function.\n \"\"\"\n if cmd.rstrip().endswith('&'):\n # this is *far* from a rigorous test\n raise OSError(\"Background processes not supported.\")\n out = getoutput(self.var_expand(cmd, depth=depth+1))\n if split:\n out = SList(out.splitlines())\n else:\n out = LSString(out)\n return out"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprints to the screen the rewritten form of the user s command.", "response": "def auto_rewrite_input(self, cmd):\n \"\"\"Print to the screen the rewritten form of the user's command.\n\n This shows visual feedback by rewriting input lines that cause\n automatic calling to kick in, like::\n\n /f x\n\n into::\n\n ------> f(x)\n\n after the user's input prompt. This helps the user understand that the\n input line was transformed automatically by IPython.\n \"\"\"\n if not self.show_rewritten_input:\n return\n \n rw = self.prompt_manager.render('rewrite') + cmd\n\n try:\n # plain ascii works better w/ pyreadline, on some machines, so\n # we use it and only print uncolored rewrite if we have unicode\n rw = str(rw)\n print >> io.stdout, rw\n except UnicodeEncodeError:\n print \"------> \" + cmd"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef user_variables(self, names):\n out = {}\n user_ns = self.user_ns\n for varname in names:\n try:\n value = repr(user_ns[varname])\n except:\n value = self._simple_error()\n out[varname] = value\n return out", "response": "Get a list of variable names from the user s namespace."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nevaluating a dict of Python expressions in the user s namespace.", "response": "def user_expressions(self, expressions):\n \"\"\"Evaluate a dict of expressions in the user's namespace.\n\n Parameters\n ----------\n expressions : dict\n A dict with string keys and string values. The expression values\n should be valid Python expressions, each of which will be evaluated\n in the user namespace.\n\n Returns\n -------\n A dict, keyed like the input expressions dict, with the repr() of each\n value.\n \"\"\"\n out = {}\n user_ns = self.user_ns\n global_ns = self.user_global_ns\n for key, expr in expressions.iteritems():\n try:\n value = repr(eval(expr, global_ns, user_ns))\n except:\n value = self._simple_error()\n out[key] = value\n return out"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ex(self, cmd):\n with self.builtin_trap:\n exec cmd in self.user_global_ns, self.user_ns", "response": "Execute a normal python statement in user namespace."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nevaluates python expression expr in user namespace.", "response": "def ev(self, expr):\n \"\"\"Evaluate python expression expr in user namespace.\n\n Returns the result of evaluation\n \"\"\"\n with self.builtin_trap:\n return eval(expr, self.user_global_ns, self.user_ns)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef safe_execfile_ipy(self, fname):\n fname = os.path.abspath(os.path.expanduser(fname))\n\n # Make sure we can open the file\n try:\n with open(fname) as thefile:\n pass\n except:\n warn('Could not open file <%s> for safe execution.' % fname)\n return\n\n # Find things also in current directory. This is needed to mimic the\n # behavior of running a script from the system command line, where\n # Python inserts the script's directory into sys.path\n dname = os.path.dirname(fname)\n\n with prepended_to_syspath(dname):\n try:\n with open(fname) as thefile:\n # self.run_cell currently captures all exceptions\n # raised in user code. It would be nice if there were\n # versions of runlines, execfile that did raise, so\n # we could catch the errors.\n self.run_cell(thefile.read(), store_history=False)\n except:\n self.showtraceback()\n warn('Unknown failure executing file: <%s>' % fname)", "response": "Like safe_execfile but for. ipython files with IPython syntax."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _run_cached_cell_magic(self, magic_name, line):\n cell = self._current_cell_magic_body\n self._current_cell_magic_body = None\n return self.run_cell_magic(magic_name, line, cell)", "response": "Special method to call a cell magic with the data stored in self. _current_cell_magic_body\n "} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun a complete IPython cell.", "response": "def run_cell(self, raw_cell, store_history=False, silent=False):\n \"\"\"Run a complete IPython cell.\n\n Parameters\n ----------\n raw_cell : str\n The code (including IPython code such as %magic functions) to run.\n store_history : bool\n If True, the raw and translated cell will be stored in IPython's\n history. For user code calling back into IPython's machinery, this\n should be set to False.\n silent : bool\n If True, avoid side-effets, such as implicit displayhooks, history,\n and logging. silent=True forces store_history=False.\n \"\"\"\n if (not raw_cell) or raw_cell.isspace():\n return\n \n if silent:\n store_history = False\n\n self.input_splitter.push(raw_cell)\n\n # Check for cell magics, which leave state behind. This interface is\n # ugly, we need to do something cleaner later... Now the logic is\n # simply that the input_splitter remembers if there was a cell magic,\n # and in that case we grab the cell body.\n if self.input_splitter.cell_magic_parts:\n self._current_cell_magic_body = \\\n ''.join(self.input_splitter.cell_magic_parts)\n cell = self.input_splitter.source_reset()\n\n with self.builtin_trap:\n prefilter_failed = False\n if len(cell.splitlines()) == 1:\n try:\n # use prefilter_lines to handle trailing newlines\n # restore trailing newline for ast.parse\n cell = self.prefilter_manager.prefilter_lines(cell) + '\\n'\n except AliasError as e:\n error(e)\n prefilter_failed = True\n except Exception:\n # don't allow prefilter errors to crash IPython\n self.showtraceback()\n prefilter_failed = True\n\n # Store raw and processed history\n if store_history:\n self.history_manager.store_inputs(self.execution_count,\n cell, raw_cell)\n if not silent:\n self.logger.log(cell, raw_cell)\n\n if not prefilter_failed:\n # don't run if prefilter failed\n cell_name = self.compile.cache(cell, self.execution_count)\n\n with self.display_trap:\n try:\n code_ast = self.compile.ast_parse(cell,\n filename=cell_name)\n except IndentationError:\n self.showindentationerror()\n if store_history:\n self.execution_count += 1\n return None\n except (OverflowError, SyntaxError, ValueError, TypeError,\n MemoryError):\n self.showsyntaxerror()\n if store_history:\n self.execution_count += 1\n return None\n \n interactivity = \"none\" if silent else self.ast_node_interactivity\n self.run_ast_nodes(code_ast.body, cell_name,\n interactivity=interactivity)\n \n # Execute any registered post-execution functions.\n # unless we are silent\n post_exec = [] if silent else self._post_execute.iteritems()\n \n for func, status in post_exec:\n if self.disable_failing_post_execute and not status:\n continue\n try:\n func()\n except KeyboardInterrupt:\n print >> io.stderr, \"\\nKeyboardInterrupt\"\n except Exception:\n # register as failing:\n self._post_execute[func] = False\n self.showtraceback()\n print >> io.stderr, '\\n'.join([\n \"post-execution function %r produced an error.\" % func,\n \"If this problem persists, you can disable failing post-exec functions with:\",\n \"\",\n \" get_ipython().disable_failing_post_execute = True\"\n ])\n\n if store_history:\n # Write output to the database. Does nothing unless\n # history output logging is enabled.\n self.history_manager.store_output(self.execution_count)\n # Each cell is a *single* input, regardless of how many lines it has\n self.execution_count += 1"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr'):\n if not nodelist:\n return\n\n if interactivity == 'last_expr':\n if isinstance(nodelist[-1], ast.Expr):\n interactivity = \"last\"\n else:\n interactivity = \"none\"\n\n if interactivity == 'none':\n to_run_exec, to_run_interactive = nodelist, []\n elif interactivity == 'last':\n to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]\n elif interactivity == 'all':\n to_run_exec, to_run_interactive = [], nodelist\n else:\n raise ValueError(\"Interactivity was %r\" % interactivity)\n\n exec_count = self.execution_count\n\n try:\n for i, node in enumerate(to_run_exec):\n mod = ast.Module([node])\n code = self.compile(mod, cell_name, \"exec\")\n if self.run_code(code):\n return True\n\n for i, node in enumerate(to_run_interactive):\n mod = ast.Interactive([node])\n code = self.compile(mod, cell_name, \"single\")\n if self.run_code(code):\n return True\n\n # Flush softspace\n if softspace(sys.stdout, 0):\n print\n\n except:\n # It's possible to have exceptions raised here, typically by\n # compilation of odd code (such as a naked 'return' outside a\n # function) that did parse but isn't valid. Typically the exception\n # is a SyntaxError, but it's safest just to catch anything and show\n # the user a traceback.\n\n # We do only one try/except outside the loop to minimize the impact\n # on runtime, and also because if any node in the node list is\n # broken, we should stop execution completely.\n self.showtraceback()\n\n return False", "response": "Runs a sequence of AST nodes."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run_code(self, code_obj):\n\n # Set our own excepthook in case the user code tries to call it\n # directly, so that the IPython crash handler doesn't get triggered\n old_excepthook,sys.excepthook = sys.excepthook, self.excepthook\n\n # we save the original sys.excepthook in the instance, in case config\n # code (such as magics) needs access to it.\n self.sys_excepthook = old_excepthook\n outflag = 1 # happens in more places, so it's easier as default\n try:\n try:\n self.hooks.pre_run_code_hook()\n #rprint('Running code', repr(code_obj)) # dbg\n exec code_obj in self.user_global_ns, self.user_ns\n finally:\n # Reset our crash handler in place\n sys.excepthook = old_excepthook\n except SystemExit:\n self.showtraceback(exception_only=True)\n warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", level=1)\n except self.custom_exceptions:\n etype,value,tb = sys.exc_info()\n self.CustomTB(etype,value,tb)\n except:\n self.showtraceback()\n else:\n outflag = 0\n return outflag", "response": "Execute a compiled code object and return the exit status."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef enable_pylab(self, gui=None, import_all=True):\n from IPython.core.pylabtools import mpl_runner\n # We want to prevent the loading of pylab to pollute the user's\n # namespace as shown by the %who* magics, so we execute the activation\n # code in an empty namespace, and we update *both* user_ns and\n # user_ns_hidden with this information.\n ns = {}\n try:\n gui = pylab_activate(ns, gui, import_all, self)\n except KeyError:\n error(\"Backend %r not supported\" % gui)\n return\n self.user_ns.update(ns)\n self.user_ns_hidden.update(ns)\n # Now we must activate the gui pylab wants to use, and fix %run to take\n # plot updates into account\n self.enable_gui(gui)\n self.magics_manager.registry['ExecutionMagics'].default_runner = \\\n mpl_runner(self.safe_execfile)", "response": "Activate pylab support at runtime."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef var_expand(self, cmd, depth=0, formatter=DollarFormatter()):\n ns = self.user_ns.copy()\n ns.update(sys._getframe(depth+1).f_locals)\n ns.pop('self', None)\n try:\n cmd = formatter.format(cmd, **ns)\n except Exception:\n # if formatter couldn't format, just let it go untransformed\n pass\n return cmd", "response": "Expand python variables in a string."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef mktempfile(self, data=None, prefix='ipython_edit_'):\n\n filename = tempfile.mktemp('.py', prefix)\n self.tempfiles.append(filename)\n\n if data:\n tmp_file = open(filename,'w')\n tmp_file.write(data)\n tmp_file.close()\n return filename", "response": "Make a new tempfile and return its filename."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef extract_input_lines(self, range_str, raw=False):\n lines = self.history_manager.get_range_by_str(range_str, raw=raw)\n return \"\\n\".join(x for _, _, x in lines)", "response": "Return as a string a set of input history slices."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_user_code(self, target, raw=True, py_only=False):\n code = self.extract_input_lines(target, raw=raw) # Grab history\n if code:\n return code\n utarget = unquote_filename(target)\n try:\n if utarget.startswith(('http://', 'https://')):\n return openpy.read_py_url(utarget, skip_encoding_cookie=True)\n except UnicodeDecodeError:\n if not py_only :\n response = urllib.urlopen(target)\n return response.read().decode('latin1')\n raise ValueError((\"'%s' seem to be unreadable.\") % utarget)\n\n potential_target = [target]\n try :\n potential_target.insert(0,get_py_filename(target))\n except IOError:\n pass\n\n for tgt in potential_target :\n if os.path.isfile(tgt): # Read file\n try :\n return openpy.read_py_file(tgt, skip_encoding_cookie=True)\n except UnicodeDecodeError :\n if not py_only :\n with io_open(tgt,'r', encoding='latin1') as f :\n return f.read()\n raise ValueError((\"'%s' seem to be unreadable.\") % target)\n\n try: # User namespace\n codeobj = eval(target, self.user_ns)\n except Exception:\n raise ValueError((\"'%s' was not found in history, as a file, url, \"\n \"nor in the user namespace.\") % target)\n if isinstance(codeobj, basestring):\n return codeobj\n elif isinstance(codeobj, Macro):\n return codeobj.value\n\n raise TypeError(\"%s is neither a string nor a macro.\" % target,\n codeobj)", "response": "Get a code string from history file filename or raw history."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef atexit_operations(self):\n # Close the history session (this stores the end time and line count)\n # this must be *before* the tempfile cleanup, in case of temporary\n # history db\n self.history_manager.end_session()\n\n # Cleanup all tempfiles left around\n for tfile in self.tempfiles:\n try:\n os.unlink(tfile)\n except OSError:\n pass\n\n # Clear all user namespaces to release all references cleanly.\n self.reset(new_session=False)\n\n # Run user hooks\n self.hooks.shutdown_hook()", "response": "This will be executed at the time of exit."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef broadcast(client, sender, msg_name, dest_name=None, block=None):\n dest_name = msg_name if dest_name is None else dest_name\n client[sender].execute('com.publish(%s)'%msg_name, block=None)\n targets = client.ids\n targets.remove(sender)\n return client[targets].execute('%s=com.consume()'%dest_name, block=None)", "response": "broadcast a message from one engine to all others."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsends a message from one to one - or - more engines.", "response": "def send(client, sender, targets, msg_name, dest_name=None, block=None):\n \"\"\"send a message from one to one-or-more engines.\"\"\"\n dest_name = msg_name if dest_name is None else dest_name\n def _send(targets, m_name):\n msg = globals()[m_name]\n return com.send(targets, msg)\n \n client[sender].apply_async(_send, targets, msg_name)\n \n return client[targets].execute('%s=com.recv()'%dest_name, block=None)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmakes function raise SkipTest exception if a given condition is true. If the condition is a callable, it is used at runtime to dynamically make the decision. This is useful for tests that may require costly imports, to delay the cost until the test suite is actually executed. Parameters ---------- skip_condition : bool or callable Flag to determine whether to skip the decorated test. msg : str, optional Message to give on raising a SkipTest exception. Default is None. Returns ------- decorator : function Decorator which, when applied to a function, causes SkipTest to be raised when `skip_condition` is True, and the function to be called normally otherwise. Notes ----- The decorator itself is decorated with the ``nose.tools.make_decorator`` function in order to transmit function name, and various other metadata.", "response": "def skipif(skip_condition, msg=None):\n \"\"\"\n Make function raise SkipTest exception if a given condition is true.\n\n If the condition is a callable, it is used at runtime to dynamically\n make the decision. This is useful for tests that may require costly\n imports, to delay the cost until the test suite is actually executed.\n\n Parameters\n ----------\n skip_condition : bool or callable\n Flag to determine whether to skip the decorated test.\n msg : str, optional\n Message to give on raising a SkipTest exception. Default is None.\n\n Returns\n -------\n decorator : function\n Decorator which, when applied to a function, causes SkipTest\n to be raised when `skip_condition` is True, and the function\n to be called normally otherwise.\n\n Notes\n -----\n The decorator itself is decorated with the ``nose.tools.make_decorator``\n function in order to transmit function name, and various other metadata.\n\n \"\"\"\n\n def skip_decorator(f):\n # Local import to avoid a hard nose dependency and only incur the\n # import time overhead at actual test-time.\n import nose\n\n # Allow for both boolean or callable skip conditions.\n if callable(skip_condition):\n skip_val = lambda : skip_condition()\n else:\n skip_val = lambda : skip_condition\n\n def get_msg(func,msg=None):\n \"\"\"Skip message with information about function being skipped.\"\"\"\n if msg is None:\n out = 'Test skipped due to test condition'\n else:\n out = '\\n'+msg\n\n return \"Skipping test: %s%s\" % (func.__name__,out)\n\n # We need to define *two* skippers because Python doesn't allow both\n # return with value and yield inside the same function.\n def skipper_func(*args, **kwargs):\n \"\"\"Skipper for normal test functions.\"\"\"\n if skip_val():\n raise nose.SkipTest(get_msg(f,msg))\n else:\n return f(*args, **kwargs)\n\n def skipper_gen(*args, **kwargs):\n \"\"\"Skipper for test generators.\"\"\"\n if skip_val():\n raise nose.SkipTest(get_msg(f,msg))\n else:\n for x in f(*args, **kwargs):\n yield x\n\n # Choose the right skipper to use when building the actual decorator.\n if nose.util.isgenerator(f):\n skipper = skipper_gen\n else:\n skipper = skipper_func\n\n return nose.tools.make_decorator(f)(skipper)\n\n return skip_decorator"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef knownfailureif(fail_condition, msg=None):\n if msg is None:\n msg = 'Test skipped due to known failure'\n\n # Allow for both boolean or callable known failure conditions.\n if callable(fail_condition):\n fail_val = lambda : fail_condition()\n else:\n fail_val = lambda : fail_condition\n\n def knownfail_decorator(f):\n # Local import to avoid a hard nose dependency and only incur the\n # import time overhead at actual test-time.\n import nose\n def knownfailer(*args, **kwargs):\n if fail_val():\n raise KnownFailureTest, msg\n else:\n return f(*args, **kwargs)\n return nose.tools.make_decorator(f)(knownfailer)\n\n return knownfail_decorator", "response": "Decorator that makes a function raise KnownFailureTest exception if given condition is true."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef deprecated(conditional=True):\n def deprecate_decorator(f):\n # Local import to avoid a hard nose dependency and only incur the\n # import time overhead at actual test-time.\n import nose\n\n def _deprecated_imp(*args, **kwargs):\n # Poor man's replacement for the with statement\n ctx = WarningManager(record=True)\n l = ctx.__enter__()\n warnings.simplefilter('always')\n try:\n f(*args, **kwargs)\n if not len(l) > 0:\n raise AssertionError(\"No warning raised when calling %s\"\n % f.__name__)\n if not l[0].category is DeprecationWarning:\n raise AssertionError(\"First warning for %s is not a \" \\\n \"DeprecationWarning( is %s)\" % (f.__name__, l[0]))\n finally:\n ctx.__exit__()\n\n if callable(conditional):\n cond = conditional()\n else:\n cond = conditional\n if cond:\n return nose.tools.make_decorator(f)(_deprecated_imp)\n else:\n return f\n return deprecate_decorator", "response": "A function decorator that marks a test as deprecated."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef list_profiles_in(path):\n files = os.listdir(path)\n profiles = []\n for f in files:\n full_path = os.path.join(path, f)\n if os.path.isdir(full_path) and f.startswith('profile_'):\n profiles.append(f.split('_',1)[-1])\n return profiles", "response": "list profiles in a given root directory"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nlist profiles that are bundled with IPython.", "response": "def list_bundled_profiles():\n \"\"\"list profiles that are bundled with IPython.\"\"\"\n path = os.path.join(get_ipython_package_dir(), u'config', u'profile')\n files = os.listdir(path)\n profiles = []\n for profile in files:\n full_path = os.path.join(path, profile)\n if os.path.isdir(full_path) and profile != \"__pycache__\":\n profiles.append(profile)\n return profiles"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nensures that the directory at path is created.", "response": "def _bypass_ensure_directory(path, mode=0o777):\n \"\"\"Sandbox-bypassing version of ensure_directory()\"\"\"\n if not WRITE_SUPPORT:\n raise IOError('\"os.mkdir\" not supported on this platform.')\n dirname, filename = split(path)\n if dirname and filename and not isdir(dirname):\n _bypass_ensure_directory(dirname)\n mkdir(dirname, mode)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfinding a distribution matching requirement req.", "response": "def find(self, req):\n \"\"\"Find a distribution matching requirement `req`\n\n If there is an active distribution for the requested project, this\n returns it as long as it meets the version requirement specified by\n `req`. But, if there is an active distribution for the project and it\n does *not* meet the `req` requirement, ``VersionConflict`` is raised.\n If there is no active distribution for the requested project, ``None``\n is returned.\n \"\"\"\n dist = self.by_key.get(req.key)\n if dist is not None and dist not in req:\n # XXX add more info\n raise VersionConflict(dist, req)\n else:\n return dist"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef resolve(self, requirements, env=None, installer=None,\n replace_conflicting=False):\n \"\"\"List all distributions needed to (recursively) meet `requirements`\n\n `requirements` must be a sequence of ``Requirement`` objects. `env`,\n if supplied, should be an ``Environment`` instance. If\n not supplied, it defaults to all distributions available within any\n entry or distribution in the working set. `installer`, if supplied,\n will be invoked with each requirement that cannot be met by an\n already-installed distribution; it should return a ``Distribution`` or\n ``None``.\n\n Unless `replace_conflicting=True`, raises a VersionConflict exception if\n any requirements are found on the path that have the correct name but\n the wrong version. Otherwise, if an `installer` is supplied it will be\n invoked to obtain the correct version of the requirement and activate\n it.\n \"\"\"\n\n # set up the stack\n requirements = list(requirements)[::-1]\n # set of processed requirements\n processed = {}\n # key -> dist\n best = {}\n to_activate = []\n\n # Mapping of requirement to set of distributions that required it;\n # useful for reporting info about conflicts.\n required_by = collections.defaultdict(set)\n\n while requirements:\n # process dependencies breadth-first\n req = requirements.pop(0)\n if req in processed:\n # Ignore cyclic or redundant dependencies\n continue\n dist = best.get(req.key)\n if dist is None:\n # Find the best distribution and add it to the map\n dist = self.by_key.get(req.key)\n if dist is None or (dist not in req and replace_conflicting):\n ws = self\n if env is None:\n if dist is None:\n env = Environment(self.entries)\n else:\n # Use an empty environment and workingset to avoid\n # any further conflicts with the conflicting\n # distribution\n env = Environment([])\n ws = WorkingSet([])\n dist = best[req.key] = env.best_match(req, ws, installer)\n if dist is None:\n #msg = (\"The '%s' distribution was not found on this \"\n # \"system, and is required by this application.\")\n #raise DistributionNotFound(msg % req)\n\n # unfortunately, zc.buildout uses a str(err)\n # to get the name of the distribution here..\n raise DistributionNotFound(req)\n to_activate.append(dist)\n if dist not in req:\n # Oops, the \"best\" so far conflicts with a dependency\n tmpl = \"%s is installed but %s is required by %s\"\n args = dist, req, list(required_by.get(req, []))\n raise VersionConflict(tmpl % args)\n\n # push the new requirements onto the stack\n new_requirements = dist.requires(req.extras)[::-1]\n requirements.extend(new_requirements)\n\n # Register the new requirements needed by req\n for new_requirement in new_requirements:\n required_by[new_requirement].add(req.project_name)\n\n processed[req] = True\n\n # return list of distros to activate\n return to_activate", "response": "Resolves all of the requirements in the working set."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef is_invalid_marker(cls, text):\n try:\n cls.evaluate_marker(text)\n except SyntaxError:\n return cls.normalize_exception(sys.exc_info()[1])\n return False", "response": "Validate text as a PEP 426 environment marker ; return an exception\n if invalid or False otherwise."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None,\n logfile=None, cwd=None, env=None, encoding='utf-8'):\n\n \"\"\"\n This function runs the given command; waits for it to finish; then\n returns all output as a string. STDERR is included in output. If the full\n path to the command is not given then the path is searched.\n\n Note that lines are terminated by CR/LF (\\\\r\\\\n) combination even on\n UNIX-like systems because this is the standard for pseudo ttys. If you set\n 'withexitstatus' to true, then run will return a tuple of (command_output,\n exitstatus). If 'withexitstatus' is false then this returns just\n command_output.\n\n The run() function can often be used instead of creating a spawn instance.\n For example, the following code uses spawn::\n\n from pexpect import *\n child = spawn('scp foo myname@host.example.com:.')\n child.expect ('(?i)password')\n child.sendline (mypassword)\n\n The previous code can be replace with the following::\n\n from pexpect import *\n run ('scp foo myname@host.example.com:.', events={'(?i)password': mypassword})\n\n Examples\n ========\n\n Start the apache daemon on the local machine::\n\n from pexpect import *\n run (\"/usr/local/apache/bin/apachectl start\")\n\n Check in a file using SVN::\n\n from pexpect import *\n run (\"svn ci -m 'automatic commit' my_file.py\")\n\n Run a command and capture exit status::\n\n from pexpect import *\n (command_output, exitstatus) = run ('ls -l /bin', withexitstatus=1)\n\n Tricky Examples\n ===============\n\n The following will run SSH and execute 'ls -l' on the remote machine. The\n password 'secret' will be sent if the '(?i)password' pattern is ever seen::\n\n run (\"ssh username@machine.example.com 'ls -l'\", events={'(?i)password':'secret\\\\n'})\n\n This will start mencoder to rip a video from DVD. This will also display\n progress ticks every 5 seconds as it runs. For example::\n\n from pexpect import *\n def print_ticks(d):\n print d['event_count'],\n run (\"mencoder dvd://1 -o video.avi -oac copy -ovc copy\", events={TIMEOUT:print_ticks}, timeout=5)\n\n The 'events' argument should be a dictionary of patterns and responses.\n Whenever one of the patterns is seen in the command out run() will send the\n associated response string. Note that you should put newlines in your\n string if Enter is necessary. The responses may also contain callback\n functions. Any callback is function that takes a dictionary as an argument.\n The dictionary contains all the locals from the run() function, so you can\n access the child spawn object or any other variable defined in run()\n (event_count, child, and extra_args are the most useful). A callback may\n return True to stop the current run process otherwise run() continues until\n the next event. A callback may also return a string which will be sent to\n the child. 'extra_args' is not used by directly run(). It provides a way to\n pass data to a callback function through run() through the locals\n dictionary passed to a callback.\"\"\"\n\n if timeout == -1:\n child = spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env,\n encoding=encoding)\n else:\n child = spawn(command, timeout=timeout, maxread=2000, logfile=logfile,\n cwd=cwd, env=env, encoding=encoding)\n if events is not None:\n patterns = events.keys()\n responses = events.values()\n else:\n patterns=None # We assume that EOF or TIMEOUT will save us.\n responses=None\n child_result_list = []\n event_count = 0\n while 1:\n try:\n index = child.expect (patterns)\n if isinstance(child.after, basestring):\n child_result_list.append(child.before + child.after)\n else: # child.after may have been a TIMEOUT or EOF, so don't cat those.\n child_result_list.append(child.before)\n if isinstance(responses[index], basestring):\n child.send(responses[index])\n elif type(responses[index]) is types.FunctionType:\n callback_result = responses[index](locals())\n sys.stdout.flush()\n if isinstance(callback_result, basestring):\n child.send(callback_result)\n elif callback_result:\n break\n else:\n raise TypeError ('The callback must be a string or function type.')\n event_count = event_count + 1\n except TIMEOUT, e:\n child_result_list.append(child.before)\n break\n except EOF, e:\n child_result_list.append(child.before)\n break\n child_result = child._empty_buffer.join(child_result_list)\n if withexitstatus:\n child.close()\n return (child_result, child.exitstatus)\n else:\n return child_result", "response": "Runs a command and returns the output of the command."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _spawn(self,command,args=[]):\n\n \"\"\"This starts the given command in a child process. This does all the\n fork/exec type of stuff for a pty. This is called by __init__. If args\n is empty then command will be parsed (split on spaces) and args will be\n set to parsed arguments. \"\"\"\n\n # The pid and child_fd of this object get set by this method.\n # Note that it is difficult for this method to fail.\n # You cannot detect if the child process cannot start.\n # So the only way you can tell if the child process started\n # or not is to try to read from the file descriptor. If you get\n # EOF immediately then it means that the child is already dead.\n # That may not necessarily be bad because you may haved spawned a child\n # that performs some task; creates no stdout output; and then dies.\n\n # If command is an int type then it may represent a file descriptor.\n if type(command) == type(0):\n raise ExceptionPexpect ('Command is an int type. If this is a file descriptor then maybe you want to use fdpexpect.fdspawn which takes an existing file descriptor instead of a command string.')\n\n if type (args) != type([]):\n raise TypeError ('The argument, args, must be a list.')\n\n if args == []:\n self.args = split_command_line(command)\n self.command = self.args[0]\n else:\n self.args = args[:] # work with a copy\n self.args.insert (0, command)\n self.command = command\n\n command_with_path = which(self.command)\n if command_with_path is None:\n raise ExceptionPexpect ('The command was not found or was not executable: %s.' % self.command)\n self.command = command_with_path\n self.args[0] = self.command\n\n self.name = '<' + ' '.join (self.args) + '>'\n\n assert self.pid is None, 'The pid member should be None.'\n assert self.command is not None, 'The command member should not be None.'\n\n if self.use_native_pty_fork:\n try:\n self.pid, self.child_fd = pty.fork()\n except OSError, e:\n raise ExceptionPexpect('Error! pty.fork() failed: ' + str(e))\n else: # Use internal __fork_pty\n self.pid, self.child_fd = self.__fork_pty()\n\n if self.pid == 0: # Child\n try:\n self.child_fd = sys.stdout.fileno() # used by setwinsize()\n self.setwinsize(24, 80)\n except:\n # Some platforms do not like setwinsize (Cygwin).\n # This will cause problem when running applications that\n # are very picky about window size.\n # This is a serious limitation, but not a show stopper.\n pass\n # Do not allow child to inherit open file descriptors from parent.\n max_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0]\n for i in range (3, max_fd):\n try:\n os.close (i)\n except OSError:\n pass\n\n # I don't know why this works, but ignoring SIGHUP fixes a\n # problem when trying to start a Java daemon with sudo\n # (specifically, Tomcat).\n signal.signal(signal.SIGHUP, signal.SIG_IGN)\n\n if self.cwd is not None:\n os.chdir(self.cwd)\n if self.env is None:\n os.execv(self.command, self.args)\n else:\n os.execvpe(self.command, self.args, self.env)\n\n # Parent\n self.terminated = False\n self.closed = False", "response": "This method starts a new child process. This method is called by _spawn_child."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __fork_pty(self):\n\n \"\"\"This implements a substitute for the forkpty system call. This\n should be more portable than the pty.fork() function. Specifically,\n this should work on Solaris.\n\n Modified 10.06.05 by Geoff Marshall: Implemented __fork_pty() method to\n resolve the issue with Python's pty.fork() not supporting Solaris,\n particularly ssh. Based on patch to posixmodule.c authored by Noah\n Spurrier::\n\n http://mail.python.org/pipermail/python-dev/2003-May/035281.html\n\n \"\"\"\n\n parent_fd, child_fd = os.openpty()\n if parent_fd < 0 or child_fd < 0:\n raise ExceptionPexpect, \"Error! Could not open pty with os.openpty().\"\n\n pid = os.fork()\n if pid < 0:\n raise ExceptionPexpect, \"Error! Failed os.fork().\"\n elif pid == 0:\n # Child.\n os.close(parent_fd)\n self.__pty_make_controlling_tty(child_fd)\n\n os.dup2(child_fd, 0)\n os.dup2(child_fd, 1)\n os.dup2(child_fd, 2)\n\n if child_fd > 2:\n os.close(child_fd)\n else:\n # Parent.\n os.close(child_fd)\n\n return pid, parent_fd", "response": "This method implements a substitute for the forkpty system call."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncloses the child application.", "response": "def close (self, force=True): # File-like object.\n\n \"\"\"This closes the connection with the child application. Note that\n calling close() more than once is valid. This emulates standard Python\n behavior with files. Set force to True if you want to make sure that\n the child is terminated (SIGKILL is sent if the child ignores SIGHUP\n and SIGINT). \"\"\"\n\n if not self.closed:\n self.flush()\n os.close (self.child_fd)\n time.sleep(self.delayafterclose) # Give kernel time to update process status.\n if self.isalive():\n if not self.terminate(force):\n raise ExceptionPexpect ('close() could not terminate the child using terminate()')\n self.child_fd = -1\n self.closed = True"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setecho (self, state):\n\n \"\"\"This sets the terminal echo mode on or off. Note that anything the\n child sent before the echo will be lost, so you should be sure that\n your input buffer is empty before you call setecho(). For example, the\n following will work as expected::\n\n p = pexpect.spawn('cat')\n p.sendline ('1234') # We will see this twice (once from tty echo and again from cat).\n p.expect (['1234'])\n p.expect (['1234'])\n p.setecho(False) # Turn off tty echo\n p.sendline ('abcd') # We will set this only once (echoed by cat).\n p.sendline ('wxyz') # We will set this only once (echoed by cat)\n p.expect (['abcd'])\n p.expect (['wxyz'])\n\n The following WILL NOT WORK because the lines sent before the setecho\n will be lost::\n\n p = pexpect.spawn('cat')\n p.sendline ('1234') # We will see this twice (once from tty echo and again from cat).\n p.setecho(False) # Turn off tty echo\n p.sendline ('abcd') # We will set this only once (echoed by cat).\n p.sendline ('wxyz') # We will set this only once (echoed by cat)\n p.expect (['1234'])\n p.expect (['1234'])\n p.expect (['abcd'])\n p.expect (['wxyz'])\n \"\"\"\n\n self.child_fd\n attr = termios.tcgetattr(self.child_fd)\n if state:\n attr[3] = attr[3] | termios.ECHO\n else:\n attr[3] = attr[3] & ~termios.ECHO\n # I tried TCSADRAIN and TCSAFLUSH, but these were inconsistent\n # and blocked on some platforms. TCSADRAIN is probably ideal if it worked.\n termios.tcsetattr(self.child_fd, termios.TCSANOW, attr)", "response": "This method sets the terminal echo mode on or off."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read_nonblocking (self, size = 1, timeout = -1):\n\n \"\"\"This reads at most size bytes from the child application. It\n includes a timeout. If the read does not complete within the timeout\n period then a TIMEOUT exception is raised. If the end of file is read\n then an EOF exception will be raised. If a log file was set using\n setlog() then all data will also be written to the log file.\n\n If timeout is None then the read may block indefinitely. If timeout is -1\n then the self.timeout value is used. If timeout is 0 then the child is\n polled and if there was no data immediately ready then this will raise\n a TIMEOUT exception.\n\n The timeout refers only to the amount of time to read at least one\n character. This is not effected by the 'size' parameter, so if you call\n read_nonblocking(size=100, timeout=30) and only one character is\n available right away then one character will be returned immediately.\n It will not wait for 30 seconds for another 99 characters to come in.\n\n This is a wrapper around os.read(). It uses select.select() to\n implement the timeout. \"\"\"\n\n if self.closed:\n raise ValueError ('I/O operation on closed file in read_nonblocking().')\n\n if timeout == -1:\n timeout = self.timeout\n\n # Note that some systems such as Solaris do not give an EOF when\n # the child dies. In fact, you can still try to read\n # from the child_fd -- it will block forever or until TIMEOUT.\n # For this case, I test isalive() before doing any reading.\n # If isalive() is false, then I pretend that this is the same as EOF.\n if not self.isalive():\n r,w,e = self.__select([self.child_fd], [], [], 0) # timeout of 0 means \"poll\"\n if not r:\n self.flag_eof = True\n raise EOF ('End Of File (EOF) in read_nonblocking(). Braindead platform.')\n elif self.__irix_hack:\n # This is a hack for Irix. It seems that Irix requires a long delay before checking isalive.\n # This adds a 2 second delay, but only when the child is terminated.\n r, w, e = self.__select([self.child_fd], [], [], 2)\n if not r and not self.isalive():\n self.flag_eof = True\n raise EOF ('End Of File (EOF) in read_nonblocking(). Pokey platform.')\n\n r,w,e = self.__select([self.child_fd], [], [], timeout)\n\n if not r:\n if not self.isalive():\n # Some platforms, such as Irix, will claim that their processes are alive;\n # then timeout on the select; and then finally admit that they are not alive.\n self.flag_eof = True\n raise EOF ('End of File (EOF) in read_nonblocking(). Very pokey platform.')\n else:\n raise TIMEOUT ('Timeout exceeded in read_nonblocking().')\n\n if self.child_fd in r:\n try:\n s = os.read(self.child_fd, size)\n except OSError, e: # Linux does this\n self.flag_eof = True\n raise EOF ('End Of File (EOF) in read_nonblocking(). Exception style platform.')\n if s == b'': # BSD style\n self.flag_eof = True\n raise EOF ('End Of File (EOF) in read_nonblocking(). Empty string style platform.')\n\n s2 = self._cast_buffer_type(s)\n if self.logfile is not None:\n self.logfile.write(s2)\n self.logfile.flush()\n if self.logfile_read is not None:\n self.logfile_read.write(s2)\n self.logfile_read.flush()\n\n return s\n\n raise ExceptionPexpect ('Reached an unexpected state in read_nonblocking().')", "response": "This function reads at most size bytes from the child application."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read (self, size = -1): # File-like object.\n\n if size == 0:\n return self._empty_buffer\n if size < 0:\n self.expect (self.delimiter) # delimiter default is EOF\n return self.before\n\n # I could have done this more directly by not using expect(), but\n # I deliberately decided to couple read() to expect() so that\n # I would catch any bugs early and ensure consistant behavior.\n # It's a little less efficient, but there is less for me to\n # worry about if I have to later modify read() or expect().\n # Note, it's OK if size==-1 in the regex. That just means it\n # will never match anything in which case we stop only on EOF.\n if self._buffer_type is bytes:\n pat = (u'.{%d}' % size).encode('ascii')\n else:\n pat = u'.{%d}' % size\n cre = re.compile(pat, re.DOTALL)\n index = self.expect ([cre, self.delimiter]) # delimiter default is EOF\n if index == 0:\n return self.after ### self.before should be ''. Should I assert this?\n return self.before", "response": "Reads at most size bytes from the file and returns them as a string object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sendcontrol(self, char):\n\n \"\"\"This sends a control character to the child such as Ctrl-C or\n Ctrl-D. For example, to send a Ctrl-G (ASCII 7)::\n\n child.sendcontrol('g')\n\n See also, sendintr() and sendeof().\n \"\"\"\n\n char = char.lower()\n a = ord(char)\n if a>=97 and a<=122:\n a = a - ord('a') + 1\n return self.send (chr(a))\n d = {'@':0, '`':0,\n '[':27, '{':27,\n '\\\\':28, '|':28,\n ']':29, '}': 29,\n '^':30, '~':30,\n '_':31,\n '?':127}\n if char not in d:\n return 0\n return self.send (chr(d[char]))", "response": "This method sends a control character to the child such as Ctrl - C Ctrl - D or Ctrl - G."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sendeof(self):\n\n \"\"\"This sends an EOF to the child. This sends a character which causes\n the pending parent output buffer to be sent to the waiting child\n program without waiting for end-of-line. If it is the first character\n of the line, the read() in the user program returns 0, which signifies\n end-of-file. This means to work as expected a sendeof() has to be\n called at the beginning of a line. This method does not send a newline.\n It is the responsibility of the caller to ensure the eof is sent at the\n beginning of a line. \"\"\"\n\n ### Hmmm... how do I send an EOF?\n ###C if ((m = write(pty, *buf, p - *buf)) < 0)\n ###C return (errno == EWOULDBLOCK) ? n : -1;\n #fd = sys.stdin.fileno()\n #old = termios.tcgetattr(fd) # remember current state\n #attr = termios.tcgetattr(fd)\n #attr[3] = attr[3] | termios.ICANON # ICANON must be set to recognize EOF\n #try: # use try/finally to ensure state gets restored\n # termios.tcsetattr(fd, termios.TCSADRAIN, attr)\n # if hasattr(termios, 'CEOF'):\n # os.write (self.child_fd, '%c' % termios.CEOF)\n # else:\n # # Silly platform does not define CEOF so assume CTRL-D\n # os.write (self.child_fd, '%c' % 4)\n #finally: # restore state\n # termios.tcsetattr(fd, termios.TCSADRAIN, old)\n if hasattr(termios, 'VEOF'):\n char = termios.tcgetattr(self.child_fd)[6][termios.VEOF]\n else:\n # platform does not define VEOF so assume CTRL-D\n char = chr(4)\n self.send(char)", "response": "This method sends an EOF to the child. It sends a character which causes the pending parent output buffer to be sent to the waiting child process to be sent to the waiting child process to be sent to the waiting child process without waiting for end - of - line."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _prepare_regex_pattern(self, p):\n \"Recompile unicode regexes as bytes regexes. Overridden in subclass.\"\n if isinstance(p.pattern, unicode):\n p = re.compile(p.pattern.encode('utf-8'), p.flags &~ re.UNICODE)\n return p", "response": "Recompile unicode regexes as bytes regexes. Overridden in subclass."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef expect_exact(self, pattern_list, timeout = -1, searchwindowsize = -1):\n\n \"\"\"This is similar to expect(), but uses plain string matching instead\n of compiled regular expressions in 'pattern_list'. The 'pattern_list'\n may be a string; a list or other sequence of strings; or TIMEOUT and\n EOF.\n\n This call might be faster than expect() for two reasons: string\n searching is faster than RE matching and it is possible to limit the\n search to just the end of the input buffer.\n\n This method is also useful when you don't want to have to worry about\n escaping regular expression characters that you want to match.\"\"\"\n\n if isinstance(pattern_list, (bytes, unicode)) or pattern_list in (TIMEOUT, EOF):\n pattern_list = [pattern_list]\n return self.expect_loop(searcher_string(pattern_list), timeout, searchwindowsize)", "response": "This method uses plain string matching instead of regular expressions in pattern_list."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setwinsize(self, r, c):\n\n \"\"\"This sets the terminal window size of the child tty. This will cause\n a SIGWINCH signal to be sent to the child. This does not change the\n physical window size. It changes the size reported to TTY-aware\n applications like vi or curses -- applications that respond to the\n SIGWINCH signal. \"\"\"\n\n # Check for buggy platforms. Some Python versions on some platforms\n # (notably OSF1 Alpha and RedHat 7.1) truncate the value for\n # termios.TIOCSWINSZ. It is not clear why this happens.\n # These platforms don't seem to handle the signed int very well;\n # yet other platforms like OpenBSD have a large negative value for\n # TIOCSWINSZ and they don't have a truncate problem.\n # Newer versions of Linux have totally different values for TIOCSWINSZ.\n # Note that this fix is a hack.\n TIOCSWINSZ = getattr(termios, 'TIOCSWINSZ', -2146929561)\n if TIOCSWINSZ == 2148037735L: # L is not required in Python >= 2.2.\n TIOCSWINSZ = -2146929561 # Same bits, but with sign.\n # Note, assume ws_xpixel and ws_ypixel are zero.\n s = struct.pack('HHHH', r, c, 0, 0)\n fcntl.ioctl(self.fileno(), TIOCSWINSZ, s)", "response": "This sets the terminal window size of the child tty. This will cause the child tty to be closed and the child tty will not be closed."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef interact(self, escape_character = b'\\x1d', input_filter = None, output_filter = None):\n\n \"\"\"This gives control of the child process to the interactive user (the\n human at the keyboard). Keystrokes are sent to the child process, and\n the stdout and stderr output of the child process is printed. This\n simply echos the child stdout and child stderr to the real stdout and\n it echos the real stdin to the child stdin. When the user types the\n escape_character this method will stop. The default for\n escape_character is ^]. This should not be confused with ASCII 27 --\n the ESC character. ASCII 29 was chosen for historical merit because\n this is the character used by 'telnet' as the escape character. The\n escape_character will not be sent to the child process.\n\n You may pass in optional input and output filter functions. These\n functions should take a string and return a string. The output_filter\n will be passed all the output from the child process. The input_filter\n will be passed all the keyboard input from the user. The input_filter\n is run BEFORE the check for the escape_character.\n\n Note that if you change the window size of the parent the SIGWINCH\n signal will not be passed through to the child. If you want the child\n window size to change when the parent's window size changes then do\n something like the following example::\n\n import pexpect, struct, fcntl, termios, signal, sys\n def sigwinch_passthrough (sig, data):\n s = struct.pack(\"HHHH\", 0, 0, 0, 0)\n a = struct.unpack('hhhh', fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ , s))\n global p\n p.setwinsize(a[0],a[1])\n p = pexpect.spawn('/bin/bash') # Note this is global and used in sigwinch_passthrough.\n signal.signal(signal.SIGWINCH, sigwinch_passthrough)\n p.interact()\n \"\"\"\n\n # Flush the buffer.\n if PY3: self.stdout.write(_cast_unicode(self.buffer, self.encoding))\n else: self.stdout.write(self.buffer)\n self.stdout.flush()\n self.buffer = self._empty_buffer\n mode = tty.tcgetattr(self.STDIN_FILENO)\n tty.setraw(self.STDIN_FILENO)\n try:\n self.__interact_copy(escape_character, input_filter, output_filter)\n finally:\n tty.tcsetattr(self.STDIN_FILENO, tty.TCSAFLUSH, mode)", "response": "This method interacts with the child process."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __select (self, iwtd, owtd, ewtd, timeout=None):\n\n \"\"\"This is a wrapper around select.select() that ignores signals. If\n select.select raises a select.error exception and errno is an EINTR\n error then it is ignored. Mainly this is used to ignore sigwinch\n (terminal resize). \"\"\"\n\n # if select() is interrupted by a signal (errno==EINTR) then\n # we loop back and enter the select() again.\n if timeout is not None:\n end_time = time.time() + timeout\n while True:\n try:\n return select.select (iwtd, owtd, ewtd, timeout)\n except select.error as e:\n if e.args[0] == errno.EINTR:\n # if we loop back we have to subtract the amount of time we already waited.\n if timeout is not None:\n timeout = end_time - time.time()\n if timeout < 0:\n return ([],[],[])\n else: # something else caused the select.error, so this really is an exception\n raise", "response": "This is a wrapper around select. select that ignores signals and returns a list of the available items."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nrecompiles bytes regexes as unicode regexes.", "response": "def _prepare_regex_pattern(self, p):\n \"Recompile bytes regexes as unicode regexes.\"\n if isinstance(p.pattern, bytes):\n p = re.compile(p.pattern.decode(self.encoding), p.flags)\n return p"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef search(self, buffer, freshlen, searchwindowsize=None):\n\n \"\"\"This searches 'buffer' for the first occurence of one of the search\n strings. 'freshlen' must indicate the number of bytes at the end of\n 'buffer' which have not been searched before. It helps to avoid\n searching the same, possibly big, buffer over and over again.\n\n See class spawn for the 'searchwindowsize' argument.\n\n If there is a match this returns the index of that string, and sets\n 'start', 'end' and 'match'. Otherwise, this returns -1. \"\"\"\n\n absurd_match = len(buffer)\n first_match = absurd_match\n\n # 'freshlen' helps a lot here. Further optimizations could\n # possibly include:\n #\n # using something like the Boyer-Moore Fast String Searching\n # Algorithm; pre-compiling the search through a list of\n # strings into something that can scan the input once to\n # search for all N strings; realize that if we search for\n # ['bar', 'baz'] and the input is '...foo' we need not bother\n # rescanning until we've read three more bytes.\n #\n # Sadly, I don't know enough about this interesting topic. /grahn\n\n for index, s in self._strings:\n if searchwindowsize is None:\n # the match, if any, can only be in the fresh data,\n # or at the very end of the old data\n offset = -(freshlen+len(s))\n else:\n # better obey searchwindowsize\n offset = -searchwindowsize\n n = buffer.find(s, offset)\n if n >= 0 and n < first_match:\n first_match = n\n best_index, best_match = index, s\n if first_match == absurd_match:\n return -1\n self.match = best_match\n self.start = first_match\n self.end = self.start + len(self.match)\n return best_index", "response": "This method searches the given buffer for the first occurence of one of the search\n strings and returns the index of the first occurence of that string."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef search(self, buffer, freshlen, searchwindowsize=None):\n\n \"\"\"This searches 'buffer' for the first occurence of one of the regular\n expressions. 'freshlen' must indicate the number of bytes at the end of\n 'buffer' which have not been searched before.\n\n See class spawn for the 'searchwindowsize' argument.\n\n If there is a match this returns the index of that string, and sets\n 'start', 'end' and 'match'. Otherwise, returns -1.\"\"\"\n\n absurd_match = len(buffer)\n first_match = absurd_match\n # 'freshlen' doesn't help here -- we cannot predict the\n # length of a match, and the re module provides no help.\n if searchwindowsize is None:\n searchstart = 0\n else:\n searchstart = max(0, len(buffer)-searchwindowsize)\n for index, s in self._searches:\n match = s.search(buffer, searchstart)\n if match is None:\n continue\n n = match.start()\n if n < first_match:\n first_match = n\n the_match = match\n best_index = index\n if first_match == absurd_match:\n return -1\n self.start = first_match\n self.match = the_match\n self.end = self.match.end()\n return best_index", "response": "This method searches the regularCOOKIE table for the first occurence of one of the regularCOOKIE classes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfinish up all displayhook activities.", "response": "def finish_displayhook(self):\n \"\"\"Finish up all displayhook activities.\"\"\"\n sys.stdout.flush()\n sys.stderr.flush()\n self.session.send(self.pub_socket, self.msg, ident=self.topic)\n self.msg = None"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef log_listener(log:logging.Logger=None, level=logging.INFO):\n if log is None:\n log = logging.getLogger(\"ProgressMonitor\")\n def listen(monitor):\n name = \"{}: \".format(monitor.name) if monitor.name is not None else \"\"\n perc = int(monitor.progress * 100)\n msg = \"[{name}{perc:3d}%] {monitor.message}\".format(**locals())\n log.log(level, msg)\n return listen", "response": "Progress Monitor listener that logs all updates to the given logger"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nunpacks a directory using the same interface as for archives AttributeNames", "response": "def unpack_directory(filename, extract_dir, progress_filter=default_filter):\n \"\"\"\"Unpack\" a directory, using the same interface as for archives\n\n Raises ``UnrecognizedFormat`` if `filename` is not a directory\n \"\"\"\n if not os.path.isdir(filename):\n raise UnrecognizedFormat(\"%s is not a directory\" % (filename,))\n\n paths = {filename:('',extract_dir)}\n for base, dirs, files in os.walk(filename):\n src,dst = paths[base]\n for d in dirs:\n paths[os.path.join(base,d)] = src+d+'/', os.path.join(dst,d)\n for f in files:\n name = src+f\n target = os.path.join(dst,f)\n target = progress_filter(src+f, target)\n if not target:\n continue # skip non-files\n ensure_directory(target)\n f = os.path.join(base,f)\n shutil.copyfile(f, target)\n shutil.copystat(f, target)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef unpack_tarfile(filename, extract_dir, progress_filter=default_filter):\n\n try:\n tarobj = tarfile.open(filename)\n except tarfile.TarError:\n raise UnrecognizedFormat(\n \"%s is not a compressed or uncompressed tar file\" % (filename,)\n )\n\n try:\n tarobj.chown = lambda *args: None # don't do any chowning!\n for member in tarobj:\n name = member.name\n # don't extract absolute paths or ones with .. in them\n if not name.startswith('/') and '..' not in name:\n prelim_dst = os.path.join(extract_dir, *name.split('/'))\n final_dst = progress_filter(name, prelim_dst)\n # If progress_filter returns None, then we do not extract\n # this file\n # TODO: Do we really need to limit to just these file types?\n # tarobj.extract() will handle all files on all platforms,\n # turning file types that aren't allowed on that platform into\n # regular files.\n if final_dst and (member.isfile() or member.isdir() or\n member.islnk() or member.issym()):\n tarobj.extract(member, extract_dir)\n if final_dst != prelim_dst:\n shutil.move(prelim_dst, final_dst)\n return True\n finally:\n tarobj.close()", "response": "Unpack tar file filename to extract_dir"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nemit a message to the user.", "response": "def emit(self, msg, level=1, debug=False):\n \"\"\"\n Emit a message to the user.\n\n :param msg: The message to emit. If ``debug`` is ``True``,\n the message will be emitted to ``stderr`` only if\n the ``debug`` attribute is ``True``. If ``debug``\n is ``False``, the message will be emitted to\n ``stdout`` under the control of the ``verbose``\n attribute.\n :param level: Ignored if ``debug`` is ``True``. The message\n will only be emitted if the ``verbose``\n attribute is greater than or equal to the value\n of this parameter. Defaults to 1.\n :param debug: If ``True``, marks the message as a debugging\n message. The message will only be emitted if\n the ``debug`` attribute is ``True``.\n \"\"\"\n\n # Is it a debug message?\n if debug:\n if not self.debug:\n # Debugging not enabled, don't emit the message\n return\n stream = sys.stderr\n else:\n # Not a debugging message; is verbose high enough?\n if self.verbose < level:\n return\n stream = sys.stdout\n\n # Emit the message\n print(msg, file=stream)\n stream.flush()"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef template(self, string):\n\n # Short-circuit if the template \"string\" isn't actually a\n # string\n if not isinstance(string, six.string_types):\n return lambda ctxt: string\n\n # Create the template and return the callable\n tmpl = self._jinja.from_string(string)\n return lambda ctxt: tmpl.render(ctxt.variables)", "response": "Interpret a template string and return a callable that will return the string rendered from the template."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef expression(self, string):\n\n # Short-circuit if the expression \"string\" isn't actually a\n # string\n if not isinstance(string, six.string_types):\n return lambda ctxt: string\n\n # Create the expression and return the callable\n expr = self._jinja.compile_expression(string)\n return lambda ctxt: expr(ctxt.variables)", "response": "Interpret an expression string and return a callable that will return the result of evaluating it."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef last_error(self):\n if not len(self.log):\n raise RuntimeError('Nothing executed')\n\n try:\n errs = [l for l in self.log if l[1] != 0]\n\n return errs[-1][2]\n except IndexError:\n # odd case where there were no errors\n #TODO\n return 'no last error'", "response": "Get the output of the last command exevuted."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrap for subprocess. check_output.", "response": "def check_output(self, cmd):\n \"\"\"Wrapper for subprocess.check_output.\"\"\"\n ret, output = self._exec(cmd)\n if not ret == 0:\n raise CommandError(self)\n\n return output"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check_call(self, cmd):\n ret, _ = self._exec(cmd)\n if not ret == 0:\n raise CommandError(self)\n\n return ret", "response": "Fake the interface of subprocess. call."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef unpack_pargs(positional_args, param_kwargs, gnu=False):\n\n def _transform(argname):\n \"\"\"Transform a python identifier into a \n shell-appropriate argument name\n \"\"\"\n if len(argname) == 1:\n return '-{}'.format(argname)\n\n return '--{}'.format(argname.replace('_', '-'))\n\n args = []\n for item in param_kwargs.keys():\n for value in param_kwargs.getlist(item):\n if gnu:\n args.append('{}={}'.format(\n _transform(item),\n value\n ))\n else:\n args.extend([\n _transform(item),\n value\n ])\n if positional_args: \n for item in positional_args: \n args.append(_transform(item))\n\n return args", "response": "Unpack multidict and positional args into a list appropriate for subprocess."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_source(self, filename):\n source = None\n\n base, ext = os.path.splitext(filename)\n TRY_EXTS = {\n '.py': ['.py', '.pyw'],\n '.pyw': ['.pyw'],\n }\n try_exts = TRY_EXTS.get(ext)\n if not try_exts:\n return filename, None\n\n for try_ext in try_exts:\n try_filename = base + try_ext\n if os.path.exists(try_filename):\n return try_filename, None\n source = self.coverage.file_locator.get_zip_data(try_filename)\n if source:\n return try_filename, source\n raise NoSource(\"No source for code: '%s'\" % filename)", "response": "Find the source for filename."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef arcs_executed(self):\n executed = self.coverage.data.executed_arcs(self.filename)\n m2fl = self.parser.first_line\n executed = [(m2fl(l1), m2fl(l2)) for (l1,l2) in executed]\n return sorted(executed)", "response": "Returns a sorted list of the arcs actually executed in the code."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef arcs_missing(self):\n possible = self.arc_possibilities()\n executed = self.arcs_executed()\n missing = [\n p for p in possible\n if p not in executed\n and p[0] not in self.no_branch\n ]\n return sorted(missing)", "response": "Returns a sorted list of the arcs in the code not executed."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a sorted list of the executed arcs missing from the code.", "response": "def arcs_unpredicted(self):\n \"\"\"Returns a sorted list of the executed arcs missing from the code.\"\"\"\n possible = self.arc_possibilities()\n executed = self.arcs_executed()\n # Exclude arcs here which connect a line to itself. They can occur\n # in executed data in some cases. This is where they can cause\n # trouble, and here is where it's the least burden to remove them.\n unpredicted = [\n e for e in executed\n if e not in possible\n and e[0] != e[1]\n ]\n return sorted(unpredicted)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef branch_lines(self):\n exit_counts = self.parser.exit_counts()\n return [l1 for l1,count in iitems(exit_counts) if count > 1]", "response": "Returns a list of line numbers that have more than one exit."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef missing_branch_arcs(self):\n missing = self.arcs_missing()\n branch_lines = set(self.branch_lines())\n mba = {}\n for l1, l2 in missing:\n if l1 in branch_lines:\n if l1 not in mba:\n mba[l1] = []\n mba[l1].append(l2)\n return mba", "response": "Return arcs that weren t executed from branch lines."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget stats about branches.", "response": "def branch_stats(self):\n \"\"\"Get stats about branches.\n\n Returns a dict mapping line numbers to a tuple:\n (total_exits, taken_exits).\n \"\"\"\n\n exit_counts = self.parser.exit_counts()\n missing_arcs = self.missing_branch_arcs()\n stats = {}\n for lnum in self.branch_lines():\n exits = exit_counts[lnum]\n try:\n missing = len(missing_arcs[lnum])\n except KeyError:\n missing = 0\n stats[lnum] = (exits, exits - missing)\n return stats"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_precision(cls, precision):\n assert 0 <= precision < 10\n cls._precision = precision\n cls._near0 = 1.0 / 10**precision\n cls._near100 = 100.0 - cls._near0", "response": "Set the number of decimal places used to report percentages."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_pc_covered(self):\n if self.n_statements > 0:\n pc_cov = (100.0 * (self.n_executed + self.n_executed_branches) /\n (self.n_statements + self.n_branches))\n else:\n pc_cov = 100.0\n return pc_cov", "response": "Returns a single percentage value for coverage."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the percent covered as a string without a percent sign.", "response": "def _get_pc_covered_str(self):\n \"\"\"Returns the percent covered, as a string, without a percent sign.\n\n Note that \"0\" is only returned when the value is truly zero, and \"100\"\n is only returned when the value is truly 100. Rounding can never\n result in either \"0\" or \"100\".\n\n \"\"\"\n pc = self.pc_covered\n if 0 < pc < self._near0:\n pc = self._near0\n elif self._near100 < pc < 100:\n pc = self._near100\n else:\n pc = round(pc, self._precision)\n return \"%.*f\" % (self._precision, pc)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhighlighting the text in haystack with the given class_name.", "response": "def highlight_text(needles, haystack, cls_name='highlighted', words=False, case=False):\n \"\"\" Applies cls_name to all needles found in haystack. \"\"\"\n\n if not needles:\n return haystack\n if not haystack:\n return ''\n\n if words:\n pattern = r\"(%s)\" % \"|\".join(['\\\\b{}\\\\b'.format(re.escape(n)) for n in needles])\n else:\n pattern = r\"(%s)\" % \"|\".join([re.escape(n) for n in needles])\n\n if case:\n regex = re.compile(pattern)\n else:\n regex = re.compile(pattern, re.I)\n\n i, out = 0, \"\"\n for m in regex.finditer(haystack):\n out += \"\".join([haystack[i:m.start()], '' % cls_name,\n haystack[m.start():m.end()], \"\"])\n i = m.end()\n return mark_safe(out + haystack[i:])"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngives a list of words this function highlights the matched text in the given string.", "response": "def highlight(string, keywords, cls_name='highlighted'):\n \"\"\" Given an list of words, this function highlights the matched text in the given string. \"\"\"\n\n if not keywords:\n return string\n if not string:\n return ''\n include, exclude = get_text_tokenizer(keywords)\n highlighted = highlight_text(include, string, cls_name)\n return highlighted"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngive a list of words this function highlights the matched words in the given string.", "response": "def highlight_words(string, keywords, cls_name='highlighted'):\n \"\"\" Given an list of words, this function highlights the matched words in the given string. \"\"\"\n\n if not keywords:\n return string\n if not string:\n return ''\n include, exclude = get_text_tokenizer(keywords)\n highlighted = highlight_text(include, string, cls_name, words=True)\n return highlighted"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run_setup(setup_script, args):\n old_dir = os.getcwd()\n save_argv = sys.argv[:]\n save_path = sys.path[:]\n setup_dir = os.path.abspath(os.path.dirname(setup_script))\n temp_dir = os.path.join(setup_dir,'temp')\n if not os.path.isdir(temp_dir): os.makedirs(temp_dir)\n save_tmp = tempfile.tempdir\n save_modules = sys.modules.copy()\n pr_state = pkg_resources.__getstate__()\n try:\n tempfile.tempdir = temp_dir\n os.chdir(setup_dir)\n try:\n sys.argv[:] = [setup_script]+list(args)\n sys.path.insert(0, setup_dir)\n DirectorySandbox(setup_dir).run(\n lambda: execfile(\n \"setup.py\",\n {'__file__':setup_script, '__name__':'__main__'}\n )\n )\n except SystemExit, v:\n if v.args and v.args[0]:\n raise\n # Normal exit, just return\n finally:\n pkg_resources.__setstate__(pr_state)\n sys.modules.update(save_modules)\n # remove any modules imported within the sandbox\n del_modules = [\n mod_name for mod_name in sys.modules\n if mod_name not in save_modules\n # exclude any encodings modules. See #285\n and not mod_name.startswith('encodings.')\n ]\n map(sys.modules.__delitem__, del_modules)\n os.chdir(old_dir)\n sys.path[:] = save_path\n sys.argv[:] = save_argv\n tempfile.tempdir = save_tmp", "response": "Run a distutils setup script sandboxed in its directory"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run(self, func):\n try:\n self._copy(self)\n if _file:\n __builtin__.file = self._file\n __builtin__.open = self._open\n self._active = True\n return func()\n finally:\n self._active = False\n if _file:\n __builtin__.file = _file\n __builtin__.open = _open\n self._copy(_os)", "response": "Run func under os sandboxing"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef open(self, file, flags, mode=0777):\n if flags & WRITE_FLAGS and not self._ok(file):\n self._violation(\"os.open\", file, flags, mode)\n return _os.open(file,flags,mode)", "response": "Called for low - level os. open."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nremove a single pair of quotes from the endpoints of a string.", "response": "def unquote_ends(istr):\n \"\"\"Remove a single pair of quotes from the endpoints of a string.\"\"\"\n\n if not istr:\n return istr\n if (istr[0]==\"'\" and istr[-1]==\"'\") or \\\n (istr[0]=='\"' and istr[-1]=='\"'):\n return istr[1:-1]\n else:\n return istr"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn grep on dir - like object", "response": "def dgrep(pat,*opts):\n \"\"\"Return grep() on dir()+dir(__builtins__).\n\n A very common use of grep() when working interactively.\"\"\"\n\n return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef indent(instr,nspaces=4, ntabs=0, flatten=False):\n if instr is None:\n return\n ind = '\\t'*ntabs+' '*nspaces\n if flatten:\n pat = re.compile(r'^\\s*', re.MULTILINE)\n else:\n pat = re.compile(r'^', re.MULTILINE)\n outstr = re.sub(pat, ind, instr)\n if outstr.endswith(os.linesep+ind):\n return outstr[:-len(ind)]\n else:\n return outstr", "response": "Indent a string by nspaces or tabstops."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef native_line_ends(filename,backup=1):\n\n backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}\n\n bak_filename = filename + backup_suffixes[os.name]\n\n original = open(filename).read()\n shutil.copy2(filename,bak_filename)\n try:\n new = open(filename,'wb')\n new.write(os.linesep.join(original.splitlines()))\n new.write(os.linesep) # ALWAYS put an eol at the end of the file\n new.close()\n except:\n os.rename(bak_filename,filename)\n if not backup:\n try:\n os.remove(bak_filename)\n except:\n pass", "response": "Convert a file to line - ends native to the current OS."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the input string centered in a marquee.", "response": "def marquee(txt='',width=78,mark='*'):\n \"\"\"Return the input string centered in a 'marquee'.\n\n :Examples:\n\n In [16]: marquee('A test',40)\n Out[16]: '**************** A test ****************'\n\n In [17]: marquee('A test',40,'-')\n Out[17]: '---------------- A test ----------------'\n\n In [18]: marquee('A test',40,' ')\n Out[18]: ' A test '\n\n \"\"\"\n if not txt:\n return (mark*width)[:width]\n nmark = (width-len(txt)-2)//len(mark)//2\n if nmark < 0: nmark =0\n marks = mark*nmark\n return '%s %s %s' % (marks,txt,marks)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef format_screen(strng):\n # Paragraph continue\n par_re = re.compile(r'\\\\$',re.MULTILINE)\n strng = par_re.sub('',strng)\n return strng", "response": "Format a string for screen printing."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef wrap_paragraphs(text, ncols=80):\n paragraph_re = re.compile(r'\\n(\\s*\\n)+', re.MULTILINE)\n text = dedent(text).strip()\n paragraphs = paragraph_re.split(text)[::2] # every other entry is space\n out_ps = []\n indent_re = re.compile(r'\\n\\s+', re.MULTILINE)\n for p in paragraphs:\n # presume indentation that survives dedent is meaningful formatting,\n # so don't fill unless text is flush.\n if indent_re.search(p) is None:\n # wrap paragraph\n p = textwrap.fill(p, ncols)\n out_ps.append(p)\n return out_ps", "response": "Wrap multiple paragraphs to fit a specified width."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the longest common substring in a list of strings.", "response": "def long_substr(data):\n \"\"\"Return the longest common substring in a list of strings.\n \n Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python\n \"\"\"\n substr = ''\n if len(data) > 1 and len(data[0]) > 0:\n for i in range(len(data[0])):\n for j in range(len(data[0])-i+1):\n if j > len(substr) and all(data[0][i:i+j] in x for x in data):\n substr = data[0][i:i+j]\n elif len(data) == 1:\n substr = data[0]\n return substr"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef strip_email_quotes(text):\n lines = text.splitlines()\n matches = set()\n for line in lines:\n prefix = re.match(r'^(\\s*>[ >]*)', line)\n if prefix:\n matches.add(prefix.group(1))\n else:\n break\n else:\n prefix = long_substr(list(matches))\n if prefix:\n strip = len(prefix)\n text = '\\n'.join([ ln[strip:] for ln in lines])\n return text", "response": "Strip leading email quotation characters from the input text."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _find_optimal(rlist , separator_size=2 , displaywidth=80):\n for nrow in range(1, len(rlist)+1) :\n chk = map(max,_chunks(rlist, nrow))\n sumlength = sum(chk)\n ncols = len(chk)\n if sumlength+separator_size*(ncols-1) <= displaywidth :\n break;\n return {'columns_numbers' : ncols,\n 'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,\n 'rows_numbers' : nrow,\n 'columns_width' : chk\n }", "response": "Calculate optimal info to columnize a list of string"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning list item number or default if don t exist", "response": "def _get_or_default(mylist, i, default=None):\n \"\"\"return list item number, or default if don't exist\"\"\"\n if i >= len(mylist):\n return default\n else :\n return mylist[i]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute_item_matrix(items, empty=None, *args, **kwargs) :\n info = _find_optimal(map(len, items), *args, **kwargs)\n nrow, ncol = info['rows_numbers'], info['columns_numbers']\n return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)", "response": "Returns a nested list of strings and dict_info to columnize items"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef columnize(items, separator=' ', displaywidth=80):\n if not items :\n return '\\n'\n matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)\n fmatrix = [filter(None, x) for x in matrix]\n sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])\n return '\\n'.join(map(sjoin, fmatrix))+'\\n'", "response": "Transform a list of strings into a single string with columns."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn all items matching pattern", "response": "def grep(self, pattern, prune = False, field = None):\n \"\"\" Return all strings matching 'pattern' (a regex or callable)\n\n This is case-insensitive. If prune is true, return all items\n NOT matching the pattern.\n\n If field is specified, the match must occur in the specified\n whitespace-separated field.\n\n Examples::\n\n a.grep( lambda x: x.startswith('C') )\n a.grep('Cha.*log', prune=1)\n a.grep('chm', field=-1)\n \"\"\"\n\n def match_target(s):\n if field is None:\n return s\n parts = s.split()\n try:\n tgt = parts[field]\n return tgt\n except IndexError:\n return \"\"\n\n if isinstance(pattern, basestring):\n pred = lambda x : re.search(pattern, x, re.IGNORECASE)\n else:\n pred = pattern\n if not prune:\n return SList([el for el in self if pred(match_target(el))])\n else:\n return SList([el for el in self if not pred(match_target(el))])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncollect whitespace - separated fields from string list", "response": "def fields(self, *fields):\n \"\"\" Collect whitespace-separated fields from string list\n\n Allows quick awk-like usage of string lists.\n\n Example data (in var a, created by 'a = !ls -l')::\n -rwxrwxrwx 1 ville None 18 Dec 14 2006 ChangeLog\n drwxrwxrwx+ 6 ville None 0 Oct 24 18:05 IPython\n\n a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']\n a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']\n (note the joining by space).\n a.fields(-1) is ['ChangeLog', 'IPython']\n\n IndexErrors are ignored.\n\n Without args, fields() just split()'s the strings.\n \"\"\"\n if len(fields) == 0:\n return [el.split() for el in self]\n\n res = SList()\n for el in [f.split() for f in self]:\n lineparts = []\n\n for fd in fields:\n try:\n lineparts.append(el[fd])\n except IndexError:\n pass\n if lineparts:\n res.append(\" \".join(lineparts))\n\n return res"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sort(self,field= None, nums = False):\n\n #decorate, sort, undecorate\n if field is not None:\n dsu = [[SList([line]).fields(field), line] for line in self]\n else:\n dsu = [[line, line] for line in self]\n if nums:\n for i in range(len(dsu)):\n numstr = \"\".join([ch for ch in dsu[i][0] if ch.isdigit()])\n try:\n n = int(numstr)\n except ValueError:\n n = 0;\n dsu[i][0] = n\n\n\n dsu.sort()\n return SList([t[1] for t in dsu])", "response": "sort by specified fields"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread a Python file into a unicode string.", "response": "def read_py_file(filename, skip_encoding_cookie=True):\n \"\"\"Read a Python file, using the encoding declared inside the file.\n \n Parameters\n ----------\n filename : str\n The path to the file to read.\n skip_encoding_cookie : bool\n If True (the default), and the encoding declaration is found in the first\n two lines, that line will be excluded from the output - compiling a\n unicode string with an encoding declaration is a SyntaxError in Python 2.\n \n Returns\n -------\n A unicode string containing the contents of the file.\n \"\"\"\n with open(filename) as f: # the open function defined in this module.\n if skip_encoding_cookie:\n return \"\".join(strip_encoding_cookie(f))\n else:\n return f.read()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_py_url(url, errors='replace', skip_encoding_cookie=True):\n response = urllib.urlopen(url)\n buffer = io.BytesIO(response.read())\n encoding, lines = detect_encoding(buffer.readline)\n buffer.seek(0)\n text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)\n text.mode = 'r'\n if skip_encoding_cookie:\n return \"\".join(strip_encoding_cookie(text))\n else:\n return text.read()", "response": "Read a Python file from a URL."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef build_kernel_argv(self, argv=None):\n if argv is None:\n argv = sys.argv[1:]\n self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags)\n # kernel should inherit default config file from frontend\n self.kernel_argv.append(\"--KernelApp.parent_appname='%s'\"%self.name)", "response": "build argv to be passed to kernel subprocess"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfinding the connection file and load the info if not already loaded", "response": "def init_connection_file(self):\n \"\"\"find the connection file, and load the info if found.\n \n The current working directory and the current profile's security\n directory will be searched for the file if it is not given by\n absolute path.\n \n When attempting to connect to an existing kernel and the `--existing`\n argument does not match an existing file, it will be interpreted as a\n fileglob, and the matching file in the current profile's security dir\n with the latest access time will be used.\n \n After this method is called, self.connection_file contains the *full path*\n to the connection file, never just its name.\n \"\"\"\n if self.existing:\n try:\n cf = find_connection_file(self.existing)\n except Exception:\n self.log.critical(\"Could not find existing kernel connection file %s\", self.existing)\n self.exit(1)\n self.log.info(\"Connecting to existing kernel: %s\" % cf)\n self.connection_file = cf\n else:\n # not existing, check if we are going to write the file\n # and ensure that self.connection_file is a full path, not just the shortname\n try:\n cf = find_connection_file(self.connection_file)\n except Exception:\n # file might not exist\n if self.connection_file == os.path.basename(self.connection_file):\n # just shortname, put it in security dir\n cf = os.path.join(self.profile_dir.security_dir, self.connection_file)\n else:\n cf = self.connection_file\n self.connection_file = cf\n \n # should load_connection_file only be used for existing?\n # as it is now, this allows reusing ports if an existing\n # file is requested\n try:\n self.load_connection_file()\n except Exception:\n self.log.error(\"Failed to load connection file: %r\", self.connection_file, exc_info=True)\n self.exit(1)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets up ssh tunnels, if needed.", "response": "def init_ssh(self):\n \"\"\"set up ssh tunnels, if needed.\"\"\"\n if not self.sshserver and not self.sshkey:\n return\n \n if self.sshkey and not self.sshserver:\n # specifying just the key implies that we are connecting directly\n self.sshserver = self.ip\n self.ip = LOCALHOST\n \n # build connection dict for tunnels:\n info = dict(ip=self.ip,\n shell_port=self.shell_port,\n iopub_port=self.iopub_port,\n stdin_port=self.stdin_port,\n hb_port=self.hb_port\n )\n \n self.log.info(\"Forwarding connections to %s via %s\"%(self.ip, self.sshserver))\n \n # tunnels return a new set of ports, which will be on localhost:\n self.ip = LOCALHOST\n try:\n newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)\n except:\n # even catch KeyboardInterrupt\n self.log.error(\"Could not setup tunnels\", exc_info=True)\n self.exit(1)\n \n self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports\n \n cf = self.connection_file\n base,ext = os.path.splitext(cf)\n base = os.path.basename(base)\n self.connection_file = os.path.basename(base)+'-ssh'+ext\n self.log.critical(\"To connect another client via this tunnel, use:\")\n self.log.critical(\"--existing %s\" % self.connection_file)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize the instance variables.", "response": "def initialize(self, argv=None):\n \"\"\"\n Classes which mix this class in should call:\n IPythonConsoleApp.initialize(self,argv)\n \"\"\"\n self.init_connection_file()\n default_secure(self.config)\n self.init_ssh()\n self.init_kernel_manager()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprepare a message dict for sending to the server.", "response": "def prepare_message(self, data=None):\n \"\"\"\n Return message as dict\n :return dict\n \"\"\"\n message = {\n 'protocol': self.protocol,\n 'node': self._node,\n 'chip_id': self._chip_id,\n 'event': '',\n 'parameters': {},\n 'response': '',\n 'targets': [\n 'ALL'\n ]\n }\n if type(data) is dict:\n for k, v in data.items():\n if k in message:\n message[k] = v\n\n return message"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef decode_message(self, message):\n try:\n message = json.loads(message)\n if not self._validate_message(message):\n message = None\n except ValueError:\n message = None\n\n return message", "response": "Decode json string to dict."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pretty(obj, verbose=False, max_width=79, newline='\\n'):\n stream = StringIO()\n printer = RepresentationPrinter(stream, verbose, max_width, newline)\n printer.pretty(obj)\n printer.flush()\n return stream.getvalue()", "response": "Pretty print the object s representation."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pprint(obj, verbose=False, max_width=79, newline='\\n'):\n printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline)\n printer.pretty(obj)\n printer.flush()\n sys.stdout.write(newline)\n sys.stdout.flush()", "response": "Like pretty but print to stdout."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets a reasonable method resolution order of a class and its superclasses .", "response": "def _get_mro(obj_class):\n \"\"\" Get a reasonable method resolution order of a class and its superclasses\n for both old-style and new-style classes.\n \"\"\"\n if not hasattr(obj_class, '__mro__'):\n # Old-style class. Mix in object to make a fake new-style class.\n try:\n obj_class = type(obj_class.__name__, (obj_class, object), {})\n except TypeError:\n # Old-style extension type that does not descend from object.\n # FIXME: try to construct a more thorough MRO.\n mro = [obj_class]\n else:\n mro = obj_class.__mro__[1:-1]\n else:\n mro = obj_class.__mro__\n return mro"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndefaults print function. Used if an object does not provide one and it s none of the builtin objects.", "response": "def _default_pprint(obj, p, cycle):\n \"\"\"\n The default print function. Used if an object does not provide one and\n it's none of the builtin objects.\n \"\"\"\n klass = getattr(obj, '__class__', None) or type(obj)\n if getattr(klass, '__repr__', None) not in _baseclass_reprs:\n # A user-provided repr.\n p.text(repr(obj))\n return\n p.begin_group(1, '<')\n p.pretty(klass)\n p.text(' at 0x%x' % id(obj))\n if cycle:\n p.text(' ...')\n elif p.verbose:\n first = True\n for key in dir(obj):\n if not key.startswith('_'):\n try:\n value = getattr(obj, key)\n except AttributeError:\n continue\n if isinstance(value, types.MethodType):\n continue\n if not first:\n p.text(',')\n p.breakable()\n p.text(key)\n p.text('=')\n step = len(key) + 1\n p.indentation += step\n p.pretty(value)\n p.indentation -= step\n first = False\n p.end_group(1, '>')"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a pprint function useful for sequences.", "response": "def _seq_pprinter_factory(start, end, basetype):\n \"\"\"\n Factory that returns a pprint function useful for sequences. Used by\n the default pprint for tuples, dicts, lists, sets and frozensets.\n \"\"\"\n def inner(obj, p, cycle):\n typ = type(obj)\n if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:\n # If the subclass provides its own repr, use it instead.\n return p.text(typ.__repr__(obj))\n\n if cycle:\n return p.text(start + '...' + end)\n step = len(start)\n p.begin_group(step, start)\n for idx, x in enumerate(obj):\n if idx:\n p.text(',')\n p.breakable()\n p.pretty(x)\n if len(obj) == 1 and type(obj) is tuple:\n # Special case for 1-item tuples.\n p.text(',')\n p.end_group(step, end)\n return inner"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _dict_pprinter_factory(start, end, basetype=None):\n def inner(obj, p, cycle):\n typ = type(obj)\n if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:\n # If the subclass provides its own repr, use it instead.\n return p.text(typ.__repr__(obj))\n\n if cycle:\n return p.text('{...}')\n p.begin_group(1, start)\n keys = obj.keys()\n try:\n keys.sort()\n except Exception, e:\n # Sometimes the keys don't sort.\n pass\n for idx, key in enumerate(keys):\n if idx:\n p.text(',')\n p.breakable()\n p.pretty(key)\n p.text(': ')\n p.pretty(obj[key])\n p.end_group(1, end)\n return inner", "response": "A function that returns a pprint function used by the default pprint of the dict proxies."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _super_pprint(obj, p, cycle):\n p.begin_group(8, '')", "response": "The pprint for the super type."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _type_pprint(obj, p, cycle):\n if obj.__module__ in ('__builtin__', 'exceptions'):\n name = obj.__name__\n else:\n name = obj.__module__ + '.' + obj.__name__\n p.text(name)", "response": "The pprint for classes and types."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _function_pprint(obj, p, cycle):\n if obj.__module__ in ('__builtin__', 'exceptions') or not obj.__module__:\n name = obj.__name__\n else:\n name = obj.__module__ + '.' + obj.__name__\n p.text('' % name)", "response": "Base pprint for all functions builtin functions and exceptions."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbases pprint for all exceptions.", "response": "def _exception_pprint(obj, p, cycle):\n \"\"\"Base pprint for all exceptions.\"\"\"\n if obj.__class__.__module__ in ('exceptions', 'builtins'):\n name = obj.__class__.__name__\n else:\n name = '%s.%s' % (\n obj.__class__.__module__,\n obj.__class__.__name__\n )\n step = len(name) + 1\n p.begin_group(step, name + '(')\n for idx, arg in enumerate(getattr(obj, 'args', ())):\n if idx:\n p.text(',')\n p.breakable()\n p.pretty(arg)\n p.end_group(step, ')')"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef for_type(typ, func):\n oldfunc = _type_pprinters.get(typ, None)\n if func is not None:\n # To support easy restoration of old pprinters, we need to ignore Nones.\n _type_pprinters[typ] = func\n return oldfunc", "response": "Add a pretty printer for a given type."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef for_type_by_name(type_module, type_name, func):\n key = (type_module, type_name)\n oldfunc = _deferred_type_pprinters.get(key, None)\n if func is not None:\n # To support easy restoration of old pprinters, we need to ignore Nones.\n _deferred_type_pprinters[key] = func\n return oldfunc", "response": "Add a pretty printer for a type specified by the module and name of a type object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef group(self, indent=0, open='', close=''):\n self.begin_group(indent, open)\n try:\n yield\n finally:\n self.end_group(indent, close)", "response": "A context manager that returns a group of the entries in the log file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd literal text to the output.", "response": "def text(self, obj):\n \"\"\"Add literal text to the output.\"\"\"\n width = len(obj)\n if self.buffer:\n text = self.buffer[-1]\n if not isinstance(text, Text):\n text = Text()\n self.buffer.append(text)\n text.add(obj, width)\n self.buffer_width += width\n self._break_outer_groups()\n else:\n self.output.write(obj)\n self.output_width += width"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a breakable separator to the output.", "response": "def breakable(self, sep=' '):\n \"\"\"\n Add a breakable separator to the output. This does not mean that it\n will automatically break here. If no breaking on this position takes\n place the `sep` is inserted which default to one space.\n \"\"\"\n width = len(sep)\n group = self.group_stack[-1]\n if group.want_break:\n self.flush()\n self.output.write(self.newline)\n self.output.write(' ' * self.indentation)\n self.output_width = self.indentation\n self.buffer_width = 0\n else:\n self.buffer.append(Breakable(sep, width, self))\n self.buffer_width += width\n self._break_outer_groups()"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbegins a group of entries.", "response": "def begin_group(self, indent=0, open=''):\n \"\"\"\n Begin a group. If you want support for python < 2.5 which doesn't has\n the with statement this is the preferred way:\n\n p.begin_group(1, '{')\n ...\n p.end_group(1, '}')\n\n The python 2.5 expression would be this:\n\n with p.group(1, '{', '}'):\n ...\n\n The first parameter specifies the indentation for the next line (usually\n the width of the opening text), the second the opening text. All\n parameters are optional.\n \"\"\"\n if open:\n self.text(open)\n group = Group(self.group_stack[-1].depth + 1)\n self.group_stack.append(group)\n self.group_queue.enq(group)\n self.indentation += indent"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef end_group(self, dedent=0, close=''):\n self.indentation -= dedent\n group = self.group_stack.pop()\n if not group.breakables:\n self.group_queue.remove(group)\n if close:\n self.text(close)", "response": "End a group. See begin_group for more details."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nflushing all data that is left in the buffer.", "response": "def flush(self):\n \"\"\"Flush data that is left in the buffer.\"\"\"\n for data in self.buffer:\n self.output_width += data.output(self.output, self.output_width)\n self.buffer.clear()\n self.buffer_width = 0"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pretty(self, obj):\n obj_id = id(obj)\n cycle = obj_id in self.stack\n self.stack.append(obj_id)\n self.begin_group()\n try:\n obj_class = getattr(obj, '__class__', None) or type(obj)\n # First try to find registered singleton printers for the type.\n try:\n printer = self.singleton_pprinters[obj_id]\n except (TypeError, KeyError):\n pass\n else:\n return printer(obj, self, cycle)\n # Next walk the mro and check for either:\n # 1) a registered printer\n # 2) a _repr_pretty_ method\n for cls in _get_mro(obj_class):\n if cls in self.type_pprinters:\n # printer registered in self.type_pprinters\n return self.type_pprinters[cls](obj, self, cycle)\n else:\n # deferred printer\n printer = self._in_deferred_types(cls)\n if printer is not None:\n return printer(obj, self, cycle)\n else:\n # Finally look for special method names.\n # Some objects automatically create any requested\n # attribute. Try to ignore most of them by checking for\n # callability.\n if '_repr_pretty_' in obj_class.__dict__:\n meth = obj_class._repr_pretty_\n if callable(meth):\n return meth(obj, self, cycle)\n return _default_pprint(obj, self, cycle)\n finally:\n self.end_group()\n self.stack.pop()", "response": "Pretty print the given object."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _in_deferred_types(self, cls):\n mod = getattr(cls, '__module__', None)\n name = getattr(cls, '__name__', None)\n key = (mod, name)\n printer = None\n if key in self.deferred_pprinters:\n # Move the printer over to the regular registry.\n printer = self.deferred_pprinters.pop(key)\n self.type_pprinters[cls] = printer\n return printer", "response": "Check if the given class is specified in the deferred type registry. If it does return the printer that is registered under the given class. If it does return None."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef exception_colors():\n\n ex_colors = ColorSchemeTable()\n\n # Populate it with color schemes\n C = TermColors # shorthand and local lookup\n ex_colors.add_scheme(ColorScheme(\n 'NoColor',\n # The color to be used for the top line\n topline = C.NoColor,\n\n # The colors to be used in the traceback\n filename = C.NoColor,\n lineno = C.NoColor,\n name = C.NoColor,\n vName = C.NoColor,\n val = C.NoColor,\n em = C.NoColor,\n\n # Emphasized colors for the last frame of the traceback\n normalEm = C.NoColor,\n filenameEm = C.NoColor,\n linenoEm = C.NoColor,\n nameEm = C.NoColor,\n valEm = C.NoColor,\n\n # Colors for printing the exception\n excName = C.NoColor,\n line = C.NoColor,\n caret = C.NoColor,\n Normal = C.NoColor\n ))\n\n # make some schemes as instances so we can copy them for modification easily\n ex_colors.add_scheme(ColorScheme(\n 'Linux',\n # The color to be used for the top line\n topline = C.LightRed,\n\n # The colors to be used in the traceback\n filename = C.Green,\n lineno = C.Green,\n name = C.Purple,\n vName = C.Cyan,\n val = C.Green,\n em = C.LightCyan,\n\n # Emphasized colors for the last frame of the traceback\n normalEm = C.LightCyan,\n filenameEm = C.LightGreen,\n linenoEm = C.LightGreen,\n nameEm = C.LightPurple,\n valEm = C.LightBlue,\n\n # Colors for printing the exception\n excName = C.LightRed,\n line = C.Yellow,\n caret = C.White,\n Normal = C.Normal\n ))\n\n # For light backgrounds, swap dark/light colors\n ex_colors.add_scheme(ColorScheme(\n 'LightBG',\n # The color to be used for the top line\n topline = C.Red,\n\n # The colors to be used in the traceback\n filename = C.LightGreen,\n lineno = C.LightGreen,\n name = C.LightPurple,\n vName = C.Cyan,\n val = C.LightGreen,\n em = C.Cyan,\n\n # Emphasized colors for the last frame of the traceback\n normalEm = C.Cyan,\n filenameEm = C.Green,\n linenoEm = C.Green,\n nameEm = C.Purple,\n valEm = C.Blue,\n\n # Colors for printing the exception\n excName = C.Red,\n #line = C.Brown, # brown often is displayed as yellow\n line = C.Red,\n caret = C.Normal,\n Normal = C.Normal,\n ))\n\n return ex_colors", "response": "Return a color table with fields for exception reporting."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _prepare_ods_columns(ods, trans_title_row):\n ods.content.getSheet(0).setSheetName('Translations')\n ods.content.makeSheet('Meta options')\n ods.content.getColumn(0).setWidth('5.0in')\n ods.content.getCell(0, 0).stringValue('metadata')\\\n .setCellColor(settings.TITLE_ROW_BG_COLOR) \\\n .setBold(True).setFontColor(settings.TITLE_ROW_FONT_COLOR)\n\n ods.content.getSheet(0)\n ods.content.getColumn(0).setWidth('1.5in')\n ods.content.getCell(0, 0) \\\n .setCellColor(settings.TITLE_ROW_BG_COLOR) \\\n .setBold(True).setFontColor(settings.TITLE_ROW_FONT_COLOR)\n for i, title in enumerate(trans_title_row):\n ods.content.getColumn(i).setWidth(settings.MSGSTR_COLUMN_WIDTH)\n ods.content.getCell(i, 0).stringValue(title)\\\n .setCellColor(settings.TITLE_ROW_BG_COLOR) \\\n .setBold(True).setFontColor(settings.TITLE_ROW_FONT_COLOR)\n ods.content.getColumn(0).setWidth(settings.NOTES_COLUMN_WIDTH)", "response": "Prepare columns in new ods file create new sheet for metadata and translations"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _write_trans_into_ods(ods, languages, locale_root,\n po_files_path, po_filename, start_row):\n \"\"\"\n Write translations from po files into ods one file.\n Assumes a directory structure:\n ///.\n \"\"\"\n ods.content.getSheet(0)\n for i, lang in enumerate(languages[1:]):\n lang_po_path = os.path.join(locale_root, lang,\n po_files_path, po_filename)\n if os.path.exists(lang_po_path):\n po_file = polib.pofile(lang_po_path)\n for j, entry in enumerate(po_file):\n # start from 4th column, 1st row\n row = j+start_row\n ods.content.getCell(i+4, row).stringValue(\n _escape_apostrophe(entry.msgstr))\n if i % 2 == 1:\n ods.content.getCell(i+4, row).setCellColor(\n settings.ODD_COLUMN_BG_COLOR)\n else:\n ods.content.getCell(i+4, row).setCellColor(\n settings.EVEN_COLUMN_BG_COLOR)", "response": "Write translations from po files into ods one file."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _write_row_into_ods(ods, sheet_no, row_no, row):\n ods.content.getSheet(sheet_no)\n for j, col in enumerate(row):\n cell = ods.content.getCell(j, row_no+1)\n cell.stringValue(_escape_apostrophe(col))\n if j % 2 == 1:\n cell.setCellColor(settings.EVEN_COLUMN_BG_COLOR)\n else:\n cell.setCellColor(settings.ODD_COLUMN_BG_COLOR)", "response": "Write row with translations to ods file into specified sheet and row_no."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef po_to_ods(languages, locale_root, po_files_path, temp_file_path):\n title_row = ['file', 'comment', 'msgid']\n title_row += map(lambda s: s + ':msgstr', languages)\n\n ods = ODS()\n\n _prepare_ods_columns(ods, title_row)\n\n po_files = _get_all_po_filenames(locale_root, languages[0], po_files_path)\n\n i = 1\n for po_filename in po_files:\n po_file_path = os.path.join(locale_root, languages[0],\n po_files_path, po_filename)\n\n start_row = i\n\n po = polib.pofile(po_file_path)\n for entry in po:\n meta = dict(entry.__dict__)\n meta.pop('msgid', None)\n meta.pop('msgstr', None)\n meta.pop('tcomment', None)\n\n ods.content.getSheet(1)\n ods.content.getCell(0, i).stringValue(\n str(meta)).setCellColor(settings.EVEN_COLUMN_BG_COLOR)\n\n ods.content.getSheet(0)\n ods.content.getCell(0, i) \\\n .stringValue(po_filename) \\\n .setCellColor(settings.ODD_COLUMN_BG_COLOR)\n ods.content.getCell(1, i) \\\n .stringValue(_escape_apostrophe(entry.tcomment)) \\\n .setCellColor(settings.ODD_COLUMN_BG_COLOR)\n ods.content.getCell(2, i) \\\n .stringValue(_escape_apostrophe(entry.msgid)) \\\n .setCellColor(settings.EVEN_COLUMN_BG_COLOR)\n ods.content.getCell(3, i) \\\n .stringValue(_escape_apostrophe(entry.msgstr))\\\n .setCellColor(settings.ODD_COLUMN_BG_COLOR)\n\n i += 1\n\n _write_trans_into_ods(ods, languages, locale_root,\n po_files_path, po_filename, start_row)\n\n ods.save(temp_file_path)", "response": "Converts a PO file to the ODS file."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconverting csv files to one ods file", "response": "def csv_to_ods(trans_csv, meta_csv, local_ods):\n \"\"\"\n Converts csv files to one ods file\n :param trans_csv: path to csv file with translations\n :param meta_csv: path to csv file with metadata\n :param local_ods: path to new ods file\n \"\"\"\n trans_reader = UnicodeReader(trans_csv)\n meta_reader = UnicodeReader(meta_csv)\n\n ods = ODS()\n\n trans_title = trans_reader.next()\n meta_reader.next()\n\n _prepare_ods_columns(ods, trans_title)\n\n for i, (trans_row, meta_row) in enumerate(izip(trans_reader, meta_reader)):\n _write_row_into_ods(ods, 0, i, trans_row)\n _write_row_into_ods(ods, 1, i, meta_row)\n\n trans_reader.close()\n meta_reader.close()\n\n ods.save(local_ods)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the current clipboard s text on Windows.", "response": "def win32_clipboard_get():\n \"\"\" Get the current clipboard's text on Windows.\n\n Requires Mark Hammond's pywin32 extensions.\n \"\"\"\n try:\n import win32clipboard\n except ImportError:\n raise TryNext(\"Getting text from the clipboard requires the pywin32 \"\n \"extensions: http://sourceforge.net/projects/pywin32/\")\n win32clipboard.OpenClipboard()\n text = win32clipboard.GetClipboardData(win32clipboard.CF_TEXT)\n # FIXME: convert \\r\\n to \\n?\n win32clipboard.CloseClipboard()\n return text"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the clipboard s text on OS X.", "response": "def osx_clipboard_get():\n \"\"\" Get the clipboard's text on OS X.\n \"\"\"\n p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'],\n stdout=subprocess.PIPE)\n text, stderr = p.communicate()\n # Text comes in with old Mac \\r line endings. Change them to \\n.\n text = text.replace('\\r', '\\n')\n return text"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef tkinter_clipboard_get():\n try:\n import Tkinter\n except ImportError:\n raise TryNext(\"Getting text from the clipboard on this platform \"\n \"requires Tkinter.\")\n root = Tkinter.Tk()\n root.withdraw()\n text = root.clipboard_get()\n root.destroy()\n return text", "response": "Get the text from the clipboard using Tkinter."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a safe build_prefix for the current node.", "response": "def _get_build_prefix():\n \"\"\" Returns a safe build_prefix \"\"\"\n path = os.path.join(\n tempfile.gettempdir(),\n 'pip_build_%s' % __get_username().replace(' ', '_')\n )\n if WINDOWS:\n \"\"\" on windows(tested on 7) temp dirs are isolated \"\"\"\n return path\n try:\n os.mkdir(path)\n write_delete_marker_file(path)\n except OSError:\n file_uid = None\n try:\n # raises OSError for symlinks\n # https://github.com/pypa/pip/pull/935#discussion_r5307003\n file_uid = get_path_uid(path)\n except OSError:\n file_uid = None\n\n if file_uid != os.geteuid():\n msg = (\n \"The temporary folder for building (%s) is either not owned by\"\n \" you, or is a symlink.\" % path\n )\n print(msg)\n print(\n \"pip will not work until the temporary folder is either \"\n \"deleted or is a real directory owned by your user account.\"\n )\n raise exceptions.InstallationError(msg)\n return path"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nprepare the communication of the current object.", "response": "def prepare_communication (self):\n \"\"\"\n Find the subdomain rank (tuple) for each processor and\n determine the neighbor info.\n \"\"\"\n \n nsd_ = self.nsd\n if nsd_<1:\n print('Number of space dimensions is %d, nothing to do' %nsd_)\n return\n \n self.subd_rank = [-1,-1,-1]\n self.subd_lo_ix = [-1,-1,-1]\n self.subd_hi_ix = [-1,-1,-1]\n self.lower_neighbors = [-1,-1,-1]\n self.upper_neighbors = [-1,-1,-1]\n\n num_procs = self.num_procs\n my_id = self.my_id\n\n num_subds = 1\n for i in range(nsd_):\n num_subds = num_subds*self.num_parts[i]\n if my_id==0:\n print(\"# subds=\", num_subds)\n # should check num_subds againt num_procs\n\n offsets = [1, 0, 0]\n\n # find the subdomain rank\n self.subd_rank[0] = my_id%self.num_parts[0]\n if nsd_>=2:\n offsets[1] = self.num_parts[0]\n self.subd_rank[1] = my_id/offsets[1]\n if nsd_==3:\n offsets[1] = self.num_parts[0]\n offsets[2] = self.num_parts[0]*self.num_parts[1]\n self.subd_rank[1] = (my_id%offsets[2])/self.num_parts[0]\n self.subd_rank[2] = my_id/offsets[2]\n \n print(\"my_id=%d, subd_rank: \"%my_id, self.subd_rank)\n if my_id==0:\n print(\"offsets=\", offsets)\n\n # find the neighbor ids\n for i in range(nsd_):\n rank = self.subd_rank[i]\n if rank>0:\n self.lower_neighbors[i] = my_id-offsets[i]\n if rank=(self.num_parts[i]-m):\n ix = ix+1 # load balancing\n if rank=0:\n self.in_lower_buffers = [zeros(1, float)]\n self.out_lower_buffers = [zeros(1, float)]\n if self.upper_neighbors[0]>=0:\n self.in_upper_buffers = [zeros(1, float)]\n self.out_upper_buffers = [zeros(1, float)]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef prepare_communication (self):\n \n RectPartitioner.prepare_communication (self)\n \n self.in_lower_buffers = [[], []]\n self.out_lower_buffers = [[], []]\n self.in_upper_buffers = [[], []]\n self.out_upper_buffers = [[], []]\n\n size1 = self.subd_hi_ix[1]-self.subd_lo_ix[1]+1\n if self.lower_neighbors[0]>=0:\n self.in_lower_buffers[0] = zeros(size1, float)\n self.out_lower_buffers[0] = zeros(size1, float)\n if self.upper_neighbors[0]>=0:\n self.in_upper_buffers[0] = zeros(size1, float)\n self.out_upper_buffers[0] = zeros(size1, float)\n\n size0 = self.subd_hi_ix[0]-self.subd_lo_ix[0]+1\n if self.lower_neighbors[1]>=0:\n self.in_lower_buffers[1] = zeros(size0, float)\n self.out_lower_buffers[1] = zeros(size0, float)\n if self.upper_neighbors[1]>=0:\n self.in_upper_buffers[1] = zeros(size0, float)\n self.out_upper_buffers[1] = zeros(size0, float)", "response": "Prepares the buffers to be used for later communications."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nupdate the inner boundary with the same send and recv pattern as the MPIPartitioner", "response": "def update_internal_boundary_x_y (self, solution_array):\n \"\"\"update the inner boundary with the same send/recv pattern as the MPIPartitioner\"\"\"\n nsd_ = self.nsd\n dtype = solution_array.dtype\n if nsd_!=len(self.in_lower_buffers) | nsd_!=len(self.out_lower_buffers):\n print(\"Buffers for communicating with lower neighbors not ready\")\n return\n if nsd_!=len(self.in_upper_buffers) | nsd_!=len(self.out_upper_buffers):\n print(\"Buffers for communicating with upper neighbors not ready\")\n return\n\n loc_nx = self.subd_hi_ix[0]-self.subd_lo_ix[0]\n loc_ny = self.subd_hi_ix[1]-self.subd_lo_ix[1]\n\n lower_x_neigh = self.lower_neighbors[0]\n upper_x_neigh = self.upper_neighbors[0]\n lower_y_neigh = self.lower_neighbors[1]\n upper_y_neigh = self.upper_neighbors[1]\n trackers = []\n flags = dict(copy=False, track=False)\n # communicate in the x-direction first\n if lower_x_neigh>-1:\n if self.slice_copy:\n self.out_lower_buffers[0] = ascontiguousarray(solution_array[1,:])\n else:\n for i in xrange(0,loc_ny+1):\n self.out_lower_buffers[0][i] = solution_array[1,i]\n t = self.comm.west.send(self.out_lower_buffers[0], **flags)\n trackers.append(t)\n \n if upper_x_neigh>-1:\n msg = self.comm.east.recv(copy=False)\n self.in_upper_buffers[0] = frombuffer(msg, dtype=dtype)\n if self.slice_copy:\n solution_array[loc_nx,:] = self.in_upper_buffers[0]\n self.out_upper_buffers[0] = ascontiguousarray(solution_array[loc_nx-1,:])\n else:\n for i in xrange(0,loc_ny+1):\n solution_array[loc_nx,i] = self.in_upper_buffers[0][i]\n self.out_upper_buffers[0][i] = solution_array[loc_nx-1,i]\n t = self.comm.east.send(self.out_upper_buffers[0], **flags)\n trackers.append(t)\n \n\n if lower_x_neigh>-1:\n msg = self.comm.west.recv(copy=False)\n self.in_lower_buffers[0] = frombuffer(msg, dtype=dtype)\n if self.slice_copy:\n solution_array[0,:] = self.in_lower_buffers[0]\n else:\n for i in xrange(0,loc_ny+1):\n solution_array[0,i] = self.in_lower_buffers[0][i]\n \n # communicate in the y-direction afterwards\n if lower_y_neigh>-1:\n if self.slice_copy:\n self.out_lower_buffers[1] = ascontiguousarray(solution_array[:,1])\n else:\n for i in xrange(0,loc_nx+1):\n self.out_lower_buffers[1][i] = solution_array[i,1]\n t = self.comm.south.send(self.out_lower_buffers[1], **flags)\n trackers.append(t)\n \n \n if upper_y_neigh>-1:\n msg = self.comm.north.recv(copy=False)\n self.in_upper_buffers[1] = frombuffer(msg, dtype=dtype)\n if self.slice_copy:\n solution_array[:,loc_ny] = self.in_upper_buffers[1]\n self.out_upper_buffers[1] = ascontiguousarray(solution_array[:,loc_ny-1])\n else:\n for i in xrange(0,loc_nx+1):\n solution_array[i,loc_ny] = self.in_upper_buffers[1][i]\n self.out_upper_buffers[1][i] = solution_array[i,loc_ny-1]\n t = self.comm.north.send(self.out_upper_buffers[1], **flags)\n trackers.append(t)\n \n if lower_y_neigh>-1:\n msg = self.comm.south.recv(copy=False)\n self.in_lower_buffers[1] = frombuffer(msg, dtype=dtype)\n if self.slice_copy:\n solution_array[:,0] = self.in_lower_buffers[1]\n else:\n for i in xrange(0,loc_nx+1):\n solution_array[i,0] = self.in_lower_buffers[1][i]\n \n # wait for sends to complete:\n if flags['track']:\n for t in trackers:\n t.wait()"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef rekey(dikt):\n for k in dikt.iterkeys():\n if isinstance(k, basestring):\n ik=fk=None\n try:\n ik = int(k)\n except ValueError:\n try:\n fk = float(k)\n except ValueError:\n continue\n if ik is not None:\n nk = ik\n else:\n nk = fk\n if nk in dikt:\n raise KeyError(\"already have key %r\"%nk)\n dikt[nk] = dikt.pop(k)\n return dikt", "response": "Rekey a dict that has been forced to use str keys where there should be be\n ints by json."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nextracts ISO8601 dates from unpacked JSON", "response": "def extract_dates(obj):\n \"\"\"extract ISO8601 dates from unpacked JSON\"\"\"\n if isinstance(obj, dict):\n obj = dict(obj) # don't clobber\n for k,v in obj.iteritems():\n obj[k] = extract_dates(v)\n elif isinstance(obj, (list, tuple)):\n obj = [ extract_dates(o) for o in obj ]\n elif isinstance(obj, basestring):\n if ISO8601_PAT.match(obj):\n obj = datetime.strptime(obj, ISO8601)\n return obj"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef squash_dates(obj):\n if isinstance(obj, dict):\n obj = dict(obj) # don't clobber\n for k,v in obj.iteritems():\n obj[k] = squash_dates(v)\n elif isinstance(obj, (list, tuple)):\n obj = [ squash_dates(o) for o in obj ]\n elif isinstance(obj, datetime):\n obj = obj.strftime(ISO8601)\n return obj", "response": "squash datetime objects into ISO8601 strings"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef date_default(obj):\n if isinstance(obj, datetime):\n return obj.strftime(ISO8601)\n else:\n raise TypeError(\"%r is not JSON serializable\"%obj)", "response": "default function for packing datetime objects in JSON"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef encode_images(format_dict):\n encoded = format_dict.copy()\n pngdata = format_dict.get('image/png')\n if isinstance(pngdata, bytes) and pngdata[:8] == PNG:\n encoded['image/png'] = encodestring(pngdata).decode('ascii')\n jpegdata = format_dict.get('image/jpeg')\n if isinstance(jpegdata, bytes) and jpegdata[:2] == JPEG:\n encoded['image/jpeg'] = encodestring(jpegdata).decode('ascii')\n return encoded", "response": "b64 - encodes images in a displaypub format dict\n \n "} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncleans an object to ensure it s safe to encode in JSON.", "response": "def json_clean(obj):\n \"\"\"Clean an object to ensure it's safe to encode in JSON.\n \n Atomic, immutable objects are returned unmodified. Sets and tuples are\n converted to lists, lists are copied and dicts are also copied.\n\n Note: dicts whose keys could cause collisions upon encoding (such as a dict\n with both the number 1 and the string '1' as keys) will cause a ValueError\n to be raised.\n\n Parameters\n ----------\n obj : any python object\n\n Returns\n -------\n out : object\n \n A version of the input which will not cause an encoding error when\n encoded as JSON. Note that this function does not *encode* its inputs,\n it simply sanitizes it so that there will be no encoding errors later.\n\n Examples\n --------\n >>> json_clean(4)\n 4\n >>> json_clean(range(10))\n [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n >>> sorted(json_clean(dict(x=1, y=2)).items())\n [('x', 1), ('y', 2)]\n >>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())\n [('x', 1), ('y', 2), ('z', [1, 2, 3])]\n >>> json_clean(True)\n True\n \"\"\"\n # types that are 'atomic' and ok in json as-is. bool doesn't need to be\n # listed explicitly because bools pass as int instances\n atomic_ok = (unicode, int, types.NoneType)\n \n # containers that we need to convert into lists\n container_to_list = (tuple, set, types.GeneratorType)\n\n if isinstance(obj, float):\n # cast out-of-range floats to their reprs\n if math.isnan(obj) or math.isinf(obj):\n return repr(obj)\n return obj\n\n if isinstance(obj, atomic_ok):\n return obj\n \n if isinstance(obj, bytes):\n return obj.decode(DEFAULT_ENCODING, 'replace')\n \n if isinstance(obj, container_to_list) or (\n hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):\n obj = list(obj)\n \n if isinstance(obj, list):\n return [json_clean(x) for x in obj]\n\n if isinstance(obj, dict):\n # First, validate that the dict won't lose data in conversion due to\n # key collisions after stringification. This can happen with keys like\n # True and 'true' or 1 and '1', which collide in JSON.\n nkeys = len(obj)\n nkeys_collapsed = len(set(map(str, obj)))\n if nkeys != nkeys_collapsed:\n raise ValueError('dict can not be safely converted to JSON: '\n 'key collision would lead to dropped values')\n # If all OK, proceed by making the new dict that will be json-safe\n out = {}\n for k,v in obj.iteritems():\n out[str(k)] = json_clean(v)\n return out\n\n # If we get here, we don't know how to handle the object, so we just get\n # its repr and return that. This will catch lambdas, open sockets, class\n # objects, and any other complicated contraption that json can't encode\n return repr(obj)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nverifies that self. install_dir is. pth - capable dir if needed", "response": "def check_site_dir(self):\n \"\"\"Verify that self.install_dir is .pth-capable dir, if needed\"\"\"\n\n instdir = normalize_path(self.install_dir)\n pth_file = os.path.join(instdir, 'easy-install.pth')\n\n # Is it a configured, PYTHONPATH, implicit, or explicit site dir?\n is_site_dir = instdir in self.all_site_dirs\n\n if not is_site_dir and not self.multi_version:\n # No? Then directly test whether it does .pth file processing\n is_site_dir = self.check_pth_processing()\n else:\n # make sure we can write to target dir\n testfile = self.pseudo_tempname() + '.write-test'\n test_exists = os.path.exists(testfile)\n try:\n if test_exists:\n os.unlink(testfile)\n open(testfile, 'w').close()\n os.unlink(testfile)\n except (OSError, IOError):\n self.cant_write_to_target()\n\n if not is_site_dir and not self.multi_version:\n # Can't install non-multi to non-site dir\n raise DistutilsError(self.no_default_version_msg())\n\n if is_site_dir:\n if self.pth_file is None:\n self.pth_file = PthDistributions(pth_file, self.all_site_dirs)\n else:\n self.pth_file = None\n\n PYTHONPATH = os.environ.get('PYTHONPATH', '').split(os.pathsep)\n if instdir not in map(normalize_path, [_f for _f in PYTHONPATH if _f]):\n # only PYTHONPATH dirs need a site.py, so pretend it's there\n self.sitepy_installed = True\n elif self.multi_version and not os.path.exists(pth_file):\n self.sitepy_installed = True # don't need site.py in this case\n self.pth_file = None # and don't create a .pth file\n self.install_dir = instdir"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite an executable file to the scripts directory", "response": "def write_script(self, script_name, contents, mode=\"t\", *ignored):\n \"\"\"Write an executable file to the scripts directory\"\"\"\n from setuptools.command.easy_install import chmod, current_umask\n log.info(\"Installing %s script to %s\", script_name, self.install_dir)\n target = os.path.join(self.install_dir, script_name)\n self.outfiles.append(target)\n\n mask = current_umask()\n if not self.dry_run:\n ensure_directory(target)\n f = open(target,\"w\"+mode)\n f.write(contents)\n f.close()\n chmod(target, 0777-mask)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sleep_here(count, t):\n import time,sys\n print(\"hi from engine %i\" % id)\n sys.stdout.flush()\n time.sleep(t)\n return count,t", "response": "simple function that prints a short message sleeps for a time and returns the same args"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _save_method_args(self, *args, **kwargs):\n self._method_args = args\n self._method_kwargs = kwargs", "response": "Save the args and kwargs for the method to get post put delete for future use."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run_from_argv(self, argv):\n parser = self.create_parser(argv[0], argv[1])\n self.arguments = parser.parse_args(argv[2:])\n handle_default_options(self.arguments)\n options = vars(self.arguments)\n self.execute(**options)", "response": "Run this command from the command line arguments."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating and return the ArgumentParser which will be used to parse the arguments to this command.", "response": "def create_parser(self, prog_name, subcommand):\n \"\"\"\n Create and return the ``ArgumentParser`` which will be used to\n parse the arguments to this command.\n\n \"\"\" \n parser = ArgumentParser(\n description=self.description,\n epilog=self.epilog,\n add_help=self.add_help,\n prog=self.prog,\n usage=self.get_usage(subcommand),\n )\n parser.add_argument('--version', action='version', version=self.get_version())\n self.add_arguments(parser)\n return parser"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconnect to the local system.", "response": "def connect(self, south_peer=None, west_peer=None):\n \"\"\"connect to peers. `peers` will be a 3-tuples, of the form:\n (location, north_addr, east_addr)\n as produced by\n \"\"\"\n if south_peer is not None:\n location, url, _ = south_peer\n self.south.connect(disambiguate_url(url, location))\n if west_peer is not None:\n location, _, url = west_peer\n self.west.connect(disambiguate_url(url, location))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _convert_pyx_sources_to_c(self):\n \"convert .pyx extensions to .c\"\n def pyx_to_c(source):\n if source.endswith('.pyx'):\n source = source[:-4] + '.c'\n return source\n self.sources = map(pyx_to_c, self.sources)", "response": "convert. pyx extensions to. c"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwatches iopub channel and print messages", "response": "def main(connection_file):\n \"\"\"watch iopub channel, and print messages\"\"\"\n \n ctx = zmq.Context.instance()\n \n with open(connection_file) as f:\n cfg = json.loads(f.read())\n \n location = cfg['location']\n reg_url = cfg['url']\n session = Session(key=str_to_bytes(cfg['exec_key']))\n \n query = ctx.socket(zmq.DEALER)\n query.connect(disambiguate_url(cfg['url'], location))\n session.send(query, \"connection_request\")\n idents,msg = session.recv(query, mode=0)\n c = msg['content']\n iopub_url = disambiguate_url(c['iopub'], location)\n sub = ctx.socket(zmq.SUB)\n # This will subscribe to all messages:\n sub.setsockopt(zmq.SUBSCRIBE, b'')\n # replace with b'' with b'engine.1.stdout' to subscribe only to engine 1's stdout\n # 0MQ subscriptions are simple 'foo*' matches, so 'engine.1.' subscribes\n # to everything from engine 1, but there is no way to subscribe to\n # just stdout from everyone.\n # multiple calls to subscribe will add subscriptions, e.g. to subscribe to\n # engine 1's stderr and engine 2's stdout:\n # sub.setsockopt(zmq.SUBSCRIBE, b'engine.1.stderr')\n # sub.setsockopt(zmq.SUBSCRIBE, b'engine.2.stdout')\n sub.connect(iopub_url)\n while True:\n try:\n idents,msg = session.recv(sub, mode=0)\n except KeyboardInterrupt:\n return\n # ident always length 1 here\n topic = idents[0]\n if msg['msg_type'] == 'stream':\n # stdout/stderr\n # stream names are in msg['content']['name'], if you want to handle\n # them differently\n print(\"%s: %s\" % (topic, msg['content']['data']))\n elif msg['msg_type'] == 'pyerr':\n # Python traceback\n c = msg['content']\n print(topic + ':')\n for line in c['traceback']:\n # indent lines\n print(' ' + line)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef decorator(caller, func=None):\n if func is not None: # returns a decorated function\n evaldict = func.func_globals.copy()\n evaldict['_call_'] = caller\n evaldict['_func_'] = func\n return FunctionMaker.create(\n func, \"return _call_(_func_, %(shortsignature)s)\",\n evaldict, undecorated=func, __wrapped__=func)\n else: # returns a decorator\n if isinstance(caller, partial):\n return partial(decorator, caller)\n # otherwise assume caller is a function\n first = inspect.getargspec(caller)[0][0] # first arg\n evaldict = caller.func_globals.copy()\n evaldict['_call_'] = caller\n evaldict['decorator'] = decorator\n return FunctionMaker.create(\n '%s(%s)' % (caller.__name__, first),\n 'return decorator(_call_, %s)' % first,\n evaldict, undecorated=caller, __wrapped__=caller,\n doc=caller.__doc__, module=caller.__module__)", "response": "Decorator that converts a caller function into a decorator ;"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbuild a package finder appropriate to this install command.", "response": "def _build_package_finder(self, options, index_urls, session):\n \"\"\"\n Create a package finder appropriate to this install command.\n This method is meant to be overridden by subclasses, not\n called directly.\n \"\"\"\n return PackageFinder(\n find_links=options.find_links,\n index_urls=index_urls,\n use_wheel=options.use_wheel,\n allow_external=options.allow_external,\n allow_unverified=options.allow_unverified,\n allow_all_external=options.allow_all_external,\n trusted_hosts=options.trusted_hosts,\n allow_all_prereleases=options.pre,\n process_dependency_links=options.process_dependency_links,\n session=session,\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef boolean_flag(name, configurable, set_help='', unset_help=''):\n # default helpstrings\n set_help = set_help or \"set %s=True\"%configurable\n unset_help = unset_help or \"set %s=False\"%configurable\n\n cls,trait = configurable.split('.')\n\n setter = {cls : {trait : True}}\n unsetter = {cls : {trait : False}}\n return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}", "response": "Helper for building basic boolean flag."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadjusting the log level when log_level is set.", "response": "def _log_level_changed(self, name, old, new):\n \"\"\"Adjust the log level when log_level is set.\"\"\"\n if isinstance(new, basestring):\n new = getattr(logging, new)\n self.log_level = new\n self.log.setLevel(new)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _log_default(self):\n log = logging.getLogger(self.__class__.__name__)\n log.setLevel(self.log_level)\n if sys.executable.endswith('pythonw.exe'):\n # this should really go to a file, but file-logging is only\n # hooked up in parallel applications\n _log_handler = logging.StreamHandler(open(os.devnull, 'w'))\n else:\n _log_handler = logging.StreamHandler()\n _log_formatter = logging.Formatter(self.log_format)\n _log_handler.setFormatter(_log_formatter)\n log.addHandler(_log_handler)\n return log", "response": "Start logging for this application."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _flags_changed(self, name, old, new):\n for key,value in new.iteritems():\n assert len(value) == 2, \"Bad flag: %r:%s\"%(key,value)\n assert isinstance(value[0], (dict, Config)), \"Bad flag: %r:%s\"%(key,value)\n assert isinstance(value[1], basestring), \"Bad flag: %r:%s\"%(key,value)", "response": "ensure flags dict is valid"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprints the alias part of the help.", "response": "def print_alias_help(self):\n \"\"\"Print the alias part of the help.\"\"\"\n if not self.aliases:\n return\n\n lines = []\n classdict = {}\n for cls in self.classes:\n # include all parents (up to, but excluding Configurable) in available names\n for c in cls.mro()[:-3]:\n classdict[c.__name__] = c\n\n for alias, longname in self.aliases.iteritems():\n classname, traitname = longname.split('.',1)\n cls = classdict[classname]\n\n trait = cls.class_traits(config=True)[traitname]\n help = cls.class_get_trait_help(trait).splitlines()\n # reformat first line\n help[0] = help[0].replace(longname, alias) + ' (%s)'%longname\n if len(alias) == 1:\n help[0] = help[0].replace('--%s='%alias, '-%s '%alias)\n lines.extend(help)\n # lines.append('')\n print os.linesep.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprints the flag part of the help.", "response": "def print_flag_help(self):\n \"\"\"Print the flag part of the help.\"\"\"\n if not self.flags:\n return\n\n lines = []\n for m, (cfg,help) in self.flags.iteritems():\n prefix = '--' if len(m) > 1 else '-'\n lines.append(prefix+m)\n lines.append(indent(dedent(help.strip())))\n # lines.append('')\n print os.linesep.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef print_subcommands(self):\n if not self.subcommands:\n return\n\n lines = [\"Subcommands\"]\n lines.append('-'*len(lines[0]))\n lines.append('')\n for p in wrap_paragraphs(self.subcommand_description):\n lines.append(p)\n lines.append('')\n for subc, (cls, help) in self.subcommands.iteritems():\n lines.append(subc)\n if help:\n lines.append(indent(dedent(help.strip())))\n lines.append('')\n print os.linesep.join(lines)", "response": "Print the subcommand part of the help."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef print_help(self, classes=False):\n self.print_subcommands()\n self.print_options()\n\n if classes:\n if self.classes:\n print \"Class parameters\"\n print \"----------------\"\n print\n for p in wrap_paragraphs(self.keyvalue_description):\n print p\n print\n\n for cls in self.classes:\n cls.class_print_help()\n print\n else:\n print \"To see all available configurables, use `--help-all`\"\n print", "response": "Print the help for each Configurable class in self. classes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nprinting usage and examples.", "response": "def print_examples(self):\n \"\"\"Print usage and examples.\n\n This usage string goes at the end of the command line help string\n and should contain examples of the application's usage.\n \"\"\"\n if self.examples:\n print \"Examples\"\n print \"--------\"\n print\n print indent(dedent(self.examples.strip()))\n print"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update_config(self, config):\n # Save a copy of the current config.\n newconfig = deepcopy(self.config)\n # Merge the new config into the current one.\n newconfig._merge(config)\n # Save the combined config as self.config, which triggers the traits\n # events.\n self.config = newconfig", "response": "Update the current config with the new config."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninitialize a subcommand with argv.", "response": "def initialize_subcommand(self, subc, argv=None):\n \"\"\"Initialize a subcommand with argv.\"\"\"\n subapp,help = self.subcommands.get(subc)\n\n if isinstance(subapp, basestring):\n subapp = import_item(subapp)\n\n # clear existing instances\n self.__class__.clear_instance()\n # instantiate\n self.subapp = subapp.instance()\n # and initialize subapp\n self.subapp.initialize(argv)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nflatten flags and aliases so cl - args override as expected.", "response": "def flatten_flags(self):\n \"\"\"flatten flags and aliases, so cl-args override as expected.\n \n This prevents issues such as an alias pointing to InteractiveShell,\n but a config file setting the same trait in TerminalInteraciveShell\n getting inappropriate priority over the command-line arg.\n\n Only aliases with exactly one descendent in the class list\n will be promoted.\n \n \"\"\"\n # build a tree of classes in our list that inherit from a particular\n # it will be a dict by parent classname of classes in our list\n # that are descendents\n mro_tree = defaultdict(list)\n for cls in self.classes:\n clsname = cls.__name__\n for parent in cls.mro()[1:-3]:\n # exclude cls itself and Configurable,HasTraits,object\n mro_tree[parent.__name__].append(clsname)\n # flatten aliases, which have the form:\n # { 'alias' : 'Class.trait' }\n aliases = {}\n for alias, cls_trait in self.aliases.iteritems():\n cls,trait = cls_trait.split('.',1)\n children = mro_tree[cls]\n if len(children) == 1:\n # exactly one descendent, promote alias\n cls = children[0]\n aliases[alias] = '.'.join([cls,trait])\n \n # flatten flags, which are of the form:\n # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}\n flags = {}\n for key, (flagdict, help) in self.flags.iteritems():\n newflag = {}\n for cls, subdict in flagdict.iteritems():\n children = mro_tree[cls]\n # exactly one descendent, promote flag section\n if len(children) == 1:\n cls = children[0]\n newflag[cls] = subdict\n flags[key] = (newflag, help)\n return flags, aliases"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef parse_command_line(self, argv=None):\n argv = sys.argv[1:] if argv is None else argv\n \n if argv and argv[0] == 'help':\n # turn `ipython help notebook` into `ipython notebook -h`\n argv = argv[1:] + ['-h']\n\n if self.subcommands and len(argv) > 0:\n # we have subcommands, and one may have been specified\n subc, subargv = argv[0], argv[1:]\n if re.match(r'^\\w(\\-?\\w)*$', subc) and subc in self.subcommands:\n # it's a subcommand, and *not* a flag or class parameter\n return self.initialize_subcommand(subc, subargv)\n\n if '-h' in argv or '--help' in argv or '--help-all' in argv:\n self.print_description()\n self.print_help('--help-all' in argv)\n self.print_examples()\n self.exit(0)\n\n if '--version' in argv or '-V' in argv:\n self.print_version()\n self.exit(0)\n \n # flatten flags&aliases, so cl-args get appropriate priority:\n flags,aliases = self.flatten_flags()\n\n loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,\n flags=flags)\n config = loader.load_config()\n self.update_config(config)\n # store unparsed args in extra_args\n self.extra_args = loader.extra_args", "response": "Parse the command line arguments."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_config_file(self, filename, path=None):\n loader = PyFileConfigLoader(filename, path=path)\n try:\n config = loader.load_config()\n except ConfigFileNotFound:\n # problem finding the file, raise\n raise\n except Exception:\n # try to get the full filename, but it will be empty in the\n # unlikely event that the error raised before filefind finished\n filename = loader.full_filename or filename\n # problem while running the file\n self.log.error(\"Exception while loading config file %s\",\n filename, exc_info=True)\n else:\n self.log.debug(\"Loaded config file: %s\", loader.full_filename)\n self.update_config(config)", "response": "Load a. py based config file by filename and path."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef generate_config_file(self):\n lines = [\"# Configuration file for %s.\"%self.name]\n lines.append('')\n lines.append('c = get_config()')\n lines.append('')\n for cls in self.classes:\n lines.append(cls.class_config_section())\n return '\\n'.join(lines)", "response": "generate default config file from Configurables"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef downsample(array, k):\n length = array.shape[0]\n indices = random.sample(xrange(length), k)\n return array[indices]", "response": "Choose k random elements of array."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef info_formatter(info):\n label_len = max([len(l) for l, _d in info])\n for label, data in info:\n if data == []:\n data = \"-none-\"\n if isinstance(data, (list, tuple)):\n prefix = \"%*s:\" % (label_len, label)\n for e in data:\n yield \"%*s %s\" % (label_len+1, prefix, e)\n prefix = \"\"\n else:\n yield \"%*s: %s\" % (label_len, label, data)", "response": "Produce a sequence of formatted lines from info."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write(self, msg):\n if self.should('pid'):\n msg = \"pid %5d: %s\" % (os.getpid(), msg)\n self.output.write(msg+\"\\n\")\n self.output.flush()", "response": "Write a line of debug output."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nupdate all the class traits having a config metadata attribute that is True as metadata.", "response": "def _config_changed(self, name, old, new):\n \"\"\"Update all the class traits having ``config=True`` as metadata.\n\n For any class trait with a ``config`` metadata attribute that is\n ``True``, we update the trait with the value of the corresponding\n config entry.\n \"\"\"\n # Get all traits with a config metadata entry that is True\n traits = self.traits(config=True)\n\n # We auto-load config section for this class as well as any parent\n # classes that are Configurable subclasses. This starts with Configurable\n # and works down the mro loading the config for each section.\n section_names = [cls.__name__ for cls in \\\n reversed(self.__class__.__mro__) if\n issubclass(cls, Configurable) and issubclass(self.__class__, cls)]\n\n for sname in section_names:\n # Don't do a blind getattr as that would cause the config to\n # dynamically create the section with name self.__class__.__name__.\n if new._has_section(sname):\n my_config = new[sname]\n for k, v in traits.iteritems():\n # Don't allow traitlets with config=True to start with\n # uppercase. Otherwise, they are confused with Config\n # subsections. But, developers shouldn't have uppercase\n # attributes anyways! (PEP 6)\n if k[0].upper()==k[0] and not k.startswith('_'):\n raise ConfigurableError('Configurable traitlets with '\n 'config=True must start with a lowercase so they are '\n 'not confused with Config subsections: %s.%s' % \\\n (self.__class__.__name__, k))\n try:\n # Here we grab the value from the config\n # If k has the naming convention of a config\n # section, it will be auto created.\n config_value = my_config[k]\n except KeyError:\n pass\n else:\n # print \"Setting %s.%s from %s.%s=%r\" % \\\n # (self.__class__.__name__,k,sname,k,config_value)\n # We have to do a deepcopy here if we don't deepcopy the entire\n # config object. If we don't, a mutable config_value will be\n # shared by all instances, effectively making it a class attribute.\n setattr(self, k, deepcopy(config_value))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef class_get_help(cls, inst=None):\n assert inst is None or isinstance(inst, cls)\n cls_traits = cls.class_traits(config=True)\n final_help = []\n final_help.append(u'%s options' % cls.__name__)\n final_help.append(len(final_help[0])*u'-')\n for k,v in sorted(cls.class_traits(config=True).iteritems()):\n help = cls.class_get_trait_help(v, inst)\n final_help.append(help)\n return '\\n'.join(final_help)", "response": "Get the help string for this class in ReST format."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef class_get_trait_help(cls, trait, inst=None):\n assert inst is None or isinstance(inst, cls)\n lines = []\n header = \"--%s.%s=<%s>\" % (cls.__name__, trait.name, trait.__class__.__name__)\n lines.append(header)\n if inst is not None:\n lines.append(indent('Current: %r' % getattr(inst, trait.name), 4))\n else:\n try:\n dvr = repr(trait.get_default_value())\n except Exception:\n dvr = None # ignore defaults we can't construct\n if dvr is not None:\n if len(dvr) > 64:\n dvr = dvr[:61]+'...'\n lines.append(indent('Default: %s' % dvr, 4))\n if 'Enum' in trait.__class__.__name__:\n # include Enum choices\n lines.append(indent('Choices: %r' % (trait.values,)))\n\n help = trait.get_metadata('help')\n if help is not None:\n help = '\\n'.join(wrap_paragraphs(help, 76))\n lines.append(indent(help, 4))\n return '\\n'.join(lines)", "response": "Get the help string for a single trait."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef class_config_section(cls):\n def c(s):\n \"\"\"return a commented, wrapped block.\"\"\"\n s = '\\n\\n'.join(wrap_paragraphs(s, 78))\n\n return '# ' + s.replace('\\n', '\\n# ')\n\n # section header\n breaker = '#' + '-'*78\n s = \"# %s configuration\"%cls.__name__\n lines = [breaker, s, breaker, '']\n # get the description trait\n desc = cls.class_traits().get('description')\n if desc:\n desc = desc.default_value\n else:\n # no description trait, use __doc__\n desc = getattr(cls, '__doc__', '')\n if desc:\n lines.append(c(desc))\n lines.append('')\n\n parents = []\n for parent in cls.mro():\n # only include parents that are not base classes\n # and are not the class itself\n # and have some configurable traits to inherit\n if parent is not cls and issubclass(parent, Configurable) and \\\n parent.class_traits(config=True):\n parents.append(parent)\n\n if parents:\n pstr = ', '.join([ p.__name__ for p in parents ])\n lines.append(c('%s will inherit config from: %s'%(cls.__name__, pstr)))\n lines.append('')\n\n for name,trait in cls.class_traits(config=True).iteritems():\n help = trait.get_metadata('help') or ''\n lines.append(c(help))\n lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value()))\n lines.append('')\n return '\\n'.join(lines)", "response": "Get the config class config section."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwalk the cls. mro() for parent classes that are also singletons", "response": "def _walk_mro(cls):\n \"\"\"Walk the cls.mro() for parent classes that are also singletons\n\n For use in instance()\n \"\"\"\n\n for subclass in cls.mro():\n if issubclass(cls, subclass) and \\\n issubclass(subclass, SingletonConfigurable) and \\\n subclass != SingletonConfigurable:\n yield subclass"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef clear_instance(cls):\n if not cls.initialized():\n return\n for subclass in cls._walk_mro():\n if isinstance(subclass._instance, cls):\n # only clear instances that are instances\n # of the calling class\n subclass._instance = None", "response": "clear _instance for this class and singleton parents."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a global instance of the given class.", "response": "def instance(cls, *args, **kwargs):\n \"\"\"Returns a global instance of this class.\n\n This method create a new instance if none have previously been created\n and returns a previously created instance is one already exists.\n\n The arguments and keyword arguments passed to this method are passed\n on to the :meth:`__init__` method of the class upon instantiation.\n\n Examples\n --------\n\n Create a singleton class using instance, and retrieve it::\n\n >>> from IPython.config.configurable import SingletonConfigurable\n >>> class Foo(SingletonConfigurable): pass\n >>> foo = Foo.instance()\n >>> foo == Foo.instance()\n True\n\n Create a subclass that is retrived using the base class instance::\n\n >>> class Bar(SingletonConfigurable): pass\n >>> class Bam(Bar): pass\n >>> bam = Bam.instance()\n >>> bam == Bar.instance()\n True\n \"\"\"\n # Create and save the instance\n if cls._instance is None:\n inst = cls(*args, **kwargs)\n # Now make sure that the instance will also be returned by\n # parent classes' _instance attribute.\n for subclass in cls._walk_mro():\n subclass._instance = inst\n\n if isinstance(cls._instance, cls):\n return cls._instance\n else:\n raise MultipleInstanceError(\n 'Multiple incompatible subclass instances of '\n '%s are being created.' % cls.__name__\n )"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconfigure the object based on the options and conf.", "response": "def configure(self, options, conf):\n \"\"\"Configure plugin.\n \"\"\"\n if not self.can_configure:\n return\n self.enabled = options.detailedErrors\n self.conf = conf"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nformat a failure message.", "response": "def formatFailure(self, test, err):\n \"\"\"Add detail from traceback inspection to error message of a failure.\n \"\"\"\n ec, ev, tb = err\n tbinfo = inspect_traceback(tb)\n test.tbinfo = tbinfo\n return (ec, '\\n'.join([str(ev), tbinfo]), tb)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef make_report(self,traceback):\n\n sec_sep = self.section_sep\n\n report = ['*'*75+'\\n\\n'+'IPython post-mortem report\\n\\n']\n rpt_add = report.append\n rpt_add(sys_info())\n\n try:\n config = pformat(self.app.config)\n rpt_add(sec_sep)\n rpt_add('Application name: %s\\n\\n' % self.app_name)\n rpt_add('Current user configuration structure:\\n\\n')\n rpt_add(config)\n except:\n pass\n rpt_add(sec_sep+'Crash traceback:\\n\\n' + traceback)\n\n return ''.join(report)", "response": "Return a string containing a crash report."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the value of a variable in a resource tree.", "response": "def setvar(parser, token):\n \"\"\" {% setvar to %} \"\"\"\n\n try:\n setvar, var_name, to_, var_value = token.split_contents()\n except ValueError:\n raise template.TemplateSyntaxError('Invalid arguments for %r' % token.split_contents()[0])\n\n return SetVarNode(var_name, var_value)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalls by the handler methods when a message is received.", "response": "def call_handlers(self, msg):\n \"\"\" Reimplemented to emit signals instead of making callbacks.\n \"\"\"\n # Emit the generic signal.\n self.message_received.emit(msg)\n\n # Emit signals for specialized message types.\n msg_type = msg['header']['msg_type']\n signal = getattr(self, msg_type, None)\n if signal:\n signal.emit(msg)\n\n if not self._handlers_called:\n self.first_reply.emit()\n self._handlers_called = True"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef call_handlers(self, msg):\n # Emit the generic signal.\n self.message_received.emit(msg)\n # Emit signals for specialized message types.\n msg_type = msg['header']['msg_type']\n signal = getattr(self, msg_type + '_received', None)\n if signal:\n signal.emit(msg)\n elif msg_type in ('stdout', 'stderr'):\n self.stream_received.emit(msg)", "response": "Called by the message handlers."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nflush the current state of the socket.", "response": "def flush(self):\n \"\"\" Reimplemented to ensure that signals are dispatched immediately.\n \"\"\"\n super(QtSubSocketChannel, self).flush()\n QtCore.QCoreApplication.instance().processEvents()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef call_handlers(self, msg):\n # Emit the generic signal.\n self.message_received.emit(msg)\n\n # Emit signals for specialized message types.\n msg_type = msg['header']['msg_type']\n if msg_type == 'input_request':\n self.input_requested.emit(msg)", "response": "Called by the event handlers when a message is received."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef start_kernel(self, *args, **kw):\n if self._shell_channel is not None:\n self._shell_channel.reset_first_reply()\n super(QtKernelManager, self).start_kernel(*args, **kw)\n self.started_kernel.emit()", "response": "Reimplemented for proper heartbeat management."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nemitting the start_channels signal.", "response": "def start_channels(self, *args, **kw):\n \"\"\" Reimplemented to emit signal.\n \"\"\"\n super(QtKernelManager, self).start_channels(*args, **kw)\n self.started_channels.emit()"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrestoring bytes of image data from unicode - only formats.", "response": "def restore_bytes(nb):\n \"\"\"Restore bytes of image data from unicode-only formats.\n \n Base64 encoding is handled elsewhere. Bytes objects in the notebook are\n always b64-encoded. We DO NOT encode/decode around file formats.\n \"\"\"\n for ws in nb.worksheets:\n for cell in ws.cells:\n if cell.cell_type == 'code':\n for output in cell.outputs:\n if 'png' in output:\n output.png = str_to_bytes(output.png, 'ascii')\n if 'jpeg' in output:\n output.jpeg = str_to_bytes(output.jpeg, 'ascii')\n return nb"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _join_lines(lines):\n if lines and lines[0].endswith(('\\n', '\\r')):\n # created by splitlines(True)\n return u''.join(lines)\n else:\n # created by splitlines()\n return u'\\n'.join(lines)", "response": "join lines that have been written by splitlines"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrejoining multiline text into strings For reversing effects of split_lines ( nb )", "response": "def rejoin_lines(nb):\n \"\"\"rejoin multiline text into strings\n \n For reversing effects of ``split_lines(nb)``.\n \n This only rejoins lines that have been split, so if text objects were not split\n they will pass through unchanged.\n \n Used when reading JSON files that may have been passed through split_lines.\n \"\"\"\n for ws in nb.worksheets:\n for cell in ws.cells:\n if cell.cell_type == 'code':\n if 'input' in cell and isinstance(cell.input, list):\n cell.input = _join_lines(cell.input)\n for output in cell.outputs:\n for key in _multiline_outputs:\n item = output.get(key, None)\n if isinstance(item, list):\n output[key] = _join_lines(item)\n else: # text, heading cell\n for key in ['source', 'rendered']:\n item = cell.get(key, None)\n if isinstance(item, list):\n cell[key] = _join_lines(item)\n return nb"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef base64_decode(nb):\n for ws in nb.worksheets:\n for cell in ws.cells:\n if cell.cell_type == 'code':\n for output in cell.outputs:\n if 'png' in output:\n if isinstance(output.png, unicode):\n output.png = output.png.encode('ascii')\n output.png = decodestring(output.png)\n if 'jpeg' in output:\n if isinstance(output.jpeg, unicode):\n output.jpeg = output.jpeg.encode('ascii')\n output.jpeg = decodestring(output.jpeg)\n return nb", "response": "Restore all bytes objects in the notebook from base64 - encoded strings."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read(self, fp, **kwargs):\n nbs = fp.read()\n if not py3compat.PY3 and not isinstance(nbs, unicode):\n nbs = py3compat.str_to_unicode(nbs)\n return self.reads(nbs, **kwargs)", "response": "Read a notebook from a file like object"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting a notebook to a file like object", "response": "def write(self, nb, fp, **kwargs):\n \"\"\"Write a notebook to a file like object\"\"\"\n nbs = self.writes(nb,**kwargs)\n if not py3compat.PY3 and not isinstance(nbs, unicode):\n # this branch is likely only taken for JSON on Python 2\n nbs = py3compat.str_to_unicode(nbs)\n return fp.write(nbs)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_mirrors(hostname=None):\n if hostname is None:\n hostname = DEFAULT_MIRROR_HOSTNAME\n\n # return the last mirror registered on PyPI.\n last_mirror_hostname = None\n try:\n last_mirror_hostname = socket.gethostbyname_ex(hostname)[0]\n except socket.gaierror:\n return []\n if not last_mirror_hostname or last_mirror_hostname == DEFAULT_MIRROR_HOSTNAME:\n last_mirror_hostname = \"z.pypi.python.org\"\n end_letter = last_mirror_hostname.split(\".\", 1)\n\n # determine the list from the last one.\n return [\"%s.%s\" % (s, end_letter[1]) for s in string_range(end_letter[0])]", "response": "Return the list of mirrors from the last record found on the DNS\n entry."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read_no_interrupt(p):\n import errno\n\n try:\n return p.read()\n except IOError, err:\n if err.errno != errno.EINTR:\n raise", "response": "Read from a pipe ignoring EINTR errors."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nopen a command in a shell subprocess and execute a callback. This function provides common scaffolding for creating subprocess.Popen() calls. It creates a Popen object and then calls the callback with it. Parameters ---------- cmd : str A string to be executed with the underlying system shell (by calling :func:`Popen` with ``shell=True``. callback : callable A one-argument function that will be called with the Popen object. stderr : file descriptor number, optional By default this is set to ``subprocess.PIPE``, but you can also pass the value ``subprocess.STDOUT`` to force the subprocess' stderr to go into the same file descriptor as its stdout. This is useful to read stdout and stderr combined in the order they are generated. Returns ------- The return value of the provided callback is returned.", "response": "def process_handler(cmd, callback, stderr=subprocess.PIPE):\n \"\"\"Open a command in a shell subprocess and execute a callback.\n\n This function provides common scaffolding for creating subprocess.Popen()\n calls. It creates a Popen object and then calls the callback with it.\n\n Parameters\n ----------\n cmd : str\n A string to be executed with the underlying system shell (by calling\n :func:`Popen` with ``shell=True``.\n\n callback : callable\n A one-argument function that will be called with the Popen object.\n\n stderr : file descriptor number, optional\n By default this is set to ``subprocess.PIPE``, but you can also pass the\n value ``subprocess.STDOUT`` to force the subprocess' stderr to go into\n the same file descriptor as its stdout. This is useful to read stdout\n and stderr combined in the order they are generated.\n\n Returns\n -------\n The return value of the provided callback is returned.\n \"\"\"\n sys.stdout.flush()\n sys.stderr.flush()\n # On win32, close_fds can't be true when using pipes for stdin/out/err\n close_fds = sys.platform != 'win32'\n p = subprocess.Popen(cmd, shell=True,\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=stderr,\n close_fds=close_fds)\n\n try:\n out = callback(p)\n except KeyboardInterrupt:\n print('^C')\n sys.stdout.flush()\n sys.stderr.flush()\n out = None\n finally:\n # Make really sure that we don't leave processes behind, in case the\n # call above raises an exception\n # We start by assuming the subprocess finished (to avoid NameErrors\n # later depending on the path taken)\n if p.returncode is None:\n try:\n p.terminate()\n p.poll()\n except OSError:\n pass\n # One last try on our way out\n if p.returncode is None:\n try:\n p.kill()\n except OSError:\n pass\n\n return out"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the standard output of executing cmd in a system shell.", "response": "def getoutput(cmd):\n \"\"\"Return standard output of executing cmd in a shell.\n\n Accepts the same arguments as os.system().\n\n Parameters\n ----------\n cmd : str\n A command to be executed in the system shell.\n\n Returns\n -------\n stdout : str\n \"\"\"\n\n out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)\n if out is None:\n return ''\n return py3compat.bytes_to_str(out)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getoutputerror(cmd):\n\n out_err = process_handler(cmd, lambda p: p.communicate())\n if out_err is None:\n return '', ''\n out, err = out_err\n return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)", "response": "Returns the standard output and standard error of executing cmd in a system shell."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef arg_split(s, posix=False, strict=True):\n\n # Unfortunately, python's shlex module is buggy with unicode input:\n # http://bugs.python.org/issue1170\n # At least encoding the input when it's unicode seems to help, but there\n # may be more problems lurking. Apparently this is fixed in python3.\n is_unicode = False\n if (not py3compat.PY3) and isinstance(s, unicode):\n is_unicode = True\n s = s.encode('utf-8')\n lex = shlex.shlex(s, posix=posix)\n lex.whitespace_split = True\n # Extract tokens, ensuring that things like leaving open quotes\n # does not cause this to raise. This is important, because we\n # sometimes pass Python source through this (e.g. %timeit f(\" \")),\n # and it shouldn't raise an exception.\n # It may be a bad idea to parse things that are not command-line args\n # through this function, but we do, so let's be safe about it.\n lex.commenters='' #fix for GH-1269\n tokens = []\n while True:\n try:\n tokens.append(lex.next())\n except StopIteration:\n break\n except ValueError:\n if strict:\n raise\n # couldn't parse, get remaining blob as last token\n tokens.append(lex.token)\n break\n \n if is_unicode:\n # Convert the tokens back to unicode.\n tokens = [x.decode('utf-8') for x in tokens]\n return tokens", "response": "Split a command line s arguments in a shell - like manner."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef collector():\n # plugins that implement any of these methods are disabled, since\n # we don't control the test runner and won't be able to run them\n # finalize() is also not called, but plugins that use it aren't disabled,\n # because capture needs it.\n setuptools_incompat = ('report', 'prepareTest',\n 'prepareTestLoader', 'prepareTestRunner',\n 'setOutputStream')\n\n plugins = RestrictedPluginManager(exclude=setuptools_incompat)\n conf = Config(files=all_config_files(),\n plugins=plugins)\n conf.configure(argv=['collector'])\n loader = defaultTestLoader(conf)\n\n if conf.testNames:\n suite = loader.loadTestsFromNames(conf.testNames)\n else:\n suite = loader.loadTestsFromNames(('.',))\n return FinalizingSuiteWrapper(suite, plugins.finalize)", "response": "TestSuite replacement entry point."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncompresses a directory history into a new list of at most 20 entries.", "response": "def compress_dhist(dh):\n \"\"\"Compress a directory history into a new one with at most 20 entries.\n\n Return a new list made from the first and last 10 elements of dhist after\n removal of duplicates.\n \"\"\"\n head, tail = dh[:-10], dh[-10:]\n\n newhead = []\n done = set()\n for h in head:\n if h in done:\n continue\n newhead.append(h)\n done.add(h)\n\n return newhead + tail"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef magics_class(cls):\n cls.registered = True\n cls.magics = dict(line = magics['line'],\n cell = magics['cell'])\n magics['line'] = {}\n magics['cell'] = {}\n return cls", "response": "Class decorator for all subclasses of Magics"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _method_magic_marker(magic_kind):\n\n validate_type(magic_kind)\n\n # This is a closure to capture the magic_kind. We could also use a class,\n # but it's overkill for just that one bit of state.\n def magic_deco(arg):\n call = lambda f, *a, **k: f(*a, **k)\n\n if callable(arg):\n # \"Naked\" decorator call (just @foo, no args)\n func = arg\n name = func.func_name\n retval = decorator(call, func)\n record_magic(magics, magic_kind, name, name)\n elif isinstance(arg, basestring):\n # Decorator called with arguments (@foo('bar'))\n name = arg\n def mark(func, *a, **kw):\n record_magic(magics, magic_kind, name, func.func_name)\n return decorator(call, func)\n retval = mark\n else:\n raise TypeError(\"Decorator can only be called with \"\n \"string or function\")\n return retval\n\n # Ensure the resulting decorator has a usable docstring\n magic_deco.__doc__ = _docstring_template.format('method', magic_kind)\n return magic_deco", "response": "Decorator factory for methods in Magics subclasses."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef lsmagic_docs(self, brief=False, missing=''):\n docs = {}\n for m_type in self.magics:\n m_docs = {}\n for m_name, m_func in self.magics[m_type].iteritems():\n if m_func.__doc__:\n if brief:\n m_docs[m_name] = m_func.__doc__.split('\\n', 1)[0]\n else:\n m_docs[m_name] = m_func.__doc__.rstrip()\n else:\n m_docs[m_name] = missing\n docs[m_type] = m_docs\n return docs", "response": "Return a dict of documentation of magic functions."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nregistering one or more instances of Magics.", "response": "def register(self, *magic_objects):\n \"\"\"Register one or more instances of Magics.\n\n Take one or more classes or instances of classes that subclass the main \n `core.Magic` class, and register them with IPython to use the magic\n functions they provide. The registration process will then ensure that\n any methods that have decorated to provide line and/or cell magics will\n be recognized with the `%x`/`%%x` syntax as a line/cell magic\n respectively.\n\n If classes are given, they will be instantiated with the default\n constructor. If your classes need a custom constructor, you should\n instanitate them first and pass the instance.\n\n The provided arguments can be an arbitrary mix of classes and instances.\n\n Parameters\n ----------\n magic_objects : one or more classes or instances\n \"\"\"\n # Start by validating them to ensure they have all had their magic\n # methods registered at the instance level\n for m in magic_objects:\n if not m.registered:\n raise ValueError(\"Class of magics %r was constructed without \"\n \"the @register_magics class decorator\")\n if type(m) in (type, MetaHasTraits):\n # If we're given an uninstantiated class\n m = m(shell=self.shell)\n\n # Now that we have an instance, we can register it and update the\n # table of callables\n self.registry[m.__class__.__name__] = m\n for mtype in magic_kinds:\n self.magics[mtype].update(m.magics[mtype])"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef register_function(self, func, magic_kind='line', magic_name=None):\n\n # Create the new method in the user_magics and register it in the\n # global table\n validate_type(magic_kind)\n magic_name = func.func_name if magic_name is None else magic_name\n setattr(self.user_magics, magic_name, func)\n record_magic(self.magics, magic_kind, magic_name, func)", "response": "Expose a standalone function as a magic function for IPython."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndefine a magic function for the user object.", "response": "def define_magic(self, name, func):\n \"\"\"[Deprecated] Expose own function as magic function for IPython.\n\n Example::\n\n def foo_impl(self, parameter_s=''):\n 'My very own magic!. (Use docstrings, IPython reads them).'\n print 'Magic function. Passed parameter is between < >:'\n print '<%s>' % parameter_s\n print 'The self object is:', self\n\n ip.define_magic('foo',foo_impl)\n \"\"\"\n meth = types.MethodType(func, self.user_magics)\n setattr(self.user_magics, name, meth)\n record_magic(self.magics, 'line', name, meth)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nformatting a string for latex inclusion.", "response": "def format_latex(self, strng):\n \"\"\"Format a string for latex inclusion.\"\"\"\n\n # Characters that need to be escaped for latex:\n escape_re = re.compile(r'(%|_|\\$|#|&)',re.MULTILINE)\n # Magic command names as headers:\n cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,\n re.MULTILINE)\n # Magic commands\n cmd_re = re.compile(r'(?P%s.+?\\b)(?!\\}\\}:)' % ESC_MAGIC,\n re.MULTILINE)\n # Paragraph continue\n par_re = re.compile(r'\\\\$',re.MULTILINE)\n\n # The \"\\n\" symbol\n newline_re = re.compile(r'\\\\n')\n\n # Now build the string for output:\n #strng = cmd_name_re.sub(r'\\n\\\\texttt{\\\\textsl{\\\\large \\1}}:',strng)\n strng = cmd_name_re.sub(r'\\n\\\\bigskip\\n\\\\texttt{\\\\textbf{ \\1}}:',\n strng)\n strng = cmd_re.sub(r'\\\\texttt{\\g}',strng)\n strng = par_re.sub(r'\\\\\\\\',strng)\n strng = escape_re.sub(r'\\\\\\1',strng)\n strng = newline_re.sub(r'\\\\textbackslash{}n',strng)\n return strng"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse options passed to an argument string.", "response": "def parse_options(self, arg_str, opt_str, *long_opts, **kw):\n \"\"\"Parse options passed to an argument string.\n\n The interface is similar to that of getopt(), but it returns back a\n Struct with the options as keys and the stripped argument string still\n as a string.\n\n arg_str is quoted as a true sys.argv vector by using shlex.split.\n This allows us to easily expand variables, glob files, quote\n arguments, etc.\n\n Options:\n -mode: default 'string'. If given as 'list', the argument string is\n returned as a list (split on whitespace) instead of a string.\n\n -list_all: put all option values in lists. Normally only options\n appearing more than once are put in a list.\n\n -posix (True): whether to split the input line in POSIX mode or not,\n as per the conventions outlined in the shlex module from the\n standard library.\"\"\"\n\n # inject default options at the beginning of the input line\n caller = sys._getframe(1).f_code.co_name\n arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)\n\n mode = kw.get('mode','string')\n if mode not in ['string','list']:\n raise ValueError,'incorrect mode given: %s' % mode\n # Get options\n list_all = kw.get('list_all',0)\n posix = kw.get('posix', os.name == 'posix')\n strict = kw.get('strict', True)\n\n # Check if we have more than one argument to warrant extra processing:\n odict = {} # Dictionary with options\n args = arg_str.split()\n if len(args) >= 1:\n # If the list of inputs only has 0 or 1 thing in it, there's no\n # need to look for options\n argv = arg_split(arg_str, posix, strict)\n # Do regular option processing\n try:\n opts,args = getopt(argv, opt_str, long_opts)\n except GetoptError,e:\n raise UsageError('%s ( allowed: \"%s\" %s)' % (e.msg,opt_str,\n \" \".join(long_opts)))\n for o,a in opts:\n if o.startswith('--'):\n o = o[2:]\n else:\n o = o[1:]\n try:\n odict[o].append(a)\n except AttributeError:\n odict[o] = [odict[o],a]\n except KeyError:\n if list_all:\n odict[o] = [a]\n else:\n odict[o] = a\n\n # Prepare opts,args for return\n opts = Struct(odict)\n if mode == 'string':\n args = ' '.join(args)\n\n return opts,args"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmake an entry in the options_table for fn with value optstr", "response": "def default_option(self, fn, optstr):\n \"\"\"Make an entry in the options_table for fn, with value optstr\"\"\"\n\n if fn not in self.lsmagic():\n error(\"%s is not a magic function\" % fn)\n self.options_table[fn] = optstr"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef page_guiref(arg_s=None):\n from IPython.core import page\n page.page(gui_reference, auto_html=True)", "response": "Show a basic reference about the GUI Console."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_member(thing_obj, member_string):\n mems = {x[0]: x[1] for x in inspect.getmembers(thing_obj)}\n if member_string in mems:\n return mems[member_string]", "response": "Get a member from an object by name"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef func_from_string(callable_str):\n components = callable_str.split('.')\n\n func = None\n\n if len(components) < 2:\n raise ValueError(\"Need full dotted path to task function\")\n elif len(components) == 2:\n mod_name = components[0]\n func_name = components[1]\n try:\n mod = import_module(mod_name)\n except ModuleNotFoundError:\n raise ValueError(f\"Module {mod_name} not found\")\n func = get_member(mod, func_name)\n if func is None:\n raise ValueError(f\"{func_name} is not a member of {mod_name}\")\n else:\n mod_name = '.'.join(components[:-1])\n func_name = components[-1]\n\n try:\n mod = import_module(mod_name)\n except ModuleNotFoundError:\n mod_name = '.'.join(components[:-2])\n class_name = components[-2]\n\n try:\n mod = import_module(mod_name)\n except ModuleNotFoundError:\n raise ValueError(f\"Module {mod_name} not found\")\n\n klass = get_member(mod, class_name)\n if klass is None:\n raise ValueError(f\"Class {class_name} is not a member of {mod_name}\")\n\n func = get_member(klass, func_name)\n if func is None:\n raise ValueError(f\"Function {func_name} is not a member of {mod_name}.{class_name}\")\n\n if func is None:\n func = get_member(mod, func_name)\n if func is None:\n raise ValueError(f\"Function {func_name} is not a member of {mod_name}\")\n\n if inspect.ismodule(func):\n raise ValueError(\"Cannot call module directly\")\n\n if inspect.isclass(func):\n raise ValueError(\"Cannot call class directly\")\n\n try:\n sig = [x for x in inspect.signature(func).parameters]\n except TypeError:\n raise ValueError(f\"{callable_str} ({str(type(func))[1:-1]}) is not a callable object\")\n\n if len(sig) == 1:\n if sig[0] == 'message':\n return func\n else:\n raise ValueError(\"Task function must have one parameter, named 'message'\")\n\n elif len(sig)==2 and sig[0]=='self' and sig[1]=='message':\n # We only check for the conventional 'self', but if you're using something else,\n # you deserve the pain you'll have trying to debug this.\n raise ValueError(\"Can't call instance method without an instance! (Try sisy.models.task_with_callable)\")\n else:\n raise ValueError(\"Improper signature for task function (needs only 'message')\")", "response": "Return a live function from a full dotted path."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef task_with_callable(the_callable, label=None, schedule=DEFAULT_SCHEDULE, userdata=None, pk_override=None):\n task = Task()\n if isinstance(the_callable, str):\n if pk_override is not None:\n components = the_callable.split('.')\n info = dict(\n func_type='instancemethod',\n module_name='.'.join(components[:-2]),\n class_name=components[-2],\n class_path='.'.join(components[:-1]),\n model_pk=pk_override,\n func_name=components[-1],\n func_path=the_callable,\n )\n task.funcinfo = info\n else:\n task.funcinfo = get_func_info(func_from_string(the_callable))\n else:\n task.funcinfo = get_func_info(the_callable)\n\n if label is None:\n task.label = task.funcinfo['func_path']\n else:\n task.label = label\n\n task.schedule = schedule\n if not croniter.is_valid(task.schedule):\n raise ValueError(f\"Cron schedule {task.schedule} is not valid\")\n \n if userdata is None:\n task.userdata = dict()\n else:\n if isinstance(userdata, dict):\n task.userdata = userdata\n else:\n raise ValueError(\"Userdata must be a dictionary of JSON-serializable data\")\n\n return task", "response": "Factory function to create a properly initialized Task object with the callable."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns task info dictionary from task label. Internal function, pretty much only used in migrations since the model methods aren't there.", "response": "def taskinfo_with_label(label):\n \"\"\"Return task info dictionary from task label. Internal function,\n pretty much only used in migrations since the model methods aren't there.\"\"\"\n task = Task.objects.get(label=label)\n info = json.loads(task._func_info)\n return info"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfinds and return a callable object from a task info dictionary", "response": "def func_from_info(self):\n \"\"\"Find and return a callable object from a task info dictionary\"\"\"\n info = self.funcinfo\n functype = info['func_type']\n if functype in ['instancemethod', 'classmethod', 'staticmethod']:\n the_modelclass = get_module_member_by_dottedpath(info['class_path'])\n if functype == 'instancemethod':\n the_modelobject = the_modelclass.objects.get(pk=info['model_pk'])\n the_callable = get_member(the_modelobject, info['func_name'])\n else:\n the_callable = get_member(the_modelclass, info['func_name'])\n return the_callable\n elif functype == 'function':\n mod = import_module(info['module_name'])\n the_callable = get_member(mod, info['func_name'])\n return the_callable\n else:\n raise ValueError(f\"Unknown functype '{functype} in task {self.pk} ({self.label})\")"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating next run time of this task", "response": "def calc_next_run(self):\n \"\"\"Calculate next run time of this task\"\"\"\n base_time = self.last_run\n if self.last_run == HAS_NOT_RUN:\n if self.wait_for_schedule is False:\n self.next_run = timezone.now()\n self.wait_for_schedule = False # reset so we don't run on every clock tick\n self.save()\n return\n else:\n base_time = timezone.now()\n self.next_run = croniter(self.schedule, base_time).get_next(datetime)\n self.save()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef submit(self, timestamp):\n Channel(RUN_TASK_CHANNEL).send({'id':self.pk, 'ts': timestamp.timestamp()})", "response": "Internal instance method to submit this task for running immediately."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef run(self, message):\n the_callable = self.func_from_info()\n try:\n task_message = dict(\n task=self,\n channel_message=message,\n )\n the_callable(task_message)\n finally:\n if self.end_running < self.next_run:\n self.enabled=False\n Channel(KILL_TASK_CHANNEL).send({'id': self.pk})\n return\n if self.iterations == 0:\n return\n else:\n self.iterations -= 1\n if self.iterations == 0:\n self.enabled = False\n Channel(KILL_TASK_CHANNEL).send({'id':self.pk})\n self.save()", "response": "Internal instance method run by worker process to actually run the task callable."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninstance method to run this task immediately.", "response": "def run_asap(self):\n \"\"\"Instance method to run this task immediately.\"\"\"\n now = timezone.now()\n self.last_run = now\n self.calc_next_run()\n self.save()\n self.submit(now)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nclass method to run a callable with a specified number of iterations", "response": "def run_iterations(cls, the_callable, iterations=1, label=None, schedule='* * * * * *', userdata = None, run_immediately=False, delay_until=None):\n \"\"\"Class method to run a callable with a specified number of iterations\"\"\"\n task = task_with_callable(the_callable, label=label, schedule=schedule, userdata=userdata)\n task.iterations = iterations\n if delay_until is not None:\n if isinstance(delay_until, datetime):\n if delay_until > timezone.now():\n task.start_running = delay_until\n else:\n raise ValueError(\"Task cannot start running in the past\")\n else:\n raise ValueError(\"delay_until must be a datetime.datetime instance\")\n if run_immediately:\n task.next_run = timezone.now()\n else:\n task.calc_next_run()\n task.save()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclass method to run a one-shot task, immediately.", "response": "def run_once(cls, the_callable, userdata=None, delay_until=None):\n \"\"\"Class method to run a one-shot task, immediately.\"\"\"\n cls.run_iterations(the_callable, userdata=userdata, run_immediately=True, delay_until=delay_until)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_url_file(self):\n config = self.config\n # Find the actual controller key file\n if not self.url_file:\n self.url_file = os.path.join(\n self.profile_dir.security_dir,\n self.url_file_name\n )", "response": "Find the actual controller key file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_connector_file(self):\n \n self.log.info(\"Loading url_file %r\", self.url_file)\n config = self.config\n \n with open(self.url_file) as f:\n d = json.loads(f.read())\n \n if 'exec_key' in d:\n config.Session.key = cast_bytes(d['exec_key'])\n \n try:\n config.EngineFactory.location\n except AttributeError:\n config.EngineFactory.location = d['location']\n \n d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)\n try:\n config.EngineFactory.url\n except AttributeError:\n config.EngineFactory.url = d['url']\n \n try:\n config.EngineFactory.sshserver\n except AttributeError:\n config.EngineFactory.sshserver = d['ssh']", "response": "load config from a JSON connector file at a lower priority than command - line or config files."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npromotes engine to listening kernel accessible to frontends.", "response": "def bind_kernel(self, **kwargs):\n \"\"\"Promote engine to listening kernel, accessible to frontends.\"\"\"\n if self.kernel_app is not None:\n return\n \n self.log.info(\"Opening ports for direct connections as an IPython kernel\")\n \n kernel = self.kernel\n \n kwargs.setdefault('config', self.config)\n kwargs.setdefault('log', self.log)\n kwargs.setdefault('profile_dir', self.profile_dir)\n kwargs.setdefault('session', self.engine.session)\n \n app = self.kernel_app = IPKernelApp(**kwargs)\n \n # allow IPKernelApp.instance():\n IPKernelApp._instance = app\n \n app.init_connection_file()\n # relevant contents of init_sockets:\n \n app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port)\n app.log.debug(\"shell ROUTER Channel on port: %i\", app.shell_port)\n \n app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port)\n app.log.debug(\"iopub PUB Channel on port: %i\", app.iopub_port)\n \n kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER)\n app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port)\n app.log.debug(\"stdin ROUTER Channel on port: %i\", app.stdin_port)\n \n # start the heartbeat, and log connection info:\n \n app.init_heartbeat()\n \n app.log_connection_info()\n app.write_connection_file()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pid_exists(pid):\n if not isinstance(pid, int):\n raise TypeError('an integer is required')\n if pid < 0:\n return False\n try:\n os.kill(pid, 0)\n except OSError:\n e = sys.exc_info()[1]\n return e.errno == errno.EPERM\n else:\n return True", "response": "Check whether a process with the given pid exists in the current process table."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_disk_usage(path):\n st = os.statvfs(path)\n free = (st.f_bavail * st.f_frsize)\n total = (st.f_blocks * st.f_frsize)\n used = (st.f_blocks - st.f_bfree) * st.f_frsize\n percent = usage_percent(used, total, _round=1)\n # NB: the percentage is -5% than what shown by df due to\n # reserved blocks that we are currently not considering:\n # http://goo.gl/sWGbH\n return nt_diskinfo(total, used, free, percent)", "response": "Return disk usage associated with path."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nexecute a test described by a YAML file.", "response": "def timid(ctxt, test, key=None, check=False, exts=None):\n \"\"\"\n Execute a test described by a YAML file.\n\n :param ctxt: A ``timid.context.Context`` object.\n :param test: The name of a YAML file containing the test\n description. Note that the current working directory\n set up in ``ctxt.environment`` does not affect the\n resolution of this file.\n :param key: An optional key into the test description file. If\n not ``None``, the file named by ``test`` must be a\n YAML dictionary of lists of steps; otherwise, it must\n be a simple list of steps.\n :param check: If ``True``, only performs a syntax check of the\n test steps indicated by ``test`` and ``key``; the\n test itself is not run.\n :param exts: An instance of ``timid.extensions.ExtensionSet``\n describing the extensions to be called while\n processing the test steps.\n \"\"\"\n\n # Normalize the extension set\n if exts is None:\n exts = extensions.ExtensionSet()\n\n # Begin by reading the steps and adding them to the list in the\n # context (which may already have elements thanks to the\n # extensions)\n ctxt.emit('Reading test steps from %s%s...' %\n (test, '[%s]' % key if key else ''), debug=True)\n ctxt.steps += exts.read_steps(ctxt, steps.Step.parse_file(ctxt, test, key))\n\n # If all we were supposed to do was check, well, we've\n # accomplished that...\n if check:\n return None\n\n # Now we execute each step in turn\n for idx, step in enumerate(ctxt.steps):\n # Emit information about what we're doing\n ctxt.emit('[Step %d]: %s . . .' % (idx, step.name))\n\n # Run through extension hooks\n if exts.pre_step(ctxt, step, idx):\n ctxt.emit('[Step %d]: `- Step %s' %\n (idx, steps.states[steps.SKIPPED]))\n continue\n\n # Now execute the step\n result = step(ctxt)\n\n # Let the extensions process the result of the step\n exts.post_step(ctxt, step, idx, result)\n\n # Emit the result\n ctxt.emit('[Step %d]: `- Step %s%s' %\n (idx, steps.states[result.state],\n ' (ignored)' if result.ignore else ''))\n\n # Was the step a success?\n if not result:\n msg = 'Test step failure'\n if result.msg:\n msg += ': %s' % result.msg\n\n return msg\n\n # All done! And a success, to boot...\n return None"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a language instance.", "response": "def create_lang_instance(var_map = None):\n\t\"\"\"\n\t>>> lang_instance = create_lang_instance()\n\t>>> lang_instance.aml_evaluate(lang_instance.aml_compile('1 = 1'))\n\tTrue\n\t>>> li = create_lang_instance()\n\t>>> c = li.aml_compile\n\t>>> e = li.aml_evaluate\n\t>>> p = li.aml_translate_python\n\t>>> s = li.aml_translate_sql\n\t>>> u = li.aml_suggest\n\t>>> e(c('1 = 0'))\n\tFalse\n\t>>> e(c('\"1\" = \"1\"'))\n\tTrue\n\t>>> e(c('(1=1)'))\n\tTrue\n\t>>> e(c('1 > 1'))\n\tFalse\n\t>>> e(c('not 1 > 1'))\n\tTrue\n\t>>> e(c('1 != 1'))\n\tFalse\n\t>>> e(c('-2 = -2'))\n\tTrue\n\t>>> eval(p(c('-2 = -2')))\n\tTrue\n\t>>> eval(p(c('-2 >= -1')))\n\tFalse\n\t>>> eval(p(c('-2 <= -1')))\n\tTrue\n\t>>> eval(p(c('2 >= 1')))\n\tTrue\n\t>>> eval(p(c('2 <= 1')))\n\tFalse\n\t>>> eval(p(c('null = null')))\n\tTrue\n\t>>> eval(p(c('1 = null')))\n\tFalse\n\t>>> e(c('\"foo\" = \"foo\"'))\n\tTrue\n\t>>> e(c('\"foo\"' '=' \"'foo'\"))\n\tTrue\n\t>>> e(c('\"foo\" = \\\\'foo\\\\''))\n\tTrue\n\t>>> e(c('\"fo\\\\'o\" = \"fo\\\\'o\"'))\n\tTrue\n\t>>> e(c(\"'foo'\" + '=' + '\"foo\"'))\n\tTrue\n\t>>> li = create_lang_instance({'foo' : 1});\n\t>>> c = li.aml_compile\n\t>>> e = li.aml_evaluate\n\t>>> e(c('foo = 1'))\n\tTrue\n\t>>> li = create_lang_instance({'foo' : 1.00})\n\t>>> c = li.aml_compile\n\t>>> e = li.aml_evaluate\n\t>>> e(c('foo = 1'))\n\tTrue\n\t>>> li = create_lang_instance({'foo' : 2.24})\n\t>>> c = li.aml_compile\n\t>>> e = li.aml_evaluate\n\t>>> e(c('foo = 2.24'))\n\tTrue\n\t>>> e(c('foo = 2.2399 or foo = 2.24'))\n\tTrue\n\t>>> e(c('foo = 2.2399 or foo = 2.2401'))\n\tFalse\n\t>>> e(c('foo in (2.2399, 2.24, null,)'))\n\tTrue\n\t>>> e(c('foo in (2.2399, 2.2401, null,)'))\n\tFalse\n\t>>> e(c('null in (2.2399, 2.2401, null)'))\n\tTrue\n\t>>> e(c('\"null\" in (2.2399, 2.2401, null)'))\n\tFalse\n\t>>> e(c('\"null\"' 'in' \"(2.2399, 'null', null)\"))\n\tTrue\n\t>>> li = create_lang_instance({'foo' : 'foo'})\n\t>>> c = li.aml_compile\n\t>>> e = li.aml_evaluate\n\t>>> e(c('foo = \"foo\"'))\n\tTrue\n\t>>> li = create_lang_instance()\n\t>>> c = li.aml_compile\n\t>>> p = li.aml_translate_python\n\t>>> s = li.aml_translate_sql\n\t>>> s(c('null = null'))\n\tu'null is null'\n\t>>> p(c('null = null'))\n\tu'None == None'\n\t>>> s(c('null != null'))\n\tu'null is not null'\n\t>>> p(c('null != null'))\n\tu'None != None'\n\t>>> s(c('5 != 3'))\n\tu'5 <> 3'\n\t>>> p(c('5 != 3'))\n\tu'5 != 3'\n\t>>> p(c('5 in (3, 4, 5)'))\n\tu'5 in (3, 4, 5,)'\n\t>>> p(s('5 in (3, 4, 5)'))\n\tu'5 in (3, 4, 5)'\n\t>>> li = create_lang_instance({'foo' : 'bar', 'fo2' : 'ba2'})\n\t>>> c = li.aml_compile\n\t>>> p = li.aml_translate_python\n\t>>> e = li.aml_evaluate\n\t>>> u = li.aml_suggest\n\t>>> u('1 = fo')\n\t[u'fo2', u'foo']\n\t>>> u('1 = FO')\n\t[u'fo2', u'foo']\n\t>>> p(c('null = null'))\n\tu'None == None'\n\t>>> e(c('foo = \"bar\"'))\n\tTrue\n\t>>> e(c('fo2 = \"ba2\"'))\n\tTrue\n\t\"\"\"\n\n\tdef py_bool_to_lit(py_bool):\n\t\treturn parse( 'true' if py_bool else 'false', BooleanLiteral)\n\n\tif not var_map:\n\t\tclass Identifier(str):\n\t\t\tgrammar = re.compile(r'$a') # This will match nothing.\n\telse:\n\t\tclass Identifier(Keyword):\n\t\t\tgrammar = Enum(*[K(v) for v in var_map.iterkeys()])\n\t\t\n\tclass StringLiteral(str):\n\n\t\tdef __new__(cls, s):\n\t\t\treturn super(StringLiteral, cls).__new__(cls, ast.literal_eval(s))\n\n\t\tgrammar = [re.compile(r'\"[^\\\\\\n\\r]+?\"'), re.compile(r\"'[^\\\\\\n\\r]+?'\")]\n\n\tclass IntegerLiteral(int):\n\t\tgrammar = re.compile(r'-?\\d+')\n\n\tclass FloatLiteral(float):\n\t\tgrammar = re.compile(r'-?\\d+.\\d+')\n\n\tclass BooleanLiteral(Keyword):\n\t\tgrammar = Enum(K('true'), K('false'))\n\n\tclass NullLiteral(Keyword):\n\t\tgrammar = Enum(K('null'))\n\n\tComparable = [NullLiteral, FloatLiteral, IntegerLiteral, \n\t\t\t\t\t\t\t\t\tStringLiteral, Identifier]\n\n\tclass ListOfComparables(List):\n\t\tpass\n\n\tListOfComparables.grammar = (\n\t\t\t'(',\n\t\t\tComparable,\n\t\t\tmaybe_some(\n\t\t\t\t',',\n\t\t\t\tblank,\n\t\t\t\tComparable,\n\t\t\t),\n\t\t\toptional(','),\n\t\t\t')'\n\t)\n\n\tclass ComparisonOperator(str):\n\t\tgrammar = re.compile(r'=|>=|<=|>|<|!=|in')\n\n\tclass BooleanFunctionName(Keyword):\n\t\tgrammar = Enum(K('and'), K('or'))\n\n\tclass ComparisonOperation(List):\n\t\tpass\n\n\tComparisonOperation.grammar = (\n\t\t\tComparable,\n\t\t\tblank,\n\t\t\tattr('comp_op', ComparisonOperator),\n\t\t\tblank,\n\t\t\t[Comparable, ListOfComparables],\n\t)\n\n\tclass BooleanOperationSimple(List):\n# The flag() pypeg2 function works great when parsing but does not work when\n# composing (the flag gets output whether it was in the source text or not). So\n# a workaround is this:\n\t\tgrammar = (\n\t\t\t\tattr('negated', optional(RE_NOT)),\n\t\t\t\tComparisonOperation,\n\t\t)\n\n\tclass BooleanOperation(List):\n\t\tpass\n\n\tBooleanOperation.grammar = (\n\t\t\tBooleanOperationSimple,\n\t\t\tmaybe_some(\n\t\t\t\tblank,\n\t\t\t\tBooleanFunctionName,\n\t\t\t\tblank,\n\t\t\t\tBooleanOperationSimple,\n\t\t\t),\n\t)\n\n\tclass Expression(List):\n\t\tpass\n\n\tExpression.grammar = (\n\t\t\t[BooleanOperationSimple, ('(', Expression, ')')],\n\t\t\tmaybe_some(\n\t\t\t\tblank,\n\t\t\t\tBooleanFunctionName,\n\t\t\t\tblank,\n\t\t\t\t[BooleanOperationSimple, ('(', Expression, ')')],\n\t\t\t),\n\t)\n\n\tdef eval_node(node):\n\t\ten = lambda n: eval_node(n)\n\t\tif isinstance(node, Identifier):\n\t\t\treturn var_map[node]\n\t\telif isinstance(node, StringLiteral):\n\t\t\treturn node\n\t\telif isinstance(node, IntegerLiteral):\n\t\t\treturn node\n\t\telif isinstance(node, FloatLiteral):\n\t\t\treturn node\n\t\telif isinstance(node, BooleanLiteral):\n\t\t\tif node == 'true':\n\t\t\t\treturn True\n\t\t\telif node == 'false':\n\t\t\t\treturn False\n\t\telif isinstance(node, NullLiteral):\n\t\t\treturn None\n\t\telif isinstance(node, ListOfComparables):\n\t\t\treturn node\n\t\telif isinstance(node, ComparisonOperation):\n\t\t\topa, opb = node[0:2]\n\t\t\tif node.comp_op == '=':\n\t\t\t\treturn en(opa) == en(opb)\n\t\t\telif node.comp_op == '>':\n\t\t\t\treturn en(opa) > en(opb)\n\t\t\telif node.comp_op == '<':\n\t\t\t\treturn en(opa) < en(opb)\n\t\t\telif node.comp_op == '!=':\n\t\t\t\treturn en(opa) != en(opb)\n\t\t\telif node.comp_op == '>=':\n\t\t\t\treturn en(opa) >= en(opb)\n\t\t\telif node.comp_op == '<=':\n\t\t\t\treturn en(opa) <= en(opb)\n\t\t\telif node.comp_op == 'in':\n\t\t\t\tenopa = en(opa)\n\t\t\t\tenopb = en(opb)\n\t\t\t\tfor other_node in list(enopb):\n\t\t\t\t\tvirtual_node = ComparisonOperation([opa, other_node])\n\t\t\t\t\tvirtual_node.comp_op = '='\n\t\t\t\t\tif en(virtual_node):\n\t\t\t\t\t\treturn True\n\t\t\t\treturn False\n\t\telif isinstance(node, BooleanOperationSimple):\n\t\t\ta = en(node[0])\n\t\t\tif node.negated:\n\t\t\t\ta = not a\n\t\t\treturn a\n\t\telif isinstance(node, BooleanOperation):\n\t\t\tif len(node) == 1:\n\t\t\t\treturn en(node[0])\n\n\t\t\tfn_map = {\n\t\t\t\t\t'and': lambda a,b: a and b,\n\t\t\t\t\t'or': lambda a,b: a or b,\n\t\t\t}\n\n\t\t\tdef simple_eval(tr):\n\t\t\t\treturn py_bool_to_lit(fn_map[tr[1]]( en(tr[0]), en(tr[2]))) \n\t\t\t\t\t\t\n\n\t\t\tfor fname in ['and', 'or']:\n\t\t\t\tfor i in xrange(1, len(node), 2):\n\t\t\t\t\tif node[i] == fname:\n\t\t\t\t\t\tnew_self = (\n\t\t\t\t\t\t\tnode[:i-1]\n\t\t\t\t\t\t\t+ [simple_eval(node[i-1:i+2])]\n\t\t\t\t\t\t\t+ node[i+2:]\n\t\t\t\t\t\t)\n\t\t\t\t\t\treturn en(BooleanOperation(new_self))\n\t\telif isinstance(node, Expression):\n\t\t\tdef iter_over_relevant():\n\t\t\t\tfor eli, el in enumerate(node):\n\t\t\t\t\tif eli % 2 == 0:\n\t\t\t\t\t\tyield eli, el\n\n\t\t\tif all(\n\t\t\t\t\tisinstance(el, BooleanOperationSimple) \n\t\t\t\t\t\tor\n\t\t\t\t\tisinstance(el, BooleanLiteral) \n\t\t\t\t\tfor eli, el in iter_over_relevant()\n\t\t\t):\n\t\t\t\tres = en(BooleanOperation(node))\n\t\t\t\treturn res\n\t\t\telse:\n\t\t\t\tfor eli, el in iter_over_relevant():\n\t\t\t\t\tif isinstance(el, Expression):\n\t\t\t\t\t\tnew_self = (\n\t\t\t\t\t\t\t\tnode[:eli]\n\t\t\t\t\t\t\t\t+ [py_bool_to_lit(en(el))]\n\t\t\t\t\t\t\t\t+ node[eli+1:]\n\t\t\t\t\t\t)\n\t\t\t\t\t\treturn en(Expression(new_self))\n\n\tdef compose_node_to_python(node):\n\t\treturn compose(node)\n\n\tdef compose_node_to_sql(node):\n\t\treturn compose(node)\n\n\tdef aml_compile(source):\n\t\treturn parse(source, Expression)\n\n\tdef aml_evaluate(aml_c):\n\t\tresult = eval_node(aml_c)\n\t\treturn result\n\n\tdef aml_translate_python(aml_c):\n\n\t\tdef comp_op_compose(self, *args, **kwargs):\n\t\t\tif self == '=':\n\t\t\t\treturn '=='\n\t\t\telse:\n\t\t\t\treturn self\n\n\t\tdef null_compose(self, *args, **kwargs):\n\t\t\treturn 'None'\n\n\t\tdef string_compose(self, *args, **kwargs):\n\t\t\treturn '\"' + self.replace('\"', r'\\\"') + '\"'\n\n\t\tComparisonOperator.compose = comp_op_compose\n\t\tNullLiteral.compose = null_compose\n\t\tStringLiteral.compose = string_compose\n\n\t\tresult = compose_node_to_python(aml_c)\n\n\t\tdelattr(ComparisonOperator, 'compose')\n\t\tdelattr(NullLiteral, 'compose')\n\t\tdelattr(StringLiteral, 'compose')\n\n\t\treturn result\n\n\tdef aml_translate_sql(aml_c):\n\n\t\tdef comp_op_compose(self, *args, **kwargs):\n\t\t\tif self == '!=':\n\t\t\t\treturn '<>'\n\t\t\telse:\n\t\t\t\treturn self\n\n\t\tdef null_compose(self, *args, **kwargs):\n\t\t\treturn 'null'\n\n\t\tdef string_compose(self, *args, **kwargs):\n\t\t\treturn \"'\" + self.replace(\"'\", \"''\") + \"'\"\n\n\t\tdef comp_operation_compose(self, *args, **kwargs):\n\t\t\tif (\n\t\t\t\t\t(\n\t\t\t\t\t\tisinstance(self[0], NullLiteral) \n\t\t\t\t\t\t\tor\n\t\t\t\t\t\tisinstance(self[1], NullLiteral)\n\t\t\t\t\t)\n\t\t\t\t\t\tand\n\t\t\t\t\t(\n\t\t\t\t\t\tself.comp_op in ('=', '!=')\n\t\t\t\t\t)\n\t\t\t):\n\n\t\t\t\tif self.comp_op == '=':\n\t\t\t\t\tmiddle = 'is'\n\t\t\t\telse:\n\t\t\t\t\tmiddle = 'is not'\n\n\t\t\telse:\n\t\t\t\tmiddle = compose(self.comp_op)\n\n\t\t\treturn ' '.join([\n\t\t\t\tcompose(self[0]),\n\t\t\t\tmiddle,\n\t\t\t\tcompose(self[1]),\n\t\t\t])\n\n\n\t\tComparisonOperator.compose = comp_op_compose\n\t\tNullLiteral.compose = null_compose\n\t\tComparisonOperation.compose = comp_operation_compose\n\t\tStringLiteral.compose = string_compose\n\n\t\tresult = compose_node_to_sql(aml_c)\n\n\t\tdelattr(ComparisonOperator, 'compose')\n\t\tdelattr(NullLiteral, 'compose')\n\t\tdelattr(ComparisonOperation, 'compose')\n\t\tdelattr(StringLiteral, 'compose')\n\n\t\treturn result\n\n\tdef aml_suggest(source):\n\t\tsuggestions = [ ]\n\t\tif var_map:\n\t\t\tif not source:\n\t\t\t\tsuggestions = list(var_map.iterkeys())\n\t\t\telse:\n\t\t\t\tsplit = [el for el in re.split(r'(?m)\\s+', source) if el]\n\t\t\t\tif split:\n\t\t\t\t\tfor candidate in var_map.iterkeys():\n\t\t\t\t\t\tif candidate.lower().startswith(split[-1].lower()):\n\t\t\t\t\t\t\tsuggestions.append(candidate)\n\t\tsuggestions.sort()\n\t\treturn suggestions\n\n\tlang_instance = LangInstance()\n\n\tlang_instance.aml_compile = aml_compile\n\tlang_instance.aml_evaluate = aml_evaluate\n\tlang_instance.aml_translate_python = aml_translate_python\n\tlang_instance.aml_translate_sql = aml_translate_sql\n\tlang_instance.aml_suggest = aml_suggest\n\n\treturn lang_instance"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_interrupt_event():\n # Create a security attributes struct that permits inheritance of the\n # handle by new processes.\n # FIXME: We can clean up this mess by requiring pywin32 for IPython.\n class SECURITY_ATTRIBUTES(ctypes.Structure):\n _fields_ = [ (\"nLength\", ctypes.c_int),\n (\"lpSecurityDescriptor\", ctypes.c_void_p),\n (\"bInheritHandle\", ctypes.c_int) ]\n sa = SECURITY_ATTRIBUTES()\n sa_p = ctypes.pointer(sa)\n sa.nLength = ctypes.sizeof(SECURITY_ATTRIBUTES)\n sa.lpSecurityDescriptor = 0\n sa.bInheritHandle = 1\n\n return ctypes.windll.kernel32.CreateEventA(\n sa_p, # lpEventAttributes\n False, # bManualReset\n False, # bInitialState\n '')", "response": "Create an interrupt event handle."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun the poll loop. This method never returns.", "response": "def run(self):\n \"\"\" Run the poll loop. This method never returns.\n \"\"\"\n try:\n from _winapi import WAIT_OBJECT_0, INFINITE\n except ImportError:\n from _subprocess import WAIT_OBJECT_0, INFINITE\n\n # Build the list of handle to listen on.\n handles = []\n if self.interrupt_handle:\n handles.append(self.interrupt_handle)\n if self.parent_handle:\n handles.append(self.parent_handle)\n arch = platform.architecture()[0]\n c_int = ctypes.c_int64 if arch.startswith('64') else ctypes.c_int\n\n # Listen forever.\n while True:\n result = ctypes.windll.kernel32.WaitForMultipleObjects(\n len(handles), # nCount\n (c_int * len(handles))(*handles), # lpHandles\n False, # bWaitAll\n INFINITE) # dwMilliseconds\n\n if WAIT_OBJECT_0 <= result < len(handles):\n handle = handles[result - WAIT_OBJECT_0]\n\n if handle == self.interrupt_handle:\n interrupt_main()\n\n elif handle == self.parent_handle:\n os._exit(1)\n elif result < 0:\n # wait failed, just give up and stop polling.\n warn(\"\"\"Parent poll failed. If the frontend dies,\n the kernel may be left running. Please let us know\n about your system (bitness, Python, etc.) at\n ipython-dev@scipy.org\"\"\")\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfunctions to initialize settings from command line and custom settings file", "response": "def initialize():\n \"\"\"\n Function to initialize settings from command line and/or custom settings file\n :return: Returns str with operation type\n \"\"\"\n if len(sys.argv) == 1:\n usage()\n sys.exit()\n\n command = _get_command(sys.argv[1])\n\n try:\n opts, args = getopt.getopt(sys.argv[2:], 'h:e:p:u:l:P:s:m:',\n ['help', 'email=', 'password=', 'url=', 'locale=',\n 'po-path=', 'settings=', 'message='])\n except getopt.GetoptError:\n usage()\n sys.exit()\n\n params = _get_params_from_options(opts)\n _set_settings_file(settings, params)\n\n if command == 'push':\n if 'GIT_MESSAGE' in params:\n return 'push', params['GIT_MESSAGE']\n return 'push', None\n\n return command, None"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns dictionaries mapping lower case typename to type WorkItem objects from the types package and vice versa.", "response": "def create_typestr2type_dicts(dont_include_in_type2typestr=[\"lambda\"]):\n \"\"\"Return dictionaries mapping lower case typename (e.g. 'tuple') to type\n objects from the types package, and vice versa.\"\"\"\n typenamelist = [tname for tname in dir(types) if tname.endswith(\"Type\")]\n typestr2type, type2typestr = {}, {}\n\n for tname in typenamelist:\n name = tname[:-4].lower() # Cut 'Type' off the end of the name\n obj = getattr(types, tname)\n typestr2type[name] = obj\n if name not in dont_include_in_type2typestr:\n type2typestr[obj] = name\n return typestr2type, type2typestr"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dict_dir(obj):\n ns = {}\n for key in dir2(obj):\n # This seemingly unnecessary try/except is actually needed\n # because there is code out there with metaclasses that\n # create 'write only' attributes, where a getattr() call\n # will fail even if the attribute appears listed in the\n # object's dictionary. Properties can actually do the same\n # thing. In particular, Traits use this pattern\n try:\n ns[key] = getattr(obj, key)\n except AttributeError:\n pass\n return ns", "response": "Produce a dictionary of an object s attributes. Builds on dir2 checking that a getattr call actually succeeds."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef filter_ns(ns, name_pattern=\"*\", type_pattern=\"all\", ignore_case=True,\n show_all=True):\n \"\"\"Filter a namespace dictionary by name pattern and item type.\"\"\"\n pattern = name_pattern.replace(\"*\",\".*\").replace(\"?\",\".\")\n if ignore_case:\n reg = re.compile(pattern+\"$\", re.I)\n else:\n reg = re.compile(pattern+\"$\")\n\n # Check each one matches regex; shouldn't be hidden; of correct type.\n return dict((key,obj) for key, obj in ns.iteritems() if reg.match(key) \\\n and show_hidden(key, show_all) \\\n and is_type(obj, type_pattern) )", "response": "Filter a namespace dictionary by name pattern and item type."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef list_namespace(namespace, type_pattern, filter, ignore_case=False, show_all=False):\n pattern_list=filter.split(\".\")\n if len(pattern_list) == 1:\n return filter_ns(namespace, name_pattern=pattern_list[0],\n type_pattern=type_pattern,\n ignore_case=ignore_case, show_all=show_all)\n else:\n # This is where we can change if all objects should be searched or\n # only modules. Just change the type_pattern to module to search only\n # modules\n filtered = filter_ns(namespace, name_pattern=pattern_list[0],\n type_pattern=\"all\",\n ignore_case=ignore_case, show_all=show_all)\n results = {}\n for name, obj in filtered.iteritems():\n ns = list_namespace(dict_dir(obj), type_pattern,\n \".\".join(pattern_list[1:]),\n ignore_case=ignore_case, show_all=show_all)\n for inner_name, inner_obj in ns.iteritems():\n results[\"%s.%s\"%(name,inner_name)] = inner_obj\n return results", "response": "Return dictionary of all objects in a namespace dictionary that match\nEventType type_pattern and filter."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking for presence of mutually exclusive keys in a dict.", "response": "def mutex_opts(dict,ex_op):\n \"\"\"Check for presence of mutually exclusive keys in a dict.\n\n Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]\"\"\"\n for op1,op2 in ex_op:\n if op1 in dict and op2 in dict:\n raise ValueError,'\\n*** ERROR in Arguments *** '\\\n 'Options '+op1+' and '+op2+' are mutually exclusive.'"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef popkey(dct,key,default=NotGiven):\n\n try:\n val = dct[key]\n except KeyError:\n if default is NotGiven:\n raise\n else:\n return default\n else:\n del dct[key]\n return val", "response": "Pop a key from a dict and return it."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef show(close=None):\n if close is None:\n close = InlineBackend.instance().close_figures\n try:\n for figure_manager in Gcf.get_all_fig_managers():\n send_figure(figure_manager.canvas.figure)\n finally:\n show._to_draw = []\n if close:\n matplotlib.pyplot.close('all')", "response": "Show all figures as SVG payloads sent to IPython clients."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalls after every pylab drawing command.", "response": "def draw_if_interactive():\n \"\"\"\n Is called after every pylab drawing command\n \"\"\"\n # signal that the current active figure should be sent at the end of\n # execution. Also sets the _draw_called flag, signaling that there will be\n # something to send. At the end of the code execution, a separate call to\n # flush_figures() will act upon these values\n\n fig = Gcf.get_active().canvas.figure\n\n # Hack: matplotlib FigureManager objects in interacive backends (at least\n # in some of them) monkeypatch the figure object and add a .show() method\n # to it. This applies the same monkeypatch in order to support user code\n # that might expect `.show()` to be part of the official API of figure\n # objects.\n # For further reference:\n # https://github.com/ipython/ipython/issues/1612\n # https://github.com/matplotlib/matplotlib/issues/835\n \n if not hasattr(fig, 'show'):\n # Queue up `fig` for display\n fig.show = lambda *a: send_figure(fig)\n\n # If matplotlib was manually set to non-interactive mode, this function\n # should be a no-op (otherwise we'll generate duplicate plots, since a user\n # who set ioff() manually expects to make separate draw/show calls).\n if not matplotlib.is_interactive():\n return\n\n # ensure current figure will be drawn, and each subsequent call\n # of draw_if_interactive() moves the active figure to ensure it is\n # drawn last\n try:\n show._to_draw.remove(fig)\n except ValueError:\n # ensure it only appears in the draw list once\n pass\n # Queue up the figure for drawing in next show() call\n show._to_draw.append(fig)\n show._draw_called = True"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsends all figures that changed to the user.", "response": "def flush_figures():\n \"\"\"Send all figures that changed\n\n This is meant to be called automatically and will call show() if, during\n prior code execution, there had been any calls to draw_if_interactive.\n \n This function is meant to be used as a post_execute callback in IPython,\n so user-caused errors are handled with showtraceback() instead of being\n allowed to raise. If this function is not called from within IPython,\n then these exceptions will raise.\n \"\"\"\n if not show._draw_called:\n return\n \n if InlineBackend.instance().close_figures:\n # ignore the tracking, just draw and close all figures\n try:\n return show(True)\n except Exception as e:\n # safely show traceback if in IPython, else raise\n try:\n get_ipython\n except NameError:\n raise e\n else:\n get_ipython().showtraceback()\n return\n try:\n # exclude any figures that were closed:\n active = set([fm.canvas.figure for fm in Gcf.get_all_fig_managers()])\n for fig in [ fig for fig in show._to_draw if fig in active ]:\n try:\n send_figure(fig)\n except Exception as e:\n # safely show traceback if in IPython, else raise\n try:\n get_ipython\n except NameError:\n raise e\n else:\n get_ipython().showtraceback()\n break\n finally:\n # clear flags for next round\n show._to_draw = []\n show._draw_called = False"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef send_figure(fig):\n fmt = InlineBackend.instance().figure_format\n data = print_figure(fig, fmt)\n # print_figure will return None if there's nothing to draw:\n if data is None:\n return\n mimetypes = { 'png' : 'image/png', 'svg' : 'image/svg+xml' }\n mime = mimetypes[fmt]\n # flush text streams before sending figures, helps a little with output\n # synchronization in the console (though it's a bandaid, not a real sln)\n sys.stdout.flush(); sys.stderr.flush()\n publish_display_data(\n 'IPython.zmq.pylab.backend_inline.send_figure',\n {mime : data}\n )", "response": "Draw the given figure and send it as a PNG payload."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nload an IPython extension by its module name.", "response": "def load_extension(self, module_str):\n \"\"\"Load an IPython extension by its module name.\n\n If :func:`load_ipython_extension` returns anything, this function\n will return that object.\n \"\"\"\n from IPython.utils.syspathcontext import prepended_to_syspath\n\n if module_str not in sys.modules:\n with prepended_to_syspath(self.ipython_extension_dir):\n __import__(module_str)\n mod = sys.modules[module_str]\n return self._call_load_ipython_extension(mod)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nunload an IPython extension by its module name.", "response": "def unload_extension(self, module_str):\n \"\"\"Unload an IPython extension by its module name.\n\n This function looks up the extension's name in ``sys.modules`` and\n simply calls ``mod.unload_ipython_extension(self)``.\n \"\"\"\n if module_str in sys.modules:\n mod = sys.modules[module_str]\n self._call_unload_ipython_extension(mod)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndownloading and install an IPython extension. Returns the full path to the installed file.", "response": "def install_extension(self, url, filename=None):\n \"\"\"Download and install an IPython extension. \n \n If filename is given, the file will be so named (inside the extension\n directory). Otherwise, the name from the URL will be used. The file must\n have a .py or .zip extension; otherwise, a ValueError will be raised.\n \n Returns the full path to the installed file.\n \"\"\"\n # Ensure the extension directory exists\n if not os.path.isdir(self.ipython_extension_dir):\n os.makedirs(self.ipython_extension_dir, mode = 0777)\n \n if os.path.isfile(url):\n src_filename = os.path.basename(url)\n copy = copyfile\n else:\n src_filename = urlparse(url).path.split('/')[-1]\n copy = urlretrieve\n \n if filename is None:\n filename = src_filename\n if os.path.splitext(filename)[1] not in ('.py', '.zip'):\n raise ValueError(\"The file must have a .py or .zip extension\", filename)\n \n filename = os.path.join(self.ipython_extension_dir, filename)\n copy(url, filename)\n return filename"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef externals_finder(dirname, filename):\n found = False\n f = open(filename,'rt')\n for line in iter(f.readline, ''): # can't use direct iter!\n parts = line.split()\n if len(parts)==2:\n kind,length = parts\n data = f.read(int(length))\n if kind=='K' and data=='svn:externals':\n found = True\n elif kind=='V' and found:\n f.close()\n break\n else:\n f.close()\n return\n\n for line in data.splitlines():\n parts = line.split()\n if parts:\n yield joinpath(dirname, parts[0])", "response": "Find any svn externals directories in a file."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef random_ports(port, n):\n for i in range(min(5, n)):\n yield port + i\n for i in range(n-5):\n yield port + random.randint(-2*n, 2*n)", "response": "Generate a list of n random ports near the given port."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninitialize tornado webapp and httpserver", "response": "def init_webapp(self):\n \"\"\"initialize tornado webapp and httpserver\"\"\"\n self.web_app = NotebookWebApplication(\n self, self.kernel_manager, self.notebook_manager, \n self.cluster_manager, self.log,\n self.base_project_url, self.webapp_settings\n )\n if self.certfile:\n ssl_options = dict(certfile=self.certfile)\n if self.keyfile:\n ssl_options['keyfile'] = self.keyfile\n else:\n ssl_options = None\n self.web_app.password = self.password\n self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)\n if ssl_options is None and not self.ip and not (self.read_only and not self.password):\n self.log.critical('WARNING: the notebook server is listening on all IP addresses '\n 'but not using any encryption or authentication. This is highly '\n 'insecure and not recommended.')\n\n success = None\n for port in random_ports(self.port, self.port_retries+1):\n try:\n self.http_server.listen(port, self.ip)\n except socket.error, e:\n if e.errno != errno.EADDRINUSE:\n raise\n self.log.info('The port %i is already in use, trying another random port.' % port)\n else:\n self.port = port\n success = True\n break\n if not success:\n self.log.critical('ERROR: the notebook server could not be started because '\n 'no available port could be found.')\n self.exit(1)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nhandling SIGINT signal and request confirmation dialog", "response": "def _handle_sigint(self, sig, frame):\n \"\"\"SIGINT handler spawns confirmation dialog\"\"\"\n # register more forceful signal handler for ^C^C case\n signal.signal(signal.SIGINT, self._signal_stop)\n # request confirmation dialog in bg thread, to avoid\n # blocking the App\n thread = threading.Thread(target=self._confirm_exit)\n thread.daemon = True\n thread.start()"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nshutting down all kernels in the kernel manager.", "response": "def cleanup_kernels(self):\n \"\"\"shutdown all kernels\n \n The kernels will shutdown themselves when this process no longer exists,\n but explicit shutdown allows the KernelManagers to cleanup the connection files.\n \"\"\"\n self.log.info('Shutting down kernels')\n km = self.kernel_manager\n # copy list, since shutdown_kernel deletes keys\n for kid in list(km.kernel_ids):\n km.shutdown_kernel(kid)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprice European and Asian options using Monte Carlo method.", "response": "def price_options(S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):\n \"\"\"\n Price European and Asian options using a Monte Carlo method.\n\n Parameters\n ----------\n S : float\n The initial price of the stock.\n K : float\n The strike price of the option.\n sigma : float\n The volatility of the stock.\n r : float\n The risk free interest rate.\n days : int\n The number of days until the option expires.\n paths : int\n The number of Monte Carlo paths used to price the option.\n\n Returns\n -------\n A tuple of (E. call, E. put, A. call, A. put) option prices.\n \"\"\"\n import numpy as np\n from math import exp,sqrt\n \n h = 1.0/days\n const1 = exp((r-0.5*sigma**2)*h)\n const2 = sigma*sqrt(h)\n stock_price = S*np.ones(paths, dtype='float64')\n stock_price_sum = np.zeros(paths, dtype='float64')\n for j in range(days):\n growth_factor = const1*np.exp(const2*np.random.standard_normal(paths))\n stock_price = stock_price*growth_factor\n stock_price_sum = stock_price_sum + stock_price\n stock_price_avg = stock_price_sum/days\n zeros = np.zeros(paths, dtype='float64')\n r_factor = exp(-r*h*days)\n euro_put = r_factor*np.mean(np.maximum(zeros, K-stock_price))\n asian_put = r_factor*np.mean(np.maximum(zeros, K-stock_price_avg))\n euro_call = r_factor*np.mean(np.maximum(zeros, stock_price-K))\n asian_call = r_factor*np.mean(np.maximum(zeros, stock_price_avg-K))\n return (euro_call, euro_put, asian_call, asian_put)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef multiple_replace(dict, text):\n\n # Function by Xavier Defrang, originally found at:\n # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81330\n\n # Create a regular expression from the dictionary keys\n regex = re.compile(\"(%s)\" % \"|\".join(map(re.escape, dict.keys())))\n # For each match, look-up corresponding value in dictionary\n return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text)", "response": "Replace in text all occurences of any key in the given\n dictionary by its corresponding value. Returns the new string."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cwd_filt(depth):\n\n cwd = os.getcwdu().replace(HOME,\"~\")\n out = os.sep.join(cwd.split(os.sep)[-depth:])\n return out or os.sep", "response": "Return the last depth elements of the current working directory."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the last depth elements of the current working directory.", "response": "def cwd_filt2(depth):\n \"\"\"Return the last depth elements of the current working directory.\n\n $HOME is always replaced with '~'.\n If depth==0, the full path is returned.\"\"\"\n\n full_cwd = os.getcwdu()\n cwd = full_cwd.replace(HOME,\"~\").split(os.sep)\n if '~' in cwd and len(cwd) == depth+1:\n depth += 1\n drivepart = ''\n if sys.platform == 'win32' and len(cwd) > depth:\n drivepart = os.path.splitdrive(full_cwd)[0]\n out = drivepart + '/'.join(cwd[-depth:])\n\n return out or os.sep"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update_prompt(self, name, new_template=None):\n if new_template is not None:\n self.templates[name] = multiple_replace(prompt_abbreviations, new_template)\n # We count invisible characters (colour escapes) on the last line of the\n # prompt, to calculate the width for lining up subsequent prompts.\n invis_chars = _lenlastline(self._render(name, color=True)) - \\\n _lenlastline(self._render(name, color=False))\n self.invisible_chars[name] = invis_chars", "response": "This is called when a prompt template is updated. It processes the prompt abbreviations used in the prompt template and updates the prompt s invisible characters."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrendering but don t justify or update the width or txtwidth attributes.", "response": "def _render(self, name, color=True, **kwargs):\n \"\"\"Render but don't justify, or update the width or txtwidth attributes.\n \"\"\"\n if name == 'rewrite':\n return self._render_rewrite(color=color)\n \n if color:\n scheme = self.color_scheme_table.active_colors\n if name=='out':\n colors = color_lists['normal']\n colors.number, colors.prompt, colors.normal = \\\n scheme.out_number, scheme.out_prompt, scheme.normal\n else:\n colors = color_lists['inp']\n colors.number, colors.prompt, colors.normal = \\\n scheme.in_number, scheme.in_prompt, scheme.in_normal\n if name=='in2':\n colors.prompt = scheme.in_prompt2\n else:\n # No color\n colors = color_lists['nocolor']\n colors.number, colors.prompt, colors.normal = '', '', ''\n \n count = self.shell.execution_count # Shorthand\n # Build the dictionary to be passed to string formatting\n fmtargs = dict(color=colors, count=count,\n dots=\".\"*len(str(count)),\n width=self.width, txtwidth=self.txtwidth )\n fmtargs.update(self.lazy_evaluate_fields)\n fmtargs.update(kwargs)\n \n # Prepare the prompt\n prompt = colors.prompt + self.templates[name] + colors.normal\n \n # Fill in required fields\n return self._formatter.format(prompt, **fmtargs)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrendering the --- > rewrite prompt.", "response": "def _render_rewrite(self, color=True):\n \"\"\"Render the ---> rewrite prompt.\"\"\"\n if color:\n scheme = self.color_scheme_table.active_colors\n # We need a non-input version of these escapes\n color_prompt = scheme.in_prompt.replace(\"\\001\",\"\").replace(\"\\002\",\"\")\n color_normal = scheme.normal\n else:\n color_prompt, color_normal = '', ''\n\n return color_prompt + \"-> \".rjust(self.txtwidth, \"-\") + color_normal"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrender the selected prompt.", "response": "def render(self, name, color=True, just=None, **kwargs):\n \"\"\"\n Render the selected prompt.\n \n Parameters\n ----------\n name : str\n Which prompt to render. One of 'in', 'in2', 'out', 'rewrite'\n color : bool\n If True (default), include ANSI escape sequences for a coloured prompt.\n just : bool\n If True, justify the prompt to the width of the last prompt. The\n default is stored in self.justify.\n **kwargs :\n Additional arguments will be passed to the string formatting operation,\n so they can override the values that would otherwise fill in the\n template.\n \n Returns\n -------\n A string containing the rendered prompt.\n \"\"\"\n res = self._render(name, color=color, **kwargs)\n \n # Handle justification of prompt\n invis_chars = self.invisible_chars[name] if color else 0\n self.txtwidth = _lenlastline(res) - invis_chars\n just = self.justify if (just is None) else just\n # If the prompt spans more than one line, don't try to justify it:\n if just and name != 'in' and ('\\n' not in res) and ('\\r' not in res):\n res = res.rjust(self.width + invis_chars)\n self.width = _lenlastline(res) - invis_chars\n return res"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting a new connection file to the file specified by fname.", "response": "def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,\n ip=LOCALHOST, key=b''):\n \"\"\"Generates a JSON config file, including the selection of random ports.\n \n Parameters\n ----------\n\n fname : unicode\n The path to the file to write\n\n shell_port : int, optional\n The port to use for ROUTER channel.\n\n iopub_port : int, optional\n The port to use for the SUB channel.\n\n stdin_port : int, optional\n The port to use for the REQ (raw input) channel.\n\n hb_port : int, optional\n The port to use for the hearbeat REP channel.\n\n ip : str, optional\n The ip address the kernel will bind to.\n\n key : str, optional\n The Session key used for HMAC authentication.\n\n \"\"\"\n # default to temporary connector file\n if not fname:\n fname = tempfile.mktemp('.json')\n \n # Find open ports as necessary.\n ports = []\n ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \\\n int(stdin_port <= 0) + int(hb_port <= 0)\n for i in xrange(ports_needed):\n sock = socket.socket()\n sock.bind(('', 0))\n ports.append(sock)\n for i, sock in enumerate(ports):\n port = sock.getsockname()[1]\n sock.close()\n ports[i] = port\n if shell_port <= 0:\n shell_port = ports.pop(0)\n if iopub_port <= 0:\n iopub_port = ports.pop(0)\n if stdin_port <= 0:\n stdin_port = ports.pop(0)\n if hb_port <= 0:\n hb_port = ports.pop(0)\n \n cfg = dict( shell_port=shell_port,\n iopub_port=iopub_port,\n stdin_port=stdin_port,\n hb_port=hb_port,\n )\n cfg['ip'] = ip\n cfg['key'] = bytes_to_str(key)\n \n with open(fname, 'w') as f:\n f.write(json.dumps(cfg, indent=2))\n \n return fname, cfg"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlaunches a kernel process on the specified ports.", "response": "def base_launch_kernel(code, fname, stdin=None, stdout=None, stderr=None,\n executable=None, independent=False, extra_arguments=[],\n cwd=None):\n \"\"\" Launches a localhost kernel, binding to the specified ports.\n\n Parameters\n ----------\n code : str,\n A string of Python code that imports and executes a kernel entry point.\n\n stdin, stdout, stderr : optional (default None)\n Standards streams, as defined in subprocess.Popen.\n\n fname : unicode, optional\n The JSON connector file, containing ip/port/hmac key information.\n\n key : str, optional\n The Session key used for HMAC authentication.\n\n executable : str, optional (default sys.executable)\n The Python executable to use for the kernel process.\n\n independent : bool, optional (default False)\n If set, the kernel process is guaranteed to survive if this process\n dies. If not set, an effort is made to ensure that the kernel is killed\n when this process dies. Note that in this case it is still good practice\n to kill kernels manually before exiting.\n\n extra_arguments : list, optional\n A list of extra arguments to pass when executing the launch code.\n \n cwd : path, optional\n The working dir of the kernel process (default: cwd of this process).\n\n Returns\n -------\n A tuple of form:\n (kernel_process, shell_port, iopub_port, stdin_port, hb_port)\n where kernel_process is a Popen object and the ports are integers.\n \"\"\"\n \n # Build the kernel launch command.\n if executable is None:\n executable = sys.executable\n arguments = [ executable, '-c', code, '-f', fname ]\n arguments.extend(extra_arguments)\n\n # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr\n # are invalid. Unfortunately, there is in general no way to detect whether\n # they are valid. The following two blocks redirect them to (temporary)\n # pipes in certain important cases.\n\n # If this process has been backgrounded, our stdin is invalid. Since there\n # is no compelling reason for the kernel to inherit our stdin anyway, we'll\n # place this one safe and always redirect.\n redirect_in = True\n _stdin = PIPE if stdin is None else stdin\n\n # If this process in running on pythonw, we know that stdin, stdout, and\n # stderr are all invalid.\n redirect_out = sys.executable.endswith('pythonw.exe')\n if redirect_out:\n _stdout = PIPE if stdout is None else stdout\n _stderr = PIPE if stderr is None else stderr\n else:\n _stdout, _stderr = stdout, stderr\n\n # Spawn a kernel.\n if sys.platform == 'win32':\n # Create a Win32 event for interrupting the kernel.\n interrupt_event = ParentPollerWindows.create_interrupt_event()\n arguments += [ '--interrupt=%i'%interrupt_event ]\n\n # If the kernel is running on pythonw and stdout/stderr are not been\n # re-directed, it will crash when more than 4KB of data is written to\n # stdout or stderr. This is a bug that has been with Python for a very\n # long time; see http://bugs.python.org/issue706263.\n # A cleaner solution to this problem would be to pass os.devnull to\n # Popen directly. Unfortunately, that does not work.\n if executable.endswith('pythonw.exe'):\n if stdout is None:\n arguments.append('--no-stdout')\n if stderr is None:\n arguments.append('--no-stderr')\n\n # Launch the kernel process.\n if independent:\n proc = Popen(arguments,\n creationflags=512, # CREATE_NEW_PROCESS_GROUP\n stdin=_stdin, stdout=_stdout, stderr=_stderr)\n else:\n try:\n from _winapi import DuplicateHandle, GetCurrentProcess, \\\n DUPLICATE_SAME_ACCESS\n except:\n from _subprocess import DuplicateHandle, GetCurrentProcess, \\\n DUPLICATE_SAME_ACCESS\n pid = GetCurrentProcess()\n handle = DuplicateHandle(pid, pid, pid, 0,\n True, # Inheritable by new processes.\n DUPLICATE_SAME_ACCESS)\n proc = Popen(arguments + ['--parent=%i'%int(handle)],\n stdin=_stdin, stdout=_stdout, stderr=_stderr)\n\n # Attach the interrupt event to the Popen objet so it can be used later.\n proc.win32_interrupt_event = interrupt_event\n\n else:\n if independent:\n proc = Popen(arguments, preexec_fn=lambda: os.setsid(),\n stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd)\n else:\n proc = Popen(arguments + ['--parent=1'],\n stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd)\n\n # Clean up pipes created to work around Popen bug.\n if redirect_in:\n if stdin is None:\n proc.stdin.close()\n if redirect_out:\n if stdout is None:\n proc.stdout.close()\n if stderr is None:\n proc.stderr.close()\n\n return proc"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_zipfile(context):\n if not prerequisites_ok():\n return\n # Create a zipfile.\n subprocess.call(['make', 'zip'])\n for zipfile in glob.glob('*.zip'):\n first_part = zipfile.split('.')[0]\n new_name = \"%s.%s.zip\" % (first_part, context['version'])\n target = os.path.join(context['workingdir'], new_name)\n shutil.copy(zipfile, target)\n print(\"Copied %s to %s\" % (zipfile, target))", "response": "Create a zip file for the release tree."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfix the version in metadata. txt Relevant context dict item for both prerelease and postrelease.", "response": "def fix_version(context):\n \"\"\"Fix the version in metadata.txt\n\n Relevant context dict item for both prerelease and postrelease:\n ``new_version``.\n\n \"\"\"\n if not prerequisites_ok():\n return\n lines = codecs.open('metadata.txt', 'rU', 'utf-8').readlines()\n for index, line in enumerate(lines):\n if line.startswith('version'):\n new_line = 'version=%s\\n' % context['new_version']\n lines[index] = new_line\n time.sleep(1)\n codecs.open('metadata.txt', 'w', 'utf-8').writelines(lines)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef mappable(obj):\n if isinstance(obj, (tuple,list)):\n return True\n for m in arrayModules:\n if isinstance(obj,m['type']):\n return True\n return False", "response": "return whether an object is mappable or not"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getPartition(self, seq, p, q):\n \n # Test for error conditions here\n if p<0 or p>=q:\n print \"No partition exists.\"\n return\n \n remainder = len(seq)%q\n basesize = len(seq)//q\n hi = []\n lo = []\n for n in range(q):\n if n < remainder:\n lo.append(n * (basesize + 1))\n hi.append(lo[-1] + basesize + 1)\n else:\n lo.append(n*basesize + remainder)\n hi.append(lo[-1] + basesize)\n \n try:\n result = seq[lo[p]:hi[p]]\n except TypeError:\n # some objects (iterators) can't be sliced,\n # use islice:\n result = list(islice(seq, lo[p], hi[p]))\n \n return result", "response": "Returns the pth partition of q partitions of seq."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pexpect_monkeypatch():\n\n if pexpect.__version__[:3] >= '2.2':\n # No need to patch, fix is already the upstream version.\n return\n\n def __del__(self):\n \"\"\"This makes sure that no system resources are left open.\n Python only garbage collects Python objects. OS file descriptors\n are not Python objects, so they must be handled explicitly.\n If the child file descriptor was opened outside of this class\n (passed to the constructor) then this does not close it.\n \"\"\"\n if not self.closed:\n try:\n self.close()\n except AttributeError:\n pass\n\n pexpect.spawn.__del__ = __del__", "response": "Monkeypatch pexpect to prevent unhandled exceptions at VM teardown."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef main():\n\n parser = optparse.OptionParser(usage=MAIN_USAGE)\n newopt = parser.add_option\n newopt('--ipython',action='store_const',dest='mode',const='ipython',\n help='IPython interactive runner (default).')\n newopt('--python',action='store_const',dest='mode',const='python',\n help='Python interactive runner.')\n newopt('--sage',action='store_const',dest='mode',const='sage',\n help='SAGE interactive runner.')\n\n opts,args = parser.parse_args()\n runners = dict(ipython=IPythonRunner,\n python=PythonRunner,\n sage=SAGERunner)\n\n try:\n ext = os.path.splitext(args[0])[-1]\n except IndexError:\n ext = ''\n modes = {'.ipy':'ipython',\n '.py':'python',\n '.sage':'sage'}\n mode = modes.get(ext,\"ipython\")\n if opts.mode:\n mode = opts.mode\n runners[mode]().main(args)", "response": "Run as a command - line script."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run_file(self,fname,interact=False,get_output=False):\n\n fobj = open(fname,'r')\n try:\n out = self.run_source(fobj,interact,get_output)\n finally:\n fobj.close()\n if get_output:\n return out", "response": "Runs the given file interactively."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run_source(self,source,interact=False,get_output=False):\n\n # if the source is a string, chop it up in lines so we can iterate\n # over it just as if it were an open file.\n if isinstance(source, basestring):\n source = source.splitlines(True)\n\n if self.echo:\n # normalize all strings we write to use the native OS line\n # separators.\n linesep = os.linesep\n stdwrite = self.out.write\n write = lambda s: stdwrite(s.replace('\\r\\n',linesep))\n else:\n # Quiet mode, all writes are no-ops\n write = lambda s: None\n\n c = self.child\n prompts = c.compile_pattern_list(self.prompts)\n prompt_idx = c.expect_list(prompts)\n\n # Flag whether the script ends normally or not, to know whether we can\n # do anything further with the underlying process.\n end_normal = True\n\n # If the output was requested, store it in a list for return at the end\n if get_output:\n output = []\n store_output = output.append\n\n for cmd in source:\n # skip blank lines for all matches to the 'main' prompt, while the\n # secondary prompts do not\n if prompt_idx==0 and \\\n (cmd.isspace() or cmd.lstrip().startswith('#')):\n write(cmd)\n continue\n\n # write('AFTER: '+c.after) # dbg\n write(c.after)\n c.send(cmd)\n try:\n prompt_idx = c.expect_list(prompts)\n except pexpect.EOF:\n # this will happen if the child dies unexpectedly\n write(c.before)\n end_normal = False\n break\n\n write(c.before)\n\n # With an echoing process, the output we get in c.before contains\n # the command sent, a newline, and then the actual process output\n if get_output:\n store_output(c.before[len(cmd+'\\n'):])\n #write('CMD: <<%s>>' % cmd) # dbg\n #write('OUTPUT: <<%s>>' % output[-1]) # dbg\n\n self.out.flush()\n if end_normal:\n if interact:\n c.send('\\n')\n print '<< Starting interactive mode >>',\n try:\n c.interact()\n except OSError:\n # This is what fires when the child stops. Simply print a\n # newline so the system prompt is aligned. The extra\n # space is there to make sure it gets printed, otherwise\n # OS buffering sometimes just suppresses it.\n write(' \\n')\n self.out.flush()\n else:\n if interact:\n e=\"Further interaction is not possible: child process is dead.\"\n print >> sys.stderr, e\n\n # Leave the child ready for more input later on, otherwise select just\n # hangs on the second invocation.\n if c.isalive():\n c.send('\\n')\n\n # Return any requested output\n if get_output:\n return ''.join(output)", "response": "Run the given source code interactively."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nruns as a command - line script.", "response": "def main(self,argv=None):\n \"\"\"Run as a command-line script.\"\"\"\n\n parser = optparse.OptionParser(usage=USAGE % self.__class__.__name__)\n newopt = parser.add_option\n newopt('-i','--interact',action='store_true',default=False,\n help='Interact with the program after the script is run.')\n\n opts,args = parser.parse_args(argv)\n\n if len(args) != 1:\n print >> sys.stderr,\"You must supply exactly one file to run.\"\n sys.exit(1)\n\n self.run_file(args[0],opts.interact)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating a Cobertura - compatible XML report for a list of modules or filenames.", "response": "def report(self, morfs, outfile=None):\n \"\"\"Generate a Cobertura-compatible XML report for `morfs`.\n\n `morfs` is a list of modules or filenames.\n\n `outfile` is a file object to write the XML to.\n\n \"\"\"\n # Initial setup.\n outfile = outfile or sys.stdout\n\n # Create the DOM that will store the data.\n impl = xml.dom.minidom.getDOMImplementation()\n docType = impl.createDocumentType(\n \"coverage\", None,\n \"http://cobertura.sourceforge.net/xml/coverage-03.dtd\"\n )\n self.xml_out = impl.createDocument(None, \"coverage\", docType)\n\n # Write header stuff.\n xcoverage = self.xml_out.documentElement\n xcoverage.setAttribute(\"version\", __version__)\n xcoverage.setAttribute(\"timestamp\", str(int(time.time()*1000)))\n xcoverage.appendChild(self.xml_out.createComment(\n \" Generated by coverage.py: %s \" % __url__\n ))\n xpackages = self.xml_out.createElement(\"packages\")\n xcoverage.appendChild(xpackages)\n\n # Call xml_file for each file in the data.\n self.packages = {}\n self.report_files(self.xml_file, morfs)\n\n lnum_tot, lhits_tot = 0, 0\n bnum_tot, bhits_tot = 0, 0\n\n # Populate the XML DOM with the package info.\n for pkg_name in sorted(self.packages.keys()):\n pkg_data = self.packages[pkg_name]\n class_elts, lhits, lnum, bhits, bnum = pkg_data\n xpackage = self.xml_out.createElement(\"package\")\n xpackages.appendChild(xpackage)\n xclasses = self.xml_out.createElement(\"classes\")\n xpackage.appendChild(xclasses)\n for class_name in sorted(class_elts.keys()):\n xclasses.appendChild(class_elts[class_name])\n xpackage.setAttribute(\"name\", pkg_name.replace(os.sep, '.'))\n xpackage.setAttribute(\"line-rate\", rate(lhits, lnum))\n xpackage.setAttribute(\"branch-rate\", rate(bhits, bnum))\n xpackage.setAttribute(\"complexity\", \"0\")\n\n lnum_tot += lnum\n lhits_tot += lhits\n bnum_tot += bnum\n bhits_tot += bhits\n\n xcoverage.setAttribute(\"line-rate\", rate(lhits_tot, lnum_tot))\n xcoverage.setAttribute(\"branch-rate\", rate(bhits_tot, bnum_tot))\n\n # Use the DOM to write the output file.\n outfile.write(self.xml_out.toprettyxml())\n\n # Return the total percentage.\n denom = lnum_tot + bnum_tot\n if denom == 0:\n pct = 0.0\n else:\n pct = 100.0 * (lhits_tot + bhits_tot) / denom\n return pct"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef xml_file(self, cu, analysis):\n\n # Create the 'lines' and 'package' XML elements, which\n # are populated later. Note that a package == a directory.\n package_name = rpartition(cu.name, \".\")[0]\n className = cu.name\n\n package = self.packages.setdefault(package_name, [{}, 0, 0, 0, 0])\n\n xclass = self.xml_out.createElement(\"class\")\n\n xclass.appendChild(self.xml_out.createElement(\"methods\"))\n\n xlines = self.xml_out.createElement(\"lines\")\n xclass.appendChild(xlines)\n\n xclass.setAttribute(\"name\", className)\n filename = cu.file_locator.relative_filename(cu.filename)\n xclass.setAttribute(\"filename\", filename.replace(\"\\\\\", \"/\"))\n xclass.setAttribute(\"complexity\", \"0\")\n\n branch_stats = analysis.branch_stats()\n\n # For each statement, create an XML 'line' element.\n for line in sorted(analysis.statements):\n xline = self.xml_out.createElement(\"line\")\n xline.setAttribute(\"number\", str(line))\n\n # Q: can we get info about the number of times a statement is\n # executed? If so, that should be recorded here.\n xline.setAttribute(\"hits\", str(int(line not in analysis.missing)))\n\n if self.arcs:\n if line in branch_stats:\n total, taken = branch_stats[line]\n xline.setAttribute(\"branch\", \"true\")\n xline.setAttribute(\"condition-coverage\",\n \"%d%% (%d/%d)\" % (100*taken/total, taken, total)\n )\n xlines.appendChild(xline)\n\n class_lines = len(analysis.statements)\n class_hits = class_lines - len(analysis.missing)\n\n if self.arcs:\n class_branches = sum([t for t,k in branch_stats.values()])\n missing_branches = sum([t-k for t,k in branch_stats.values()])\n class_br_hits = class_branches - missing_branches\n else:\n class_branches = 0.0\n class_br_hits = 0.0\n\n # Finalize the statistics that are collected in the XML DOM.\n xclass.setAttribute(\"line-rate\", rate(class_hits, class_lines))\n xclass.setAttribute(\"branch-rate\", rate(class_br_hits, class_branches))\n package[0][className] = xclass\n package[1] += class_hits\n package[2] += class_lines\n package[3] += class_br_hits\n package[4] += class_branches", "response": "Add to the XML report for a single file."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef phistogram(view, a, bins=10, rng=None, normed=False):\n nengines = len(view.targets)\n \n # view.push(dict(bins=bins, rng=rng))\n with view.sync_imports():\n import numpy\n rets = view.apply_sync(lambda a, b, rng: numpy.histogram(a,b,rng), Reference(a), bins, rng)\n hists = [ r[0] for r in rets ]\n lower_edges = [ r[1] for r in rets ]\n # view.execute('hist, lower_edges = numpy.histogram(%s, bins, rng)' % a)\n lower_edges = view.pull('lower_edges', targets=0)\n hist_array = numpy.array(hists).reshape(nengines, -1)\n # hist_array.shape = (nengines,-1)\n total_hist = numpy.sum(hist_array, 0)\n if normed:\n total_hist = total_hist/numpy.sum(total_hist,dtype=float)\n return total_hist, lower_edges", "response": "Compute the histogram of a remote array a."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fetch_pi_file(filename):\n import os, urllib\n ftpdir=\"ftp://pi.super-computing.org/.2/pi200m/\"\n if os.path.exists(filename):\n # we already have it\n return\n else:\n # download it\n urllib.urlretrieve(ftpdir+filename,filename)", "response": "This will download a segment of pi from super - Computing. org if the file is already present."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef reduce_freqs(freqlist):\n allfreqs = np.zeros_like(freqlist[0])\n for f in freqlist:\n allfreqs += f\n return allfreqs", "response": "Reduce the frequency list to get the total counts."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads digits of pi from a file and compute the n digit frequencies.", "response": "def compute_n_digit_freqs(filename, n):\n \"\"\"\n Read digits of pi from a file and compute the n digit frequencies.\n \"\"\"\n d = txt_file_to_digits(filename)\n freqs = n_digit_freqs(d, n)\n return freqs"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nyield the digits of pi read from a. txt file.", "response": "def txt_file_to_digits(filename, the_type=str):\n \"\"\"\n Yield the digits of pi read from a .txt file.\n \"\"\"\n with open(filename, 'r') as f:\n for line in f.readlines():\n for c in line:\n if c != '\\n' and c!= ' ':\n yield the_type(c)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the frequency of one digit pi.", "response": "def one_digit_freqs(digits, normalize=False):\n \"\"\"\n Consume digits of pi and compute 1 digit freq. counts.\n \"\"\"\n freqs = np.zeros(10, dtype='i4')\n for d in digits:\n freqs[int(d)] += 1\n if normalize:\n freqs = freqs/freqs.sum()\n return freqs"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef two_digit_freqs(digits, normalize=False):\n freqs = np.zeros(100, dtype='i4')\n last = digits.next()\n this = digits.next()\n for d in digits:\n index = int(last + this)\n freqs[index] += 1\n last = this\n this = d\n if normalize:\n freqs = freqs/freqs.sum()\n return freqs", "response": "Compute 2 digits frequency for each pi."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute n - digit frequency for a given set of digits.", "response": "def n_digit_freqs(digits, n, normalize=False):\n \"\"\"\n Consume digits of pi and compute n digits freq. counts.\n\n This should only be used for 1-6 digits.\n \"\"\"\n freqs = np.zeros(pow(10,n), dtype='i4')\n current = np.zeros(n, dtype=int)\n for i in range(n):\n current[i] = digits.next()\n for d in digits:\n index = int(''.join(map(str, current)))\n freqs[index] += 1\n current[0:-1] = current[1:]\n current[-1] = d\n if normalize:\n freqs = freqs/freqs.sum()\n return freqs"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef plot_two_digit_freqs(f2):\n f2_copy = f2.copy()\n f2_copy.shape = (10,10)\n ax = plt.matshow(f2_copy)\n plt.colorbar()\n for i in range(10):\n for j in range(10):\n plt.text(i-0.2, j+0.2, str(j)+str(i))\n plt.ylabel('First digit')\n plt.xlabel('Second digit')\n return ax", "response": "Plot two digits frequency counts using matplotlib."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nplots one digit frequency counts using matplotlib.", "response": "def plot_one_digit_freqs(f1):\n \"\"\"\n Plot one digit frequency counts using matplotlib.\n \"\"\"\n ax = plt.plot(f1,'bo-')\n plt.title('Single digit counts in pi')\n plt.xlabel('Digit')\n plt.ylabel('Count')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __extend_uri(self, short):\n if short == 'a':\n return RDF.type\n for prefix in sorted(self.__prefixes, key=lambda x: len(x), reverse=True):\n if short.startswith(prefix):\n return URIRef(short.replace(prefix + ':', self.__prefixes[prefix]))\n return short", "response": "Extend a prefixed uri with the help of a specific dictionary of prefixes\n "} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_object_or_none(qs, *args, **kwargs):\n\n try:\n return qs.get(*args, **kwargs)\n except models.ObjectDoesNotExist:\n return None", "response": "Try to retrieve a model and return None if it is not found."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef extract_vars(*names,**kw):\n\n depth = kw.get('depth',0)\n \n callerNS = sys._getframe(depth+1).f_locals\n return dict((k,callerNS[k]) for k in names)", "response": "Extract a set of variables by name from another frame."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef extract_vars_above(*names):\n\n callerNS = sys._getframe(2).f_locals\n return dict((k,callerNS[k]) for k in names)", "response": "Extract a set of variables by name from another frame."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef debugx(expr,pre_msg=''):\n\n cf = sys._getframe(1)\n print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,\n eval(expr,cf.f_globals,cf.f_locals))", "response": "Print the value of an expression from the caller s frame and returns it."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the module and locals of the funciton depth frames away from the caller", "response": "def extract_module_locals(depth=0):\n \"\"\"Returns (module, locals) of the funciton `depth` frames away from the caller\"\"\"\n f = sys._getframe(depth + 1)\n global_ns = f.f_globals\n module = sys.modules[global_ns['__name__']]\n return (module, f.f_locals)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef reverse(view, *args, **kwargs):\n\t'''\n\tUser-friendly reverse. Pass arguments and keyword arguments to Django's `reverse`\n\tas `args` and `kwargs` arguments, respectively.\n\n\tThe special optional keyword argument `query` is a dictionary of query (or GET) parameters\n\tthat can be appended to the `reverse`d URL.\n\n\tExample:\n\n\t\treverse('products:category', categoryId = 5, query = {'page': 2})\n\n\tis equivalent to\n\n\t\tdjango.core.urlresolvers.reverse('products:category', kwargs = {'categoryId': 5}) + '?page=2'\n\n\t'''\n\tif 'query' in kwargs:\n\t\tquery = kwargs.pop('query')\n\telse:\n\t\tquery = None\n\tbase = urlresolvers.reverse(view, args = args, kwargs = kwargs)\n\tif query:\n\t\treturn '{}?{}'.format(base, django.utils.http.urlencode(query))\n\telse:\n\t\treturn base", "response": "A function that uses Django s reverse method to provide a base URL for the view."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns True iff prefix +. + base is private.", "response": "def is_private(prefix, base):\n \"\"\"prefix, base -> true iff name prefix + \".\" + base is \"private\".\n\n Prefix may be an empty string, and base does not contain a period.\n Prefix is ignored (although functions you write conforming to this\n protocol may make use of it).\n Return true iff base begins with an (at least one) underscore, but\n does not both begin and end with (at least) two underscores.\n \"\"\"\n warnings.warn(\"is_private is deprecated; it wasn't useful; \"\n \"examine DocTestFinder.find() lists instead\",\n DeprecationWarning, stacklevel=2)\n return base[:1] == \"_\" and not base[:2] == \"__\" == base[-2:]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _extract_future_flags(globs):\n flags = 0\n for fname in __future__.all_feature_names:\n feature = globs.get(fname, None)\n if feature is getattr(__future__, fname):\n flags |= feature.compiler_flag\n return flags", "response": "Extract the compiler - flags associated with the future features that have been imported into the given namespace."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the module specified by module.", "response": "def _normalize_module(module, depth=2):\n \"\"\"\n Return the module specified by `module`. In particular:\n - If `module` is a module, then return module.\n - If `module` is a string, then import and return the\n module with that name.\n - If `module` is None, then return the calling module.\n The calling module is assumed to be the module of\n the stack frame at the given depth in the call stack.\n \"\"\"\n if inspect.ismodule(module):\n return module\n elif isinstance(module, (str, unicode)):\n return __import__(module, globals(), locals(), [\"*\"])\n elif module is None:\n return sys.modules[sys._getframe(depth).f_globals['__name__']]\n else:\n raise TypeError(\"Expected a module, string, or None\")"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a string containing a traceback message for the given exception tuple.", "response": "def _exception_traceback(exc_info):\n \"\"\"\n Return a string containing a traceback message for the given\n exc_info tuple (as returned by sys.exc_info()).\n \"\"\"\n # Get a traceback message.\n excout = StringIO()\n exc_type, exc_val, exc_tb = exc_info\n traceback.print_exception(exc_type, exc_val, exc_tb, file=excout)\n return excout.getvalue()"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nruns all examples in a given object s docstring.", "response": "def run_docstring_examples(f, globs, verbose=False, name=\"NoName\",\n compileflags=None, optionflags=0):\n \"\"\"\n Test examples in the given object's docstring (`f`), using `globs`\n as globals. Optional argument `name` is used in failure messages.\n If the optional argument `verbose` is true, then generate output\n even if there are no failures.\n\n `compileflags` gives the set of flags that should be used by the\n Python compiler when running the examples. If not specified, then\n it will default to the set of future-import flags that apply to\n `globs`.\n\n Optional keyword arg `optionflags` specifies options for the\n testing and output. See the documentation for `testmod` for more\n information.\n \"\"\"\n # Find, parse, and run all tests in the given module.\n finder = DocTestFinder(verbose=verbose, recurse=False)\n runner = DocTestRunner(verbose=verbose, optionflags=optionflags)\n for test in finder.find(f, name, globs=globs):\n runner.run(test, compileflags=compileflags)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DocFileSuite(*paths, **kw):\n suite = unittest.TestSuite()\n\n # We do this here so that _normalize_module is called at the right\n # level. If it were called in DocFileTest, then this function\n # would be the caller and we might guess the package incorrectly.\n if kw.get('module_relative', True):\n kw['package'] = _normalize_module(kw.get('package'))\n\n for path in paths:\n suite.addTest(DocFileTest(path, **kw))\n\n return suite", "response": "A unittest suite for one or more doctest files."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef debug_src(src, pm=False, globs=None):\n testsrc = script_from_examples(src)\n debug_script(testsrc, pm, globs)", "response": "Debug a single doctest docstring in argument src."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef debug_script(src, pm=False, globs=None):\n \"Debug a test script. `src` is the script, as a string.\"\n import pdb\n\n # Note that tempfile.NameTemporaryFile() cannot be used. As the\n # docs say, a file so created cannot be opened by name a second time\n # on modern Windows boxes, and execfile() needs to open it.\n srcfilename = tempfile.mktemp(\".py\", \"doctestdebug\")\n f = open(srcfilename, 'w')\n f.write(src)\n f.close()\n\n try:\n if globs:\n globs = globs.copy()\n else:\n globs = {}\n\n if pm:\n try:\n execfile(srcfilename, globs, globs)\n except:\n print sys.exc_info()[1]\n pdb.post_mortem(sys.exc_info()[2])\n else:\n # Note that %r is vital here. '%s' instead can, e.g., cause\n # backslashes to get treated as metacharacters on Windows.\n pdb.run(\"execfile(%r)\" % srcfilename, globs, globs)\n\n finally:\n os.remove(srcfilename)", "response": "Debug a test script. src is the script as a string."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndebug a single doctest docstring.", "response": "def debug(module, name, pm=False):\n \"\"\"Debug a single doctest docstring.\n\n Provide the module (or dotted name of the module) containing the\n test to be debugged and the name (within the module) of the object\n with the docstring with tests to be debugged.\n \"\"\"\n module = _normalize_module(module)\n testsrc = testsource(module, name)\n debug_script(testsrc, pm, module.__dict__)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking if the actual output from an example got matches the expected output.", "response": "def check_output(self, want, got, optionflags):\n \"\"\"\n Return True iff the actual output from an example (`got`)\n matches the expected output (`want`). These strings are\n always considered to match if they are identical; but\n depending on what option flags the test runner is using,\n several non-exact match types are also possible. See the\n documentation for `TestRunner` for more information about\n option flags.\n \"\"\"\n # Handle the common case first, for efficiency:\n # if they're string-identical, always return true.\n if got == want:\n return True\n\n # The values True and False replaced 1 and 0 as the return\n # value for boolean comparisons in Python 2.3.\n if not (optionflags & DONT_ACCEPT_TRUE_FOR_1):\n if (got,want) == (\"True\\n\", \"1\\n\"):\n return True\n if (got,want) == (\"False\\n\", \"0\\n\"):\n return True\n\n # can be used as a special sequence to signify a\n # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used.\n if not (optionflags & DONT_ACCEPT_BLANKLINE):\n # Replace in want with a blank line.\n want = re.sub('(?m)^%s\\s*?$' % re.escape(BLANKLINE_MARKER),\n '', want)\n # If a line in got contains only spaces, then remove the\n # spaces.\n got = re.sub('(?m)^\\s*?$', '', got)\n if got == want:\n return True\n\n # This flag causes doctest to ignore any differences in the\n # contents of whitespace strings. Note that this can be used\n # in conjunction with the ELLIPSIS flag.\n if optionflags & NORMALIZE_WHITESPACE:\n got = ' '.join(got.split())\n want = ' '.join(want.split())\n if got == want:\n return True\n\n # The ELLIPSIS flag says to let the sequence \"...\" in `want`\n # match any substring in `got`.\n if optionflags & ELLIPSIS:\n if _ellipsis_match(want, got):\n return True\n\n # We didn't find any match; return false.\n return False"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a string describing the differences between the expected and actual entries for a given example and the actual entries for a given set of options.", "response": "def output_difference(self, example, got, optionflags):\n \"\"\"\n Return a string describing the differences between the\n expected output for a given example (`example`) and the actual\n output (`got`). `optionflags` is the set of option flags used\n to compare `want` and `got`.\n \"\"\"\n want = example.want\n # If s are being used, then replace blank lines\n # with in the actual output string.\n if not (optionflags & DONT_ACCEPT_BLANKLINE):\n got = re.sub('(?m)^[ ]*(?=\\n)', BLANKLINE_MARKER, got)\n\n # Check if we should use diff.\n if self._do_a_fancy_diff(want, got, optionflags):\n # Split want & got into lines.\n want_lines = want.splitlines(True) # True == keep line ends\n got_lines = got.splitlines(True)\n # Use difflib to find their differences.\n if optionflags & REPORT_UDIFF:\n diff = difflib.unified_diff(want_lines, got_lines, n=2)\n diff = list(diff)[2:] # strip the diff header\n kind = 'unified diff with -expected +actual'\n elif optionflags & REPORT_CDIFF:\n diff = difflib.context_diff(want_lines, got_lines, n=2)\n diff = list(diff)[2:] # strip the diff header\n kind = 'context diff with expected followed by actual'\n elif optionflags & REPORT_NDIFF:\n engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)\n diff = list(engine.compare(want_lines, got_lines))\n kind = 'ndiff with -expected +actual'\n else:\n assert 0, 'Bad diff option'\n # Remove trailing whitespace on diff output.\n diff = [line.rstrip() + '\\n' for line in diff]\n return 'Differences (%s):\\n' % kind + _indent(''.join(diff))\n\n # If we're not using diff, then simply list the expected\n # output followed by the actual output.\n if want and got:\n return 'Expected:\\n%sGot:\\n%s' % (_indent(want), _indent(got))\n elif want:\n return 'Expected:\\n%sGot nothing\\n' % _indent(want)\n elif got:\n return 'Expected nothing\\nGot:\\n%s' % _indent(got)\n else:\n return 'Expected nothing\\nGot nothing\\n'"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting a hash entry", "response": "def hset(self, hashroot, key, value):\n \"\"\" hashed set \"\"\"\n hroot = self.root / hashroot\n if not hroot.isdir():\n hroot.makedirs()\n hfile = hroot / gethashfile(key)\n d = self.get(hfile, {})\n d.update( {key : value})\n self[hfile] = d"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets all data contained in hashed category hashroot as dict", "response": "def hdict(self, hashroot):\n \"\"\" Get all data contained in hashed category 'hashroot' as dict \"\"\"\n hfiles = self.keys(hashroot + \"/*\")\n hfiles.sort()\n last = len(hfiles) and hfiles[-1] or ''\n if last.endswith('xx'):\n # print \"using xx\"\n hfiles = [last] + hfiles[:-1]\n\n all = {}\n\n for f in hfiles:\n # print \"using\",f\n try:\n all.update(self[f])\n except KeyError:\n print \"Corrupt\",f,\"deleted - hset is not threadsafe!\"\n del self[f]\n\n self.uncache(f)\n\n return all"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef hcompress(self, hashroot):\n hfiles = self.keys(hashroot + \"/*\")\n all = {}\n for f in hfiles:\n # print \"using\",f\n all.update(self[f])\n self.uncache(f)\n\n self[hashroot + '/xx'] = all\n for f in hfiles:\n p = self.root / f\n if p.basename() == 'xx':\n continue\n p.remove()", "response": "Compress category hashroot so that fast_only is True for compressed items."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef show_items(self, cursor, items):\n if not items :\n return\n self.cancel_completion()\n strng = text.columnize(items)\n self._console_widget._fill_temporary_buffer(cursor, strng, html=False)", "response": "Shows the completion widget with items at the position specified\n by cursor."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning whether this record should be printed", "response": "def allow(self, record):\n \"\"\"returns whether this record should be printed\"\"\"\n if not self:\n # nothing to filter\n return True\n return self._allow(record) and not self._deny(record)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _any_match(matchers, record):\n def record_matches_key(key):\n return record == key or record.startswith(key + '.')\n return anyp(bool, map(record_matches_key, matchers))", "response": "return the bool of whether record starts with\n any item in matchers"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef configure(self, options, conf):\n self.conf = conf\n # Disable if explicitly disabled, or if logging is\n # configured via logging config file\n if not options.logcapture or conf.loggingConfig:\n self.enabled = False\n self.logformat = options.logcapture_format\n self.logdatefmt = options.logcapture_datefmt\n self.clear = options.logcapture_clear\n self.loglevel = options.logcapture_level\n if options.logcapture_filters:\n self.filters = options.logcapture_filters.split(',')", "response": "Configure the object based on the options and conf."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd captured log messages to error output.", "response": "def formatError(self, test, err):\n \"\"\"Add captured log messages to error output.\n \"\"\"\n # logic flow copied from Capture.formatError\n test.capturedLogging = records = self.formatLogRecords()\n if not records:\n return err\n ec, ev, tb = err\n return (ec, self.addCaptureToErr(ev, records), tb)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef embed(**kwargs):\n config = kwargs.get('config')\n header = kwargs.pop('header', u'')\n if config is None:\n config = load_default_config()\n config.InteractiveShellEmbed = config.TerminalInteractiveShell\n kwargs['config'] = config\n global _embedded_shell\n if _embedded_shell is None:\n _embedded_shell = InteractiveShellEmbed(**kwargs)\n _embedded_shell(header=header, stack_depth=2)", "response": "Embed the current IPython environment into a new instance of the same name."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef mainloop(self, local_ns=None, module=None, stack_depth=0,\n display_banner=None, global_ns=None):\n \"\"\"Embeds IPython into a running python program.\n\n Input:\n\n - header: An optional header message can be specified.\n\n - local_ns, module: working local namespace (a dict) and module (a\n module or similar object). If given as None, they are automatically\n taken from the scope where the shell was called, so that\n program variables become visible.\n\n - stack_depth: specifies how many levels in the stack to go to\n looking for namespaces (when local_ns or module is None). This\n allows an intermediate caller to make sure that this function gets\n the namespace from the intended level in the stack. By default (0)\n it will get its locals and globals from the immediate caller.\n\n Warning: it's possible to use this in a program which is being run by\n IPython itself (via %run), but some funny things will happen (a few\n globals get overwritten). In the future this will be cleaned up, as\n there is no fundamental reason why it can't work perfectly.\"\"\"\n \n if (global_ns is not None) and (module is None):\n class DummyMod(object):\n \"\"\"A dummy module object for embedded IPython.\"\"\"\n pass\n warnings.warn(\"global_ns is deprecated, use module instead.\", DeprecationWarning)\n module = DummyMod()\n module.__dict__ = global_ns\n\n # Get locals and globals from caller\n if (local_ns is None or module is None) and self.default_user_namespaces:\n call_frame = sys._getframe(stack_depth).f_back\n\n if local_ns is None:\n local_ns = call_frame.f_locals\n if module is None:\n global_ns = call_frame.f_globals\n module = sys.modules[global_ns['__name__']]\n \n # Save original namespace and module so we can restore them after \n # embedding; otherwise the shell doesn't shut down correctly.\n orig_user_module = self.user_module\n orig_user_ns = self.user_ns\n \n # Update namespaces and fire up interpreter\n \n # The global one is easy, we can just throw it in\n if module is not None:\n self.user_module = module\n\n # But the user/local one is tricky: ipython needs it to store internal\n # data, but we also need the locals. We'll throw our hidden variables\n # like _ih and get_ipython() into the local namespace, but delete them\n # later.\n if local_ns is not None:\n self.user_ns = local_ns\n self.init_user_ns()\n\n # Patch for global embedding to make sure that things don't overwrite\n # user globals accidentally. Thanks to Richard \n # FIXME. Test this a bit more carefully (the if.. is new)\n # N.B. This can't now ever be called. Not sure what it was for.\n # And now, since it wasn't called in the previous version, I'm\n # commenting out these lines so they can't be called with my new changes\n # --TK, 2011-12-10\n #if local_ns is None and module is None:\n # self.user_global_ns.update(__main__.__dict__)\n\n # make sure the tab-completer has the correct frame information, so it\n # actually completes using the frame's locals/globals\n self.set_completer_frame()\n\n with nested(self.builtin_trap, self.display_trap):\n self.interact(display_banner=display_banner)\n \n # now, purge out the local namespace of IPython's hidden variables.\n if local_ns is not None:\n for name in self.user_ns_hidden:\n local_ns.pop(name, None)\n \n # Restore original namespace so shell can shut down when we exit.\n self.user_module = orig_user_module\n self.user_ns = orig_user_ns", "response": "This function is called by the IPython mainloop function."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget all po filenames from locale folder and return list of them.", "response": "def _get_all_po_filenames(locale_root, lang, po_files_path):\n \"\"\"\n Get all po filenames from locale folder and return list of them.\n Assumes a directory structure:\n ///.\n \"\"\"\n all_files = os.listdir(os.path.join(locale_root, lang, po_files_path))\n return filter(lambda s: s.endswith('.po'), all_files)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_new_csv_writers(trans_title, meta_title,\n trans_csv_path, meta_csv_path):\n \"\"\"\n Prepare new csv writers, write title rows and return them.\n \"\"\"\n trans_writer = UnicodeWriter(trans_csv_path)\n trans_writer.writerow(trans_title)\n\n meta_writer = UnicodeWriter(meta_csv_path)\n meta_writer.writerow(meta_title)\n\n return trans_writer, meta_writer", "response": "Prepare new csv writers write title rows and return them."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _prepare_locale_dirs(languages, locale_root):\n trans_languages = []\n for i, t in enumerate(languages):\n lang = t.split(':')[0]\n trans_languages.append(lang)\n lang_path = os.path.join(locale_root, lang)\n if not os.path.exists(lang_path):\n os.makedirs(lang_path)\n return trans_languages", "response": "Prepare locale dirs for writing po files."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprepare a dictionary of files for writing and reading from them.", "response": "def _prepare_polib_files(files_dict, filename, languages,\n locale_root, po_files_path, header):\n \"\"\"\n Prepare polib file object for writing/reading from them.\n Create directories and write header if needed. For each language,\n ensure there's a translation file named \"filename\" in the correct place.\n Assumes (and creates) a directory structure:\n ///.\n \"\"\"\n files_dict[filename] = {}\n for lang in languages:\n file_path = os.path.join(locale_root, lang, po_files_path)\n if not os.path.exists(file_path):\n os.makedirs(file_path)\n\n if header is not None:\n _write_header(os.path.join(file_path, filename), lang, header)\n\n files_dict[filename][lang] = polib.pofile(\n os.path.join(file_path, filename), encoding=\"UTF-8\")"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _write_entries(po_files, languages, msgid, msgstrs, metadata, comment):\n start = re.compile(r'^[\\s]+')\n end = re.compile(r'[\\s]+$')\n for i, lang in enumerate(languages):\n meta = ast.literal_eval(metadata)\n entry = polib.POEntry(**meta)\n entry.tcomment = comment\n entry.msgid = msgid\n if msgstrs[i]:\n start_ws = start.search(msgid)\n end_ws = end.search(msgid)\n entry.msgstr = str(start_ws.group() if start_ws else '') + \\\n unicode(msgstrs[i].strip()) + \\\n str(end_ws.group() if end_ws else '')\n else:\n entry.msgstr = ''\n po_files[lang].append(entry)", "response": "Write all entries in the languages list to the PO files."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _write_header(po_path, lang, header):\n po_file = open(po_path, 'w')\n po_file.write(header + '\\n')\n po_file.write(\n 'msgid \"\"' +\n '\\nmsgstr \"\"' +\n '\\n\"MIME-Version: ' + settings.METADATA['MIME-Version'] + r'\\n\"'\n '\\n\"Content-Type: ' + settings.METADATA['Content-Type'] + r'\\n\"'\n '\\n\"Content-Transfer-Encoding: ' +\n settings.METADATA['Content-Transfer-Encoding'] + r'\\n\"'\n '\\n\"Language: ' + lang + r'\\n\"' + '\\n')\n po_file.close()", "response": "Write header into po file for specific lang."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _write_new_messages(po_file_path, trans_writer, meta_writer,\n msgids, msgstrs, languages):\n \"\"\"\n Write new msgids which appeared in po files with empty msgstrs values\n and metadata. Look for all new msgids which are diffed with msgids list\n provided as an argument.\n \"\"\"\n po_filename = os.path.basename(po_file_path)\n po_file = polib.pofile(po_file_path)\n\n new_trans = 0\n for entry in po_file:\n if entry.msgid not in msgids:\n new_trans += 1\n trans = [po_filename, entry.tcomment, entry.msgid, entry.msgstr]\n for lang in languages[1:]:\n trans.append(msgstrs[lang].get(entry.msgid, ''))\n\n meta = dict(entry.__dict__)\n meta.pop('msgid', None)\n meta.pop('msgstr', None)\n meta.pop('tcomment', None)\n\n trans_writer.writerow(trans)\n meta_writer.writerow([str(meta)])\n\n return new_trans", "response": "Write new messages in a single language."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting new msgstrs from a po file.", "response": "def _get_new_msgstrs(po_file_path, msgids):\n \"\"\"\n Write new msgids which appeared in po files with empty msgstrs values\n and metadata. Look for all new msgids which are diffed with msgids list\n provided as an argument.\n \"\"\"\n po_file = polib.pofile(po_file_path)\n\n msgstrs = {}\n\n for entry in po_file:\n if entry.msgid not in msgids:\n msgstrs[entry.msgid] = entry.msgstr\n\n return msgstrs"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting po file to csv GDocs spreadsheet readable format.", "response": "def po_to_csv_merge(languages, locale_root, po_files_path,\n local_trans_csv, local_meta_csv,\n gdocs_trans_csv, gdocs_meta_csv):\n \"\"\"\n Converts po file to csv GDocs spreadsheet readable format.\n Merges them if some msgid aren't in the spreadsheet.\n :param languages: list of language codes\n :param locale_root: path to locale root folder containing directories\n with languages\n :param po_files_path: path from lang directory to po file\n :param local_trans_csv: path where local csv with translations\n will be created\n :param local_meta_csv: path where local csv with metadata will be created\n :param gdocs_trans_csv: path to gdoc csv with translations\n \"\"\"\n msgids = []\n\n trans_reader = UnicodeReader(gdocs_trans_csv)\n meta_reader = UnicodeReader(gdocs_meta_csv)\n\n try:\n trans_title = trans_reader.next()\n meta_title = meta_reader.next()\n except StopIteration:\n trans_title = ['file', 'comment', 'msgid']\n trans_title += map(lambda s: s + ':msgstr', languages)\n meta_title = ['metadata']\n\n trans_writer, meta_writer = _get_new_csv_writers(\n trans_title, meta_title, local_trans_csv, local_meta_csv)\n\n for trans_row, meta_row in izip_longest(trans_reader, meta_reader):\n msgids.append(trans_row[2])\n trans_writer.writerow(trans_row)\n meta_writer.writerow(meta_row if meta_row else [METADATA_EMPTY])\n\n trans_reader.close()\n meta_reader.close()\n\n po_files = _get_all_po_filenames(locale_root, languages[0], po_files_path)\n\n new_trans = False\n for po_filename in po_files:\n new_msgstrs = {}\n for lang in languages[1:]:\n po_file_path = os.path.join(locale_root, lang,\n po_files_path, po_filename)\n if not os.path.exists(po_file_path):\n open(po_file_path, 'a').close()\n new_msgstrs[lang] = _get_new_msgstrs(po_file_path, msgids)\n\n if len(new_msgstrs[languages[1]].keys()) > 0:\n new_trans = True\n po_file_path = os.path.join(locale_root, languages[0],\n po_files_path, po_filename)\n _write_new_messages(po_file_path, trans_writer, meta_writer,\n msgids, new_msgstrs, languages)\n\n trans_writer.close()\n meta_writer.close()\n\n return new_trans"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef csv_to_po(trans_csv_path, meta_csv_path, locale_root,\n po_files_path, header=None):\n \"\"\"\n Converts GDocs spreadsheet generated csv file into po file.\n :param trans_csv_path: path to temporary file with translations\n :param meta_csv_path: path to temporary file with meta information\n :param locale_root: path to locale root folder containing directories\n with languages\n :param po_files_path: path from lang directory to po file\n \"\"\"\n pattern = \"^\\w+.*po$\"\n for root, dirs, files in os.walk(locale_root):\n for f in filter(lambda x: re.match(pattern, x), files):\n os.remove(os.path.join(root, f))\n\n # read title row and prepare descriptors for po files in each lang\n trans_reader = UnicodeReader(trans_csv_path)\n meta_reader = UnicodeReader(meta_csv_path)\n try:\n title_row = trans_reader.next()\n except StopIteration:\n # empty file\n return\n\n trans_languages = _prepare_locale_dirs(title_row[3:], locale_root)\n\n po_files = {}\n\n meta_reader.next()\n # go through every row in downloaded csv file\n for trans_row, meta_row in izip_longest(trans_reader, meta_reader):\n filename = trans_row[0].rstrip()\n metadata = meta_row[0].rstrip() if meta_row else METADATA_EMPTY\n comment = trans_row[1]\n msgid = trans_row[2]\n\n if filename not in po_files:\n _prepare_polib_files(po_files, filename, trans_languages,\n locale_root, po_files_path, header)\n\n _write_entries(po_files[filename], trans_languages, msgid,\n trans_row[3:], metadata, comment)\n for filename in po_files:\n for lang in po_files[filename]:\n po_files[filename][lang].save()\n\n trans_reader.close()\n meta_reader.close()", "response": "Converts GDocs generated csv file into po file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsending a message to a user", "response": "def send_notification(self, to=None, msg=None, label=None,\n title=None, uri=None):\n \"\"\" method to send a message to a user\n\n Parameters:\n to -> recipient\n msg -> message to send\n label -> application description\n title -> name of the notification event\n uri -> callback uri\n \"\"\"\n url = self.root_url + \"send_notification\"\n values = {}\n if to is not None:\n values[\"to\"] = to\n if msg is not None:\n values[\"msg\"] = msg\n if label is not None:\n values[\"label\"] = label\n if title is not None:\n values[\"title\"] = title\n if uri is not None:\n values[\"uri\"] = uri\n return self._query(url, values)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nquery method to do HTTP POST/GET Parameters: url -> the url to POST/GET data -> header_data as a dict (only for POST) Returns: Parsed JSON data as dict or None on error", "response": "def _query(self, url, data = None):\n \"\"\" query method to do HTTP POST/GET\n\n Parameters:\n url -> the url to POST/GET\n data -> header_data as a dict (only for POST)\n\n Returns:\n Parsed JSON data as dict\n or\n None on error\n \"\"\"\n auth = encodestring('%s:%s' % (self.user, self.secret)).replace('\\n', '')\n\n if data is not None: # we have POST data if there is data\n values = urllib.urlencode(data)\n request = urllib2.Request(url, values)\n request.add_header(\"Authorization\", \"Basic %s\" % auth)\n else: # do a GET otherwise\n request = urllib2.Request(url)\n request.add_header(\"Authorization\", \"Basic %s\" % auth)\n try:\n response = urllib2.urlopen(request)\n except IOError, e: # no connection\n return {\"status\" : \"error\",\n \"response_code\" : e.code,\n \"response_message\" : e.msg\n }\n return json.loads(response.read())"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfunctioning to init option parser", "response": "def init_parser():\n \"\"\" function to init option parser \"\"\"\n usage = \"usage: %prog -u user -s secret -n name [-l label] \\\n[-t title] [-c callback] [TEXT]\"\n\n parser = OptionParser(usage, version=\"%prog \" + notifo.__version__)\n parser.add_option(\"-u\", \"--user\", action=\"store\", dest=\"user\",\n help=\"your notifo username\")\n parser.add_option(\"-s\", \"--secret\", action=\"store\", dest=\"secret\",\n help=\"your notifo API secret\")\n parser.add_option(\"-n\", \"--name\", action=\"store\", dest=\"name\",\n help=\"recipient for the notification\")\n parser.add_option(\"-l\", \"--label\", action=\"store\", dest=\"label\",\n help=\"label for the notification\")\n parser.add_option(\"-t\", \"--title\", action=\"store\", dest=\"title\",\n help=\"title of the notification\")\n parser.add_option(\"-c\", \"--callback\", action=\"store\", dest=\"callback\",\n help=\"callback URL to call\")\n parser.add_option(\"-m\", \"--message\", action=\"store_true\", dest=\"message\",\n default=False, help=\"send message instead of notification\")\n\n (options, args) = parser.parse_args()\n return (parser, options, args)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun a python module.", "response": "def run_python_module(modulename, args):\n \"\"\"Run a python module, as though with ``python -m name args...``.\n\n `modulename` is the name of the module, possibly a dot-separated name.\n `args` is the argument array to present as sys.argv, including the first\n element naming the module being executed.\n\n \"\"\"\n openfile = None\n glo, loc = globals(), locals()\n try:\n try:\n # Search for the module - inside its parent package, if any - using\n # standard import mechanics.\n if '.' in modulename:\n packagename, name = rsplit1(modulename, '.')\n package = __import__(packagename, glo, loc, ['__path__'])\n searchpath = package.__path__\n else:\n packagename, name = None, modulename\n searchpath = None # \"top-level search\" in imp.find_module()\n openfile, pathname, _ = imp.find_module(name, searchpath)\n\n # Complain if this is a magic non-file module.\n if openfile is None and pathname is None:\n raise NoSource(\n \"module does not live in a file: %r\" % modulename\n )\n\n # If `modulename` is actually a package, not a mere module, then we\n # pretend to be Python 2.7 and try running its __main__.py script.\n if openfile is None:\n packagename = modulename\n name = '__main__'\n package = __import__(packagename, glo, loc, ['__path__'])\n searchpath = package.__path__\n openfile, pathname, _ = imp.find_module(name, searchpath)\n except ImportError:\n _, err, _ = sys.exc_info()\n raise NoSource(str(err))\n finally:\n if openfile:\n openfile.close()\n\n # Finally, hand the file off to run_python_file for execution.\n pathname = os.path.abspath(pathname)\n args[0] = pathname\n run_python_file(pathname, args, package=packagename)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run_python_file(filename, args, package=None):\n # Create a module to serve as __main__\n old_main_mod = sys.modules['__main__']\n main_mod = imp.new_module('__main__')\n sys.modules['__main__'] = main_mod\n main_mod.__file__ = filename\n if package:\n main_mod.__package__ = package\n main_mod.__builtins__ = BUILTINS\n\n # Set sys.argv properly.\n old_argv = sys.argv\n sys.argv = args\n\n try:\n # Make a code object somehow.\n if filename.endswith(\".pyc\") or filename.endswith(\".pyo\"):\n code = make_code_from_pyc(filename)\n else:\n code = make_code_from_py(filename)\n\n # Execute the code object.\n try:\n exec_code_object(code, main_mod.__dict__)\n except SystemExit:\n # The user called sys.exit(). Just pass it along to the upper\n # layers, where it will be handled.\n raise\n except:\n # Something went wrong while executing the user code.\n # Get the exc_info, and pack them into an exception that we can\n # throw up to the outer loop. We peel two layers off the traceback\n # so that the coverage.py code doesn't appear in the final printed\n # traceback.\n typ, err, tb = sys.exc_info()\n raise ExceptionDuringRun(typ, err, tb.tb_next.tb_next)\n finally:\n # Restore the old __main__\n sys.modules['__main__'] = old_main_mod\n\n # Restore the old argv and path\n sys.argv = old_argv", "response": "Runs a python file as if it were the main program on the command line."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget source from filename and make a code object of it.", "response": "def make_code_from_py(filename):\n \"\"\"Get source from `filename` and make a code object of it.\"\"\"\n # Open the source file.\n try:\n source_file = open_source(filename)\n except IOError:\n raise NoSource(\"No file to run: %r\" % filename)\n\n try:\n source = source_file.read()\n finally:\n source_file.close()\n\n # We have the source. `compile` still needs the last line to be clean,\n # so make sure it is, then compile a code object from it.\n if not source or source[-1] != '\\n':\n source += '\\n'\n code = compile(source, filename, \"exec\")\n\n return code"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting a code object from a. pyc file.", "response": "def make_code_from_pyc(filename):\n \"\"\"Get a code object from a .pyc file.\"\"\"\n try:\n fpyc = open(filename, \"rb\")\n except IOError:\n raise NoCode(\"No file to run: %r\" % filename)\n\n try:\n # First four bytes are a version-specific magic number. It has to\n # match or we won't run the file.\n magic = fpyc.read(4)\n if magic != imp.get_magic():\n raise NoCode(\"Bad magic number in .pyc file\")\n\n # Skip the junk in the header that we don't need.\n fpyc.read(4) # Skip the moddate.\n if sys.version_info >= (3, 3):\n # 3.3 added another long to the header (size), skip it.\n fpyc.read(4)\n\n # The rest of the file is the code object we want.\n code = marshal.load(fpyc)\n finally:\n fpyc.close()\n\n return code"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets current cursor position", "response": "def current(self, value):\n \"\"\"set current cursor position\"\"\"\n current = min(max(self._min, value), self._max)\n\n self._current = current\n\n if current > self._stop : \n self._stop = current\n self._start = current-self._width\n elif current < self._start : \n self._start = current\n self._stop = current + self._width\n\n if abs(self._start - self._min) <= self._sticky_lenght :\n self._start = self._min\n \n if abs(self._stop - self._max) <= self._sticky_lenght :\n self._stop = self._max"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef eventFilter(self, obj, event):\n if obj == self._text_edit:\n etype = event.type()\n if etype == QtCore.QEvent.KeyPress:\n key = event.key()\n if self._consecutive_tab == 0 and key in (QtCore.Qt.Key_Tab,):\n return False\n elif self._consecutive_tab == 1 and key in (QtCore.Qt.Key_Tab,):\n # ok , called twice, we grab focus, and show the cursor\n self._consecutive_tab = self._consecutive_tab+1\n self._update_list()\n return True\n elif self._consecutive_tab == 2:\n if key in (QtCore.Qt.Key_Return, QtCore.Qt.Key_Enter):\n self._complete_current()\n return True\n if key in (QtCore.Qt.Key_Tab,):\n self.select_right()\n self._update_list()\n return True\n elif key in ( QtCore.Qt.Key_Down,):\n self.select_down()\n self._update_list()\n return True\n elif key in (QtCore.Qt.Key_Right,):\n self.select_right()\n self._update_list()\n return True\n elif key in ( QtCore.Qt.Key_Up,):\n self.select_up()\n self._update_list()\n return True\n elif key in ( QtCore.Qt.Key_Left,):\n self.select_left()\n self._update_list()\n return True\n elif key in ( QtCore.Qt.Key_Escape,):\n self.cancel_completion()\n return True\n else :\n self.cancel_completion()\n else:\n self.cancel_completion()\n\n elif etype == QtCore.QEvent.FocusOut:\n self.cancel_completion()\n\n return super(CompletionHtml, self).eventFilter(obj, event)", "response": "Reimplemented to handle keyboard input and to auto - hide when the text edit loses focus."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncancels the completion of the current locale.", "response": "def cancel_completion(self):\n \"\"\"Cancel the completion\n\n should be called when the completer have to be dismissed\n\n This reset internal variable, clearing the temporary buffer\n of the console where the completion are shown.\n \"\"\"\n self._consecutive_tab = 0\n self._slice_start = 0\n self._console_widget._clear_temporary_buffer()\n self._index = (0, 0)\n if(self._sliding_interval):\n self._sliding_interval = None"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _select_index(self, row, col):\n\n nr, nc = self._size\n nr = nr-1\n nc = nc-1\n\n # case 1\n if (row > nr and col >= nc) or (row >= nr and col > nc):\n self._select_index(0, 0)\n # case 2\n elif (row <= 0 and col < 0) or (row < 0 and col <= 0):\n self._select_index(nr, nc)\n # case 3\n elif row > nr :\n self._select_index(0, col+1)\n # case 4\n elif row < 0 :\n self._select_index(nr, col-1)\n # case 5\n elif col > nc :\n self._select_index(row+1, 0)\n # case 6\n elif col < 0 :\n self._select_index(row-1, nc)\n elif 0 <= row and row <= nr and 0 <= col and col <= nc :\n self._index = (row, col)\n else :\n raise NotImplementedError(\"you'r trying to go where no completion\\\n have gone before : %d:%d (%d:%d)\"%(row, col, nr, nc) )", "response": "Change the selection index and make sure it stays in the right range."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nshow the completion widget with items at the position specified by cursor.", "response": "def show_items(self, cursor, items):\n \"\"\" Shows the completion widget with 'items' at the position specified\n by 'cursor'.\n \"\"\"\n if not items :\n return\n self._start_position = cursor.position()\n self._consecutive_tab = 1\n items_m, ci = text.compute_item_matrix(items, empty=' ')\n self._sliding_interval = SlidingInterval(len(items_m)-1)\n\n self._items = items_m\n self._size = (ci['rows_numbers'], ci['columns_numbers'])\n self._old_cursor = cursor\n self._index = (0, 0)\n sjoin = lambda x : [ y.ljust(w, ' ') for y, w in zip(x, ci['columns_width'])]\n self._justified_items = map(sjoin, items_m)\n self._update_list(hilight=False)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _update_list(self, hilight=True):\n self._sliding_interval.current = self._index[0]\n head = None\n foot = None\n if self._sliding_interval.start > 0 : \n head = '...'\n\n if self._sliding_interval.stop < self._sliding_interval._max:\n foot = '...'\n items_m = self._justified_items[\\\n self._sliding_interval.start:\\\n self._sliding_interval.stop+1\\\n ]\n\n self._console_widget._clear_temporary_buffer()\n if(hilight):\n sel = (self._sliding_interval.nth, self._index[1])\n else :\n sel = None\n\n strng = html_tableify(items_m, select=sel, header=head, footer=foot)\n self._console_widget._fill_temporary_buffer(self._old_cursor, strng, html=True)", "response": "update the list of completion and hilight the currently selected completion"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nperforms the completion with the currently selected item.", "response": "def _complete_current(self):\n \"\"\" Perform the completion with the currently selected item.\n \"\"\"\n i = self._index\n item = self._items[i[0]][i[1]]\n item = item.strip()\n if item :\n self._current_text_cursor().insertText(item)\n self.cancel_completion()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a dictionary of words and word counts in a string.", "response": "def wordfreq(text, is_filename=False):\n \"\"\"Return a dictionary of words and word counts in a string.\"\"\"\n if is_filename:\n with open(text) as f:\n text = f.read()\n freqs = {}\n for word in text.split():\n lword = word.lower()\n freqs[lword] = freqs.get(lword, 0) + 1\n return freqs"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprint the n most common words and counts in the freqs dict.", "response": "def print_wordfreq(freqs, n=10):\n \"\"\"Print the n most common words and counts in the freqs dict.\"\"\"\n \n words, counts = freqs.keys(), freqs.values()\n items = zip(counts, words)\n items.sort(reverse=True)\n for (count, word) in items[:n]:\n print(word, count)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the string representation of the job description XML.", "response": "def tostring(self):\n \"\"\"Return the string representation of the job description XML.\"\"\"\n root = self.as_element()\n indent(root)\n txt = ET.tostring(root, encoding=\"utf-8\")\n # Now remove the tokens used to order the attributes.\n txt = re.sub(r'_[A-Z]_','',txt)\n txt = '\\n' + txt\n return txt"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite the XML job description to a file.", "response": "def write(self, filename):\n \"\"\"Write the XML job description to a file.\"\"\"\n txt = self.tostring()\n with open(filename, 'w') as f:\n f.write(txt)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef validate_pin(pin):\n v = _Validator(schemas.pin)\n if v.validate(pin):\n return\n else:\n raise schemas.DocumentError(errors=v.errors)", "response": "Validate the given pin against the schema."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends a shared pin for the given topics.", "response": "def send_shared_pin(self, topics, pin, skip_validation=False):\n \"\"\"\n Send a shared pin for the given topics.\n\n :param list topics: The list of topics.\n :param dict pin: The pin.\n :param bool skip_validation: Whether to skip the validation.\n :raises pypebbleapi.schemas.DocumentError: If the validation process failed.\n :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.\n \"\"\"\n if not self.api_key:\n raise ValueError(\"You need to specify an api_key.\")\n if not skip_validation:\n validate_pin(pin)\n\n response = _request('PUT',\n url=self.url_v1('/shared/pins/' + pin['id']),\n user_agent=self.user_agent,\n api_key=self.api_key,\n topics_list=topics,\n json=pin,\n )\n _raise_for_status(response)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndeletes a shared pin.", "response": "def delete_shared_pin(self, pin_id):\n \"\"\"\n Delete a shared pin.\n\n :param str pin_id: The id of the pin to delete.\n :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.\n \"\"\"\n if not self.api_key:\n raise ValueError(\"You need to specify an api_key.\")\n\n response = _request('DELETE',\n url=self.url_v1('/shared/pins/' + pin_id),\n user_agent=self.user_agent,\n api_key=self.api_key,\n )\n _raise_for_status(response)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef send_user_pin(self, user_token, pin, skip_validation=False):\n if not skip_validation:\n validate_pin(pin)\n\n response = _request('PUT',\n url=self.url_v1('/user/pins/' + pin['id']),\n user_agent=self.user_agent,\n user_token=user_token,\n json=pin,\n )\n _raise_for_status(response)", "response": "Send a user pin."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndeletes a user pin.", "response": "def delete_user_pin(self, user_token, pin_id):\n \"\"\"\n Delete a user pin.\n\n :param str user_token: The token of the user.\n :param str pin_id: The id of the pin to delete.\n :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.\n \"\"\"\n\n response = _request('DELETE',\n url=self.url_v1('/user/pins/' + pin_id),\n user_agent=self.user_agent,\n user_token=user_token,\n )\n _raise_for_status(response)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef subscribe(self, user_token, topic):\n response = _request('POST',\n url=self.url_v1('/user/subscriptions/' + topic),\n user_agent=self.user_agent,\n user_token=user_token,\n )\n _raise_for_status(response)", "response": "Subscribe a user to a topic."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the list of the topics which a user is subscribed to.", "response": "def list_subscriptions(self, user_token):\n \"\"\"\n Get the list of the topics which a user is subscribed to.\n\n :param str user_token: The token of the user.\n :return: The list of the topics.\n :rtype: list\n :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.\n \"\"\"\n response = _request('GET',\n url=self.url_v1('/user/subscriptions'),\n user_agent=self.user_agent,\n user_token=user_token,\n )\n _raise_for_status(response)\n\n return response.json()['topics']"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef monitored(total: int, name=None, message=None):\n def decorator(f):\n nonlocal name\n monitor_index = list(inspect.signature(f).parameters.keys()).index('monitor')\n if name is None:\n name=f.__name__\n @wraps(f)\n def wrapper(*args, **kargs):\n if len(args) > monitor_index:\n monitor = args[monitor_index]\n elif 'monitor' in kargs:\n monitor = kargs['monitor']\n else:\n monitor = kargs['monitor'] = NullMonitor()\n with monitor.task(total, name, message):\n f(*args, **kargs)\n return wrapper\n return decorator", "response": "Decorator to automatically begin and end a task on the progressmonitor.\n "} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalls before starting work on a monitor", "response": "def begin(self, total: int, name=None, message=None):\n \"\"\"Call before starting work on a monitor, specifying name and amount of work\"\"\"\n self.total = total\n message = message or name or \"Working...\"\n self.name = name or \"ProgressMonitor\"\n self.update(0, message)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwrap code into a begin and end call on this monitor", "response": "def task(self, total: int, name=None, message=None):\n \"\"\"Wrap code into a begin and end call on this monitor\"\"\"\n self.begin(total, name, message)\n try:\n yield self\n finally:\n self.done()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef subtask(self, units: int):\n sm = self.submonitor(units)\n try:\n yield sm\n finally:\n if sm.total is None:\n # begin was never called, so the subtask cannot be closed\n self.update(units)\n else:\n sm.done()", "response": "Create a submonitor with the given units"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the percentage of work done including submonitors.", "response": "def progress(self)-> float:\n \"\"\"What percentage (range 0-1) of work is done (including submonitors)\"\"\"\n if self.total is None:\n return 0\n my_progress = self.worked\n my_progress += sum(s.progress * weight\n for (s, weight) in self.sub_monitors.items())\n return min(1, my_progress / self.total)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update(self, units: int=1, message: str=None):\n if self.total is None:\n raise Exception(\"Cannot call progressmonitor.update before calling begin\")\n self.worked = min(self.total, self.worked+units)\n if message:\n self.message = message\n for listener in self.listeners:\n listener(self)", "response": "Increment the monitor with N units worked and an optional message"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef submonitor(self, units: int, *args, **kargs) -> 'ProgressMonitor':\n submonitor = ProgressMonitor(*args, **kargs)\n self.sub_monitors[submonitor] = units\n submonitor.add_listener(self._submonitor_update)\n return submonitor", "response": "Create a sub monitor that stands for N units of work in this monitor\n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsignal that this task is done.", "response": "def done(self, message: str=None):\n \"\"\"\n Signal that this task is done.\n This is completely optional and will just call .update with the remaining work.\n \"\"\"\n if message is None:\n message = \"{self.name} done\".format(**locals()) if self.name else \"Done\"\n self.update(units=self.total - self.worked, message=message)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprint a string through a pager and return a new object.", "response": "def page(strng, start=0, screen_lines=0, pager_cmd=None,\n html=None, auto_html=False):\n \"\"\"Print a string, piping through a pager.\n\n This version ignores the screen_lines and pager_cmd arguments and uses\n IPython's payload system instead.\n\n Parameters\n ----------\n strng : str\n Text to page.\n\n start : int\n Starting line at which to place the display.\n \n html : str, optional\n If given, an html string to send as well.\n\n auto_html : bool, optional\n If true, the input string is assumed to be valid reStructuredText and is\n converted to HTML with docutils. Note that if docutils is not found,\n this option is silently ignored.\n\n Note\n ----\n\n Only one of the ``html`` and ``auto_html`` options can be given, not\n both.\n \"\"\"\n\n # Some routines may auto-compute start offsets incorrectly and pass a\n # negative value. Offset to 0 for robustness.\n start = max(0, start)\n shell = InteractiveShell.instance()\n\n if auto_html:\n try:\n # These defaults ensure user configuration variables for docutils\n # are not loaded, only our config is used here.\n defaults = {'file_insertion_enabled': 0,\n 'raw_enabled': 0,\n '_disable_config': 1}\n html = publish_string(strng, writer_name='html',\n settings_overrides=defaults)\n except:\n pass\n \n payload = dict(\n source='IPython.zmq.page.page',\n text=strng,\n html=html,\n start_line_number=start\n )\n shell.payload_manager.write_payload(payload)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef from_line(cls, name, comes_from=None, isolated=False):\n from pip.index import Link\n\n url = None\n if is_url(name):\n marker_sep = '; '\n else:\n marker_sep = ';'\n if marker_sep in name:\n name, markers = name.split(marker_sep, 1)\n markers = markers.strip()\n if not markers:\n markers = None\n else:\n markers = None\n name = name.strip()\n req = None\n path = os.path.normpath(os.path.abspath(name))\n link = None\n\n if is_url(name):\n link = Link(name)\n elif (os.path.isdir(path)\n and (os.path.sep in name or name.startswith('.'))):\n if not is_installable_dir(path):\n raise InstallationError(\n \"Directory %r is not installable. File 'setup.py' not \"\n \"found.\" % name\n )\n link = Link(path_to_url(name))\n elif is_archive_file(path):\n if not os.path.isfile(path):\n logger.warning(\n 'Requirement %r looks like a filename, but the file does '\n 'not exist',\n name\n )\n link = Link(path_to_url(name))\n\n # it's a local file, dir, or url\n if link:\n\n url = link.url\n # Handle relative file URLs\n if link.scheme == 'file' and re.search(r'\\.\\./', url):\n url = path_to_url(os.path.normpath(os.path.abspath(link.path)))\n\n # wheel file\n if link.ext == wheel_ext:\n wheel = Wheel(link.filename) # can raise InvalidWheelFilename\n if not wheel.supported():\n raise UnsupportedWheel(\n \"%s is not a supported wheel on this platform.\" %\n wheel.filename\n )\n req = \"%s==%s\" % (wheel.name, wheel.version)\n else:\n # set the req to the egg fragment. when it's not there, this\n # will become an 'unnamed' requirement\n req = link.egg_fragment\n\n # a requirement specifier\n else:\n req = name\n\n return cls(req, comes_from, url=url, markers=markers,\n isolated=isolated)", "response": "Creates an InstallRequirement object from a line of text."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_pyconfig_files(config_files, path):\n config = Config()\n for cf in config_files:\n loader = PyFileConfigLoader(cf, path=path)\n try:\n next_config = loader.load_config()\n except ConfigFileNotFound:\n pass\n except:\n raise\n else:\n config._merge(next_config)\n return config", "response": "Load multiple Python config files into a single config object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nload the config from a file and return it as a Struct.", "response": "def load_config(self):\n \"\"\"Load the config from a file and return it as a Struct.\"\"\"\n self.clear()\n try:\n self._find_file()\n except IOError as e:\n raise ConfigFileNotFound(str(e))\n self._read_file_as_dict()\n self._convert_to_config()\n return self.config"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads the config file into self. config with recursive loading.", "response": "def _read_file_as_dict(self):\n \"\"\"Load the config file into self.config, with recursive loading.\"\"\"\n # This closure is made available in the namespace that is used\n # to exec the config file. It allows users to call\n # load_subconfig('myconfig.py') to load config files recursively.\n # It needs to be a closure because it has references to self.path\n # and self.config. The sub-config is loaded with the same path\n # as the parent, but it uses an empty config which is then merged\n # with the parents.\n\n # If a profile is specified, the config file will be loaded\n # from that profile\n\n def load_subconfig(fname, profile=None):\n # import here to prevent circular imports\n from IPython.core.profiledir import ProfileDir, ProfileDirError\n if profile is not None:\n try:\n profile_dir = ProfileDir.find_profile_dir_by_name(\n get_ipython_dir(),\n profile,\n )\n except ProfileDirError:\n return\n path = profile_dir.location\n else:\n path = self.path\n loader = PyFileConfigLoader(fname, path)\n try:\n sub_config = loader.load_config()\n except ConfigFileNotFound:\n # Pass silently if the sub config is not there. This happens\n # when a user s using a profile, but not the default config.\n pass\n else:\n self.config._merge(sub_config)\n\n # Again, this needs to be a closure and should be used in config\n # files to get the config being loaded.\n def get_config():\n return self.config\n\n namespace = dict(load_subconfig=load_subconfig, get_config=get_config)\n fs_encoding = sys.getfilesystemencoding() or 'ascii'\n conf_filename = self.full_filename.encode(fs_encoding)\n py3compat.execfile(conf_filename, namespace)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nexecute the config string.", "response": "def _exec_config_str(self, lhs, rhs):\n \"\"\"execute self.config. = \n \n * expands ~ with expanduser\n * tries to assign with raw eval, otherwise assigns with just the string,\n allowing `--C.a=foobar` and `--C.a=\"foobar\"` to be equivalent. *Not*\n equivalent are `--C.a=4` and `--C.a='4'`.\n \"\"\"\n rhs = os.path.expanduser(rhs)\n try:\n # Try to see if regular Python syntax will work. This\n # won't handle strings as the quote marks are removed\n # by the system shell.\n value = eval(rhs)\n except (NameError, SyntaxError):\n # This case happens if the rhs is a string.\n value = rhs\n\n exec u'self.config.%s = value' % lhs"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates self. config from a flag which can be a dict or Config", "response": "def _load_flag(self, cfg):\n \"\"\"update self.config from a flag, which can be a dict or Config\"\"\"\n if isinstance(cfg, (dict, Config)):\n # don't clobber whole config sections, update\n # each section from config:\n for sec,c in cfg.iteritems():\n self.config[sec].update(c)\n else:\n raise TypeError(\"Invalid flag: %r\" % cfg)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndecodes argv if bytes using stin. encoding falling back on default enc", "response": "def _decode_argv(self, argv, enc=None):\n \"\"\"decode argv if bytes, using stin.encoding, falling back on default enc\"\"\"\n uargv = []\n if enc is None:\n enc = DEFAULT_ENCODING\n for arg in argv:\n if not isinstance(arg, unicode):\n # only decode if not already decoded\n arg = arg.decode(enc)\n uargv.append(arg)\n return uargv"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_config(self, argv=None, aliases=None, flags=None):\n from IPython.config.configurable import Configurable\n\n self.clear()\n if argv is None:\n argv = self.argv\n if aliases is None:\n aliases = self.aliases\n if flags is None:\n flags = self.flags\n\n # ensure argv is a list of unicode strings:\n uargv = self._decode_argv(argv)\n for idx,raw in enumerate(uargv):\n # strip leading '-'\n item = raw.lstrip('-')\n\n if raw == '--':\n # don't parse arguments after '--'\n # this is useful for relaying arguments to scripts, e.g.\n # ipython -i foo.py --pylab=qt -- args after '--' go-to-foo.py\n self.extra_args.extend(uargv[idx+1:])\n break\n\n if kv_pattern.match(raw):\n lhs,rhs = item.split('=',1)\n # Substitute longnames for aliases.\n if lhs in aliases:\n lhs = aliases[lhs]\n if '.' not in lhs:\n # probably a mistyped alias, but not technically illegal\n warn.warn(\"Unrecognized alias: '%s', it will probably have no effect.\"%lhs)\n try:\n self._exec_config_str(lhs, rhs)\n except Exception:\n raise ArgumentError(\"Invalid argument: '%s'\" % raw)\n\n elif flag_pattern.match(raw):\n if item in flags:\n cfg,help = flags[item]\n self._load_flag(cfg)\n else:\n raise ArgumentError(\"Unrecognized flag: '%s'\"%raw)\n elif raw.startswith('-'):\n kv = '--'+item\n if kv_pattern.match(kv):\n raise ArgumentError(\"Invalid argument: '%s', did you mean '%s'?\"%(raw, kv))\n else:\n raise ArgumentError(\"Invalid argument: '%s'\"%raw)\n else:\n # keep all args that aren't valid in a list,\n # in case our parent knows what to do with them.\n self.extra_args.append(item)\n return self.config", "response": "Parse the configuration and generate a Config object."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_config(self, argv=None, aliases=None, flags=None):\n self.clear()\n if argv is None:\n argv = self.argv\n if aliases is None:\n aliases = self.aliases\n if flags is None:\n flags = self.flags\n self._create_parser(aliases, flags)\n self._parse_args(argv)\n self._convert_to_config()\n return self.config", "response": "Parse command line arguments and return as a Config object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing command - line arguments.", "response": "def _parse_args(self, args):\n \"\"\"self.parser->self.parsed_data\"\"\"\n # decode sys.argv to support unicode command-line options\n enc = DEFAULT_ENCODING\n uargs = [py3compat.cast_unicode(a, enc) for a in args]\n self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _convert_to_config(self):\n for k, v in vars(self.parsed_data).iteritems():\n exec \"self.config.%s = v\"%k in locals(), globals()", "response": "Convert the parsed data to config."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _convert_to_config(self):\n # remove subconfigs list from namespace before transforming the Namespace\n if '_flags' in self.parsed_data:\n subcs = self.parsed_data._flags\n del self.parsed_data._flags\n else:\n subcs = []\n\n for k, v in vars(self.parsed_data).iteritems():\n if v is None:\n # it was a flag that shares the name of an alias\n subcs.append(self.alias_flags[k])\n else:\n # eval the KV assignment\n self._exec_config_str(k, v)\n\n for subc in subcs:\n self._load_flag(subc)\n\n if self.extra_args:\n sub_parser = KeyValueConfigLoader()\n sub_parser.load_config(self.extra_args)\n self.config._merge(sub_parser.config)\n self.extra_args = sub_parser.extra_args", "response": "Convert the parsed_data to the config."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfinds a module in sys. path and return the full path of the module.", "response": "def find_module(name, path=None):\n \"\"\"imp.find_module variant that only return path of module.\n \n The `imp.find_module` returns a filehandle that we are not interested in.\n Also we ignore any bytecode files that `imp.find_module` finds.\n\n Parameters\n ----------\n name : str\n name of module to locate\n path : list of str\n list of paths to search for `name`. If path=None then search sys.path\n\n Returns\n -------\n filename : str\n Return full path of module or None if module is missing or does not have\n .py or .pyw extension\n \"\"\"\n if name is None:\n return None\n try:\n file, filename, _ = imp.find_module(name, path)\n except ImportError:\n return None\n if file is None:\n return filename\n else:\n file.close()\n if os.path.splitext(filename)[1] in [\".py\", \"pyc\"]:\n return filename\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_init(dirname):\n fbase = os.path.join(dirname, \"__init__\")\n for ext in [\".py\", \".pyw\"]:\n fname = fbase + ext\n if os.path.isfile(fname):\n return fname", "response": "Get the path to the __init__ file for the specified module directory."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find_mod(module_name):\n parts = module_name.split(\".\")\n basepath = find_module(parts[0])\n for submodname in parts[1:]:\n basepath = find_module(submodname, [basepath])\n if basepath and os.path.isdir(basepath):\n basepath = get_init(basepath)\n return basepath", "response": "Find module module_name on sys. path\n \n Return the path to module or None if module is missing or does not have a. py or. pyw\n extension."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nregistering a callback to be called with this Launcher s stop_data when the process actually finishes.", "response": "def on_stop(self, f):\n \"\"\"Register a callback to be called with this Launcher's stop_data\n when the process actually finishes.\n \"\"\"\n if self.state=='after':\n return f(self.stop_data)\n else:\n self.stop_callbacks.append(f)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef notify_start(self, data):\n\n self.log.debug('Process %r started: %r', self.args[0], data)\n self.start_data = data\n self.state = 'running'\n return data", "response": "Called by the daemon when the process starts."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalling this to trigger process stop actions.", "response": "def notify_stop(self, data):\n \"\"\"Call this to trigger process stop actions.\n\n This logs the process stopping and sets the state to 'after'. Call\n this to trigger callbacks registered via :meth:`on_stop`.\"\"\"\n\n self.log.debug('Process %r stopped: %r', self.args[0], data)\n self.stop_data = data\n self.state = 'after'\n for i in range(len(self.stop_callbacks)):\n d = self.stop_callbacks.pop()\n d(data)\n return data"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsend INT wait a delay and then send KILL.", "response": "def interrupt_then_kill(self, delay=2.0):\n \"\"\"Send INT, wait a delay and then send KILL.\"\"\"\n try:\n self.signal(SIGINT)\n except Exception:\n self.log.debug(\"interrupt failed\")\n pass\n self.killer = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)\n self.killer.start()"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting n engines by profile or profile_dir.", "response": "def start(self, n):\n \"\"\"Start n engines by profile or profile_dir.\"\"\"\n dlist = []\n for i in range(n):\n if i > 0:\n time.sleep(self.delay)\n el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log,\n profile_dir=self.profile_dir, cluster_id=self.cluster_id,\n )\n\n # Copy the engine args over to each engine launcher.\n el.engine_cmd = copy.deepcopy(self.engine_cmd)\n el.engine_args = copy.deepcopy(self.engine_args)\n el.on_stop(self._notice_engine_stopped)\n d = el.start()\n self.launchers[i] = el\n dlist.append(d)\n self.notify_start(dlist)\n return dlist"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find_args(self):\n return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \\\n self.program + self.program_args", "response": "Build self. args using all the fields."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef start(self, n):\n self.n = n\n return super(MPILauncher, self).start()", "response": "Start n instances of the program using mpiexec."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef start(self, n):\n self.n = n\n return super(MPIEngineSetLauncher, self).start(n)", "response": "Start n engines by profile or profile_dir."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsend a single file to the remote host", "response": "def _send_file(self, local, remote):\n \"\"\"send a single file\"\"\"\n remote = \"%s:%s\" % (self.location, remote)\n for i in range(10):\n if not os.path.exists(local):\n self.log.debug(\"waiting for %s\" % local)\n time.sleep(1)\n else:\n break\n self.log.info(\"sending %s to %s\", local, remote)\n check_output(self.scp_cmd + [local, remote])"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef send_files(self):\n if not self.to_send:\n return\n for local_file, remote_file in self.to_send:\n self._send_file(local_file, remote_file)", "response": "send our files (called before start)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfetches a single file from a remote host", "response": "def _fetch_file(self, remote, local):\n \"\"\"fetch a single file\"\"\"\n full_remote = \"%s:%s\" % (self.location, remote)\n self.log.info(\"fetching %s from %s\", local, full_remote)\n for i in range(10):\n # wait up to 10s for remote file to exist\n check = check_output(self.ssh_cmd + self.ssh_args + \\\n [self.location, 'test -e', remote, \"&& echo 'yes' || echo 'no'\"])\n check = check.strip()\n if check == 'no':\n time.sleep(1)\n elif check == 'yes':\n break\n check_output(self.scp_cmd + [full_remote, local])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfetch remote files ( called after start", "response": "def fetch_files(self):\n \"\"\"fetch remote files (called after start)\"\"\"\n if not self.to_fetch:\n return\n for remote_file, local_file in self.to_fetch:\n self._fetch_file(remote_file, local_file)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _remote_profile_dir_default(self):\n home = get_home_dir()\n if not home.endswith('/'):\n home = home+'/'\n \n if self.profile_dir.startswith(home):\n return self.profile_dir[len(home):]\n else:\n return self.profile_dir", "response": "turns self. profile_dir into. ipython"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndetermining engine count from engines dict", "response": "def engine_count(self):\n \"\"\"determine engine count from `engines` dict\"\"\"\n count = 0\n for n in self.engines.itervalues():\n if isinstance(n, (tuple,list)):\n n,args = n\n count += n\n return count"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef start(self, n):\n\n dlist = []\n for host, n in self.engines.iteritems():\n if isinstance(n, (tuple, list)):\n n, args = n\n else:\n args = copy.deepcopy(self.engine_args)\n\n if '@' in host:\n user,host = host.split('@',1)\n else:\n user=None\n for i in range(n):\n if i > 0:\n time.sleep(self.delay)\n el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log,\n profile_dir=self.profile_dir, cluster_id=self.cluster_id,\n )\n if i > 0:\n # only send files for the first engine on each host\n el.to_send = []\n\n # Copy the engine args over to each engine launcher.\n el.engine_cmd = self.engine_cmd\n el.engine_args = args\n el.on_stop(self._notice_engine_stopped)\n d = el.start(user=user, hostname=host)\n self.launchers[ \"%s/%i\" % (host,i) ] = el\n dlist.append(d)\n self.notify_start(dlist)\n return dlist", "response": "Start engines by profile or profile_dir."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nstarts n copies of the process using the Win HPC job scheduler.", "response": "def start(self, n):\n \"\"\"Start n copies of the process using the Win HPC job scheduler.\"\"\"\n self.write_job_file(n)\n args = [\n 'submit',\n '/jobfile:%s' % self.job_file,\n '/scheduler:%s' % self.scheduler\n ]\n self.log.debug(\"Starting Win HPC Job: %s\" % (self.job_cmd + ' ' + ' '.join(args),))\n\n output = check_output([self.job_cmd]+args,\n env=os.environ,\n cwd=self.work_dir,\n stderr=STDOUT\n )\n job_id = self.parse_job_id(output)\n self.notify_start(job_id)\n return job_id"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nloading the default context with the default values for the basic keys", "response": "def _context_default(self):\n \"\"\"load the default context with the default values for the basic keys\n\n because the _trait_changed methods only load the context if they\n are set to something other than the default value.\n \"\"\"\n return dict(n=1, queue=u'', profile_dir=u'', cluster_id=u'')"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntaking the output of the submit command and return the job id.", "response": "def parse_job_id(self, output):\n \"\"\"Take the output of the submit command and return the job id.\"\"\"\n m = self.job_id_regexp.search(output)\n if m is not None:\n job_id = m.group()\n else:\n raise LauncherError(\"Job id couldn't be determined: %s\" % output)\n self.job_id = job_id\n self.log.info('Job submitted with job id: %r', job_id)\n return job_id"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nstarting n copies of the process using a batch system.", "response": "def start(self, n):\n \"\"\"Start n copies of the process using a batch system.\"\"\"\n self.log.debug(\"Starting %s: %r\", self.__class__.__name__, self.args)\n # Here we save profile_dir in the context so they\n # can be used in the batch script template as {profile_dir}\n self.write_batch_script(n)\n output = check_output(self.args, env=os.environ)\n\n job_id = self.parse_job_id(output)\n self.notify_start(job_id)\n return job_id"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nstarts n copies of the process using LSF batch system.", "response": "def start(self, n):\n \"\"\"Start n copies of the process using LSF batch system.\n This cant inherit from the base class because bsub expects\n to be piped a shell script in order to honor the #BSUB directives :\n bsub < script\n \"\"\"\n # Here we save profile_dir in the context so they\n # can be used in the batch script template as {profile_dir}\n self.write_batch_script(n)\n #output = check_output(self.args, env=os.environ)\n piped_cmd = self.args[0]+'<\\\"'+self.args[1]+'\\\"'\n self.log.debug(\"Starting %s: %s\", self.__class__.__name__, piped_cmd)\n p = Popen(piped_cmd, shell=True,env=os.environ,stdout=PIPE)\n output,err = p.communicate()\n job_id = self.parse_job_id(output)\n self.notify_start(job_id)\n return job_id"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef check_filemode(filepath, mode):\n\n filemode = stat.S_IMODE(os.stat(filepath).st_mode)\n return (oct(filemode) == mode)", "response": "Check if file is in octal mode."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _context_menu_make(self, pos):\n format = self._control.cursorForPosition(pos).charFormat()\n name = format.stringProperty(QtGui.QTextFormat.ImageName)\n if name:\n menu = QtGui.QMenu()\n\n menu.addAction('Copy Image', lambda: self._copy_image(name))\n menu.addAction('Save Image As...', lambda: self._save_image(name))\n menu.addSeparator()\n\n svg = self._name_to_svg_map.get(name, None)\n if svg is not None:\n menu.addSeparator()\n menu.addAction('Copy SVG', lambda: svg_to_clipboard(svg))\n menu.addAction('Save SVG As...',\n lambda: save_svg(svg, self._control))\n else:\n menu = super(RichIPythonWidget, self)._context_menu_make(pos)\n return menu", "response": "Reimplemented to return a custom context menu for images."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nappend the prompt and make the output nicer", "response": "def _pre_image_append(self, msg, prompt_number):\n \"\"\" Append the Out[] prompt and make the output nicer\n\n Shared code for some the following if statement\n \"\"\"\n self.log.debug(\"pyout: %s\", msg.get('content', ''))\n self._append_plain_text(self.output_sep, True)\n self._append_html(self._make_out_prompt(prompt_number), True)\n self._append_plain_text('\\n', True)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _handle_pyout(self, msg):\n if not self._hidden and self._is_from_this_session(msg):\n content = msg['content']\n prompt_number = content.get('execution_count', 0)\n data = content['data']\n if data.has_key('image/svg+xml'):\n self._pre_image_append(msg, prompt_number)\n self._append_svg(data['image/svg+xml'], True)\n self._append_html(self.output_sep2, True)\n elif data.has_key('image/png'):\n self._pre_image_append(msg, prompt_number)\n self._append_png(decodestring(data['image/png'].encode('ascii')), True)\n self._append_html(self.output_sep2, True)\n elif data.has_key('image/jpeg') and self._jpg_supported:\n self._pre_image_append(msg, prompt_number)\n self._append_jpg(decodestring(data['image/jpeg'].encode('ascii')), True)\n self._append_html(self.output_sep2, True)\n else:\n # Default back to the plain text representation.\n return super(RichIPythonWidget, self)._handle_pyout(msg)", "response": "Override to handle rich data types like SVG and JPEG."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _handle_display_data(self, msg):\n if not self._hidden and self._is_from_this_session(msg):\n source = msg['content']['source']\n data = msg['content']['data']\n metadata = msg['content']['metadata']\n # Try to use the svg or html representations.\n # FIXME: Is this the right ordering of things to try?\n if data.has_key('image/svg+xml'):\n self.log.debug(\"display: %s\", msg.get('content', ''))\n svg = data['image/svg+xml']\n self._append_svg(svg, True)\n elif data.has_key('image/png'):\n self.log.debug(\"display: %s\", msg.get('content', ''))\n # PNG data is base64 encoded as it passes over the network\n # in a JSON structure so we decode it.\n png = decodestring(data['image/png'].encode('ascii'))\n self._append_png(png, True)\n elif data.has_key('image/jpeg') and self._jpg_supported:\n self.log.debug(\"display: %s\", msg.get('content', ''))\n jpg = decodestring(data['image/jpeg'].encode('ascii'))\n self._append_jpg(jpg, True)\n else:\n # Default back to the plain text representation.\n return super(RichIPythonWidget, self)._handle_display_data(msg)", "response": "Override to handle rich data types like SVG and HTML."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _append_jpg(self, jpg, before_prompt=False):\n self._append_custom(self._insert_jpg, jpg, before_prompt)", "response": "Append raw JPG data to the widget."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _append_png(self, png, before_prompt=False):\n self._append_custom(self._insert_png, png, before_prompt)", "response": "Append raw PNG data to the widget."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nappends raw SVG data to the widget.", "response": "def _append_svg(self, svg, before_prompt=False):\n \"\"\" Append raw SVG data to the widget.\n \"\"\"\n self._append_custom(self._insert_svg, svg, before_prompt)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds the specified QImage to the document and returns a QTextImageFormat that references it.", "response": "def _add_image(self, image):\n \"\"\" Adds the specified QImage to the document and returns a\n QTextImageFormat that references it.\n \"\"\"\n document = self._control.document()\n name = str(image.cacheKey())\n document.addResource(QtGui.QTextDocument.ImageResource,\n QtCore.QUrl(name), image)\n format = QtGui.QTextImageFormat()\n format.setName(name)\n return format"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncopying the ImageResource with name to the clipboard.", "response": "def _copy_image(self, name):\n \"\"\" Copies the ImageResource with 'name' to the clipboard.\n \"\"\"\n image = self._get_image(name)\n QtGui.QApplication.clipboard().setImage(image)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the QImage stored as the ImageResource with name.", "response": "def _get_image(self, name):\n \"\"\" Returns the QImage stored as the ImageResource with 'name'.\n \"\"\"\n document = self._control.document()\n image = document.resource(QtGui.QTextDocument.ImageResource,\n QtCore.QUrl(name))\n return image"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_image_tag(self, match, path = None, format = \"png\"):\n if format in (\"png\",\"jpg\"):\n try:\n image = self._get_image(match.group(\"name\"))\n except KeyError:\n return \"Couldn't find image %s\" % match.group(\"name\")\n\n if path is not None:\n if not os.path.exists(path):\n os.mkdir(path)\n relpath = os.path.basename(path)\n if image.save(\"%s/qt_img%s.%s\" % (path, match.group(\"name\"), format),\n \"PNG\"):\n return '' % (relpath,\n match.group(\"name\"),format)\n else:\n return \"Couldn't save image!\"\n else:\n ba = QtCore.QByteArray()\n buffer_ = QtCore.QBuffer(ba)\n buffer_.open(QtCore.QIODevice.WriteOnly)\n image.save(buffer_, format.upper())\n buffer_.close()\n return '' % (\n format,re.sub(r'(.{60})',r'\\1\\n',str(ba.toBase64())))\n\n elif format == \"svg\":\n try:\n svg = str(self._name_to_svg_map[match.group(\"name\")])\n except KeyError:\n if not self._svg_warning_displayed:\n QtGui.QMessageBox.warning(self, 'Error converting PNG to SVG.',\n 'Cannot convert a PNG to SVG. To fix this, add this '\n 'to your ipython config:\\n\\n'\n '\\tc.InlineBackendConfig.figure_format = \\'svg\\'\\n\\n'\n 'And regenerate the figures.',\n QtGui.QMessageBox.Ok)\n self._svg_warning_displayed = True\n return (\"Cannot convert a PNG to SVG. \"\n \"To fix this, add this to your config: \"\n \"c.InlineBackendConfig.figure_format = 'svg' \"\n \"and regenerate the figures.\")\n\n # Not currently checking path, because it's tricky to find a\n # cross-browser way to embed external SVG images (e.g., via\n # object or embed tags).\n\n # Chop stand-alone header from matplotlib SVG\n offset = svg.find(\" -1)\n\n return svg[offset:]\n\n else:\n return 'Unrecognized image format'", "response": "Returns an HTML image tag given by a match."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _insert_img(self, cursor, img, fmt):\n try:\n image = QtGui.QImage()\n image.loadFromData(img, fmt.upper())\n except ValueError:\n self._insert_plain_text(cursor, 'Received invalid %s data.'%fmt)\n else:\n format = self._add_image(image)\n cursor.insertBlock()\n cursor.insertImage(format)\n cursor.insertBlock()", "response": "insert a raw image jpg or png"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _insert_svg(self, cursor, svg):\n try:\n image = svg_to_image(svg)\n except ValueError:\n self._insert_plain_text(cursor, 'Received invalid SVG data.')\n else:\n format = self._add_image(image)\n self._name_to_svg_map[format.name()] = svg\n cursor.insertBlock()\n cursor.insertImage(format)\n cursor.insertBlock()", "response": "Insert SVG data into the widet."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _save_image(self, name, format='PNG'):\n dialog = QtGui.QFileDialog(self._control, 'Save Image')\n dialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)\n dialog.setDefaultSuffix(format.lower())\n dialog.setNameFilter('%s file (*.%s)' % (format, format.lower()))\n if dialog.exec_():\n filename = dialog.selectedFiles()[0]\n image = self._get_image(name)\n image.save(filename, format)", "response": "Shows a save dialog for the ImageResource with name."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef safe_unicode(e):\n try:\n return unicode(e)\n except UnicodeError:\n pass\n\n try:\n return py3compat.str_to_unicode(str(e))\n except UnicodeError:\n pass\n\n try:\n return py3compat.str_to_unicode(repr(e))\n except UnicodeError:\n pass\n\n return u'Unrecoverably corrupt evalue'", "response": "Unicode function that can be used to call unicode on a node."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstop eventloop when exit_now fires", "response": "def _exit_now_changed(self, name, old, new):\n \"\"\"stop eventloop when exit_now fires\"\"\"\n if new:\n loop = ioloop.IOLoop.instance()\n loop.add_timeout(time.time()+0.1, loop.stop)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef init_environment(self):\n env = os.environ\n # These two ensure 'ls' produces nice coloring on BSD-derived systems\n env['TERM'] = 'xterm-color'\n env['CLICOLOR'] = '1'\n # Since normal pagers don't work at all (over pexpect we don't have\n # single-key control of the subprocess), try to disable paging in\n # subprocesses as much as possible.\n env['PAGER'] = 'cat'\n env['GIT_PAGER'] = 'cat'\n \n # And install the payload version of page.\n install_payload_page()", "response": "Configure the user s environment."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef auto_rewrite_input(self, cmd):\n new = self.prompt_manager.render('rewrite') + cmd\n payload = dict(\n source='IPython.zmq.zmqshell.ZMQInteractiveShell.auto_rewrite_input',\n transformed_input=new,\n )\n self.payload_manager.write_payload(payload)", "response": "Called to show the auto - rewritten input for autocall and friends."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nengaging the exit actions.", "response": "def ask_exit(self):\n \"\"\"Engage the exit actions.\"\"\"\n self.exit_now = True\n payload = dict(\n source='IPython.zmq.zmqshell.ZMQInteractiveShell.ask_exit',\n exit=True,\n keepkernel=self.keepkernel_on_exit,\n )\n self.payload_manager.write_payload(payload)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_next_input(self, text):\n payload = dict(\n source='IPython.zmq.zmqshell.ZMQInteractiveShell.set_next_input',\n text=text\n )\n self.payload_manager.write_payload(payload)", "response": "Send the specified text to the frontend to be presented at the next\n input cell."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking if process is running", "response": "def check_running(process_name=\"apache2\"):\n '''\n CHECK if process (default=apache2) is not running\n '''\n if not gurumate.base.get_pid_list(process_name):\n fail(\"Apache process '%s' doesn't seem to be working\" % process_name)\n return False #unreachable\n return True"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_listening_ports(process_name=\"apache2\"):\n '''\n returns a list of listening ports for running process (default=apache2)\n '''\n ports = set()\n for _, address_info in gurumate.base.get_listening_ports(process_name):\n ports.add(address_info[1])\n return list(ports)", "response": "get_listening_ports returns a list of listening ports for running process"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads a filename as UTF - 8 configuration data.", "response": "def read(self, filename):\n \"\"\"Read a filename as UTF-8 configuration data.\"\"\"\n kwargs = {}\n if sys.version_info >= (3, 2):\n kwargs['encoding'] = \"utf-8\"\n return configparser.RawConfigParser.read(self, filename, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getlist(self, section, option):\n value_list = self.get(section, option)\n values = []\n for value_line in value_list.split('\\n'):\n for value in value_line.split(','):\n value = value.strip()\n if value:\n values.append(value)\n return values", "response": "Read a list of strings."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getlinelist(self, section, option):\n value_list = self.get(section, option)\n return list(filter(None, value_list.split('\\n')))", "response": "Read a list of full - line strings."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreads configuration from the env_var environment variable.", "response": "def from_environment(self, env_var):\n \"\"\"Read configuration from the `env_var` environment variable.\"\"\"\n # Timidity: for nose users, read an environment variable. This is a\n # cheap hack, since the rest of the command line arguments aren't\n # recognized, but it solves some users' problems.\n env = os.environ.get(env_var, '')\n if env:\n self.timid = ('--timid' in env)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef from_args(self, **kwargs):\n for k, v in iitems(kwargs):\n if v is not None:\n if k in self.MUST_BE_LIST and isinstance(v, string_class):\n v = [v]\n setattr(self, k, v)", "response": "Read config values from kwargs."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading configuration from a. rc file.", "response": "def from_file(self, filename):\n \"\"\"Read configuration from a .rc file.\n\n `filename` is a file name to read.\n\n \"\"\"\n self.attempted_config_files.append(filename)\n\n cp = HandyConfigParser()\n files_read = cp.read(filename)\n if files_read is not None: # return value changed in 2.4\n self.config_files.extend(files_read)\n\n for option_spec in self.CONFIG_FILE_OPTIONS:\n self.set_attr_from_config_option(cp, *option_spec)\n\n # [paths] is special\n if cp.has_section('paths'):\n for option in cp.options('paths'):\n self.paths[option] = cp.getlist('paths', option)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset an attribute on self if it exists in the ConfigParser.", "response": "def set_attr_from_config_option(self, cp, attr, where, type_=''):\n \"\"\"Set an attribute on self if it exists in the ConfigParser.\"\"\"\n section, option = where.split(\":\")\n if cp.has_option(section, option):\n method = getattr(cp, 'get'+type_)\n setattr(self, attr, method(section, option))"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef expand_user(path):\n # Default values\n tilde_expand = False\n tilde_val = ''\n newpath = path\n\n if path.startswith('~'):\n tilde_expand = True\n rest = len(path)-1\n newpath = os.path.expanduser(path)\n if rest:\n tilde_val = newpath[:-rest]\n else:\n tilde_val = newpath\n\n return newpath, tilde_expand, tilde_val", "response": "Expand ~ - style usernames in strings."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the delimiters for line splitting.", "response": "def delims(self, delims):\n \"\"\"Set the delimiters for line splitting.\"\"\"\n expr = '[' + ''.join('\\\\'+ c for c in delims) + ']'\n self._delim_re = re.compile(expr)\n self._delims = delims\n self._delim_expr = expr"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsplit a line of text with a cursor at the given position.", "response": "def split_line(self, line, cursor_pos=None):\n \"\"\"Split a line of text with a cursor at the given position.\n \"\"\"\n l = line if cursor_pos is None else line[:cursor_pos]\n return self._delim_re.split(l)[-1]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute matches when text is a simple name.", "response": "def global_matches(self, text):\n \"\"\"Compute matches when text is a simple name.\n\n Return a list of all keywords, built-in functions and names currently\n defined in self.namespace or self.global_namespace that match.\n\n \"\"\"\n #print 'Completer->global_matches, txt=%r' % text # dbg\n matches = []\n match_append = matches.append\n n = len(text)\n for lst in [keyword.kwlist,\n __builtin__.__dict__.keys(),\n self.namespace.keys(),\n self.global_namespace.keys()]:\n for word in lst:\n if word[:n] == text and word != \"__builtins__\":\n match_append(word)\n return matches"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef attr_matches(self, text):\n\n #io.rprint('Completer->attr_matches, txt=%r' % text) # dbg\n # Another option, seems to work great. Catches things like ''.\n m = re.match(r\"(\\S+(\\.\\w+)*)\\.(\\w*)$\", text)\n \n if m:\n expr, attr = m.group(1, 3)\n elif self.greedy:\n m2 = re.match(r\"(.+)\\.(\\w*)$\", self.line_buffer)\n if not m2:\n return []\n expr, attr = m2.group(1,2)\n else:\n return []\n \n try:\n obj = eval(expr, self.namespace)\n except:\n try:\n obj = eval(expr, self.global_namespace)\n except:\n return []\n\n if self.limit_to__all__ and hasattr(obj, '__all__'):\n words = get__all__entries(obj)\n else: \n words = dir2(obj)\n\n try:\n words = generics.complete_object(obj, words)\n except TryNext:\n pass\n except Exception:\n # Silence errors from completion function\n #raise # dbg\n pass\n # Build match list to return\n n = len(attr)\n res = [\"%s.%s\" % (expr, w) for w in words if w[:n] == attr ]\n return res", "response": "Compute matches when text contains a dot."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _greedy_changed(self, name, old, new):\n if new:\n self.splitter.delims = GREEDY_DELIMS\n else:\n self.splitter.delims = DELIMS\n\n if self.readline:\n self.readline.set_completer_delims(self.splitter.delims)", "response": "update the splitter and readline delims when greedy is changed"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef file_matches(self, text):\n\n #io.rprint('Completer->file_matches: <%r>' % text) # dbg\n\n # chars that require escaping with backslash - i.e. chars\n # that readline treats incorrectly as delimiters, but we\n # don't want to treat as delimiters in filename matching\n # when escaped with backslash\n if text.startswith('!'):\n text = text[1:]\n text_prefix = '!'\n else:\n text_prefix = ''\n\n text_until_cursor = self.text_until_cursor\n # track strings with open quotes\n open_quotes = has_open_quotes(text_until_cursor)\n\n if '(' in text_until_cursor or '[' in text_until_cursor:\n lsplit = text\n else:\n try:\n # arg_split ~ shlex.split, but with unicode bugs fixed by us\n lsplit = arg_split(text_until_cursor)[-1]\n except ValueError:\n # typically an unmatched \", or backslash without escaped char.\n if open_quotes:\n lsplit = text_until_cursor.split(open_quotes)[-1]\n else:\n return []\n except IndexError:\n # tab pressed on empty line\n lsplit = \"\"\n\n if not open_quotes and lsplit != protect_filename(lsplit):\n # if protectables are found, do matching on the whole escaped name\n has_protectables = True\n text0,text = text,lsplit\n else:\n has_protectables = False\n text = os.path.expanduser(text)\n\n if text == \"\":\n return [text_prefix + protect_filename(f) for f in self.glob(\"*\")]\n\n # Compute the matches from the filesystem\n m0 = self.clean_glob(text.replace('\\\\',''))\n\n if has_protectables:\n # If we had protectables, we need to revert our changes to the\n # beginning of filename so that we don't double-write the part\n # of the filename we have so far\n len_lsplit = len(lsplit)\n matches = [text_prefix + text0 +\n protect_filename(f[len_lsplit:]) for f in m0]\n else:\n if open_quotes:\n # if we have a string with an open quote, we don't need to\n # protect the names at all (and we _shouldn't_, as it\n # would cause bugs when the filesystem call is made).\n matches = m0\n else:\n matches = [text_prefix +\n protect_filename(f) for f in m0]\n\n #io.rprint('mm', matches) # dbg\n\n # Mark directories in input list by appending '/' to their names.\n matches = [x+'/' if os.path.isdir(x) else x for x in matches]\n return matches", "response": "Match filenames expanding ~USER type strings."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef magic_matches(self, text):\n #print 'Completer->magic_matches:',text,'lb',self.text_until_cursor # dbg\n # Get all shell magics now rather than statically, so magics loaded at\n # runtime show up too.\n lsm = self.shell.magics_manager.lsmagic()\n line_magics = lsm['line']\n cell_magics = lsm['cell']\n pre = self.magic_escape\n pre2 = pre+pre\n \n # Completion logic:\n # - user gives %%: only do cell magics\n # - user gives %: do both line and cell magics\n # - no prefix: do both\n # In other words, line magics are skipped if the user gives %% explicitly\n bare_text = text.lstrip(pre)\n comp = [ pre2+m for m in cell_magics if m.startswith(bare_text)]\n if not text.startswith(pre2):\n comp += [ pre+m for m in line_magics if m.startswith(bare_text)]\n return comp", "response": "Match magics in text."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef alias_matches(self, text):\n #print 'Completer->alias_matches:',text,'lb',self.text_until_cursor # dbg\n\n # if we are not in the first 'item', alias matching\n # doesn't make sense - unless we are starting with 'sudo' command.\n main_text = self.text_until_cursor.lstrip()\n if ' ' in main_text and not main_text.startswith('sudo'):\n return []\n text = os.path.expanduser(text)\n aliases = self.alias_table.keys()\n if text == '':\n return aliases\n else:\n return [a for a in aliases if a.startswith(text)]", "response": "Match internal system aliases"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef python_matches(self,text):\n \n #io.rprint('Completer->python_matches, txt=%r' % text) # dbg\n if \".\" in text:\n try:\n matches = self.attr_matches(text)\n if text.endswith('.') and self.omit__names:\n if self.omit__names == 1:\n # true if txt is _not_ a __ name, false otherwise:\n no__name = (lambda txt:\n re.match(r'.*\\.__.*?__',txt) is None)\n else:\n # true if txt is _not_ a _ name, false otherwise:\n no__name = (lambda txt:\n re.match(r'.*\\._.*?',txt) is None)\n matches = filter(no__name, matches)\n except NameError:\n # catches .\n matches = []\n else:\n matches = self.global_matches(text)\n\n return matches", "response": "Match attributes or global python names."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the list of default arguments of obj if it is callable or empty list otherwise.", "response": "def _default_arguments(self, obj):\n \"\"\"Return the list of default arguments of obj if it is callable,\n or empty list otherwise.\"\"\"\n\n if not (inspect.isfunction(obj) or inspect.ismethod(obj)):\n # for classes, check for __init__,__new__\n if inspect.isclass(obj):\n obj = (getattr(obj,'__init__',None) or\n getattr(obj,'__new__',None))\n # for all others, check if they are __call__able\n elif hasattr(obj, '__call__'):\n obj = obj.__call__\n # XXX: is there a way to handle the builtins ?\n try:\n args,_,_1,defaults = inspect.getargspec(obj)\n if defaults:\n return args[-len(defaults):]\n except TypeError: pass\n return []"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef python_func_kw_matches(self,text):\n\n if \".\" in text: # a parameter cannot be dotted\n return []\n try: regexp = self.__funcParamsRegex\n except AttributeError:\n regexp = self.__funcParamsRegex = re.compile(r'''\n '.*?(?,a=1)\", the candidate is \"foo\"\n tokens = regexp.findall(self.text_until_cursor)\n tokens.reverse()\n iterTokens = iter(tokens); openPar = 0\n for token in iterTokens:\n if token == ')':\n openPar -= 1\n elif token == '(':\n openPar += 1\n if openPar > 0:\n # found the last unclosed parenthesis\n break\n else:\n return []\n # 2. Concatenate dotted names (\"foo.bar\" for \"foo.bar(x, pa\" )\n ids = []\n isId = re.compile(r'\\w+$').match\n while True:\n try:\n ids.append(iterTokens.next())\n if not isId(ids[-1]):\n ids.pop(); break\n if not iterTokens.next() == '.':\n break\n except StopIteration:\n break\n # lookup the candidate callable matches either using global_matches\n # or attr_matches for dotted names\n if len(ids) == 1:\n callableMatches = self.global_matches(ids[0])\n else:\n callableMatches = self.attr_matches('.'.join(ids[::-1]))\n argMatches = []\n for callableMatch in callableMatches:\n try:\n namedArgs = self._default_arguments(eval(callableMatch,\n self.namespace))\n except:\n continue\n for namedArg in namedArgs:\n if namedArg.startswith(text):\n argMatches.append(\"%s=\" %namedArg)\n return argMatches", "response": "Match named parameters of the last open function."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind completions for the given text and line context.", "response": "def complete(self, text=None, line_buffer=None, cursor_pos=None):\n \"\"\"Find completions for the given text and line context.\n\n This is called successively with state == 0, 1, 2, ... until it\n returns None. The completion should begin with 'text'.\n\n Note that both the text and the line_buffer are optional, but at least\n one of them must be given.\n\n Parameters\n ----------\n text : string, optional\n Text to perform the completion on. If not given, the line buffer\n is split using the instance's CompletionSplitter object.\n\n line_buffer : string, optional\n If not given, the completer attempts to obtain the current line\n buffer via readline. This keyword allows clients which are\n requesting for text completions in non-readline contexts to inform\n the completer of the entire text.\n\n cursor_pos : int, optional\n Index of the cursor in the full line buffer. Should be provided by\n remote frontends where kernel has no access to frontend state.\n\n Returns\n -------\n text : str\n Text that was actually used in the completion.\n\n matches : list\n A list of completion matches.\n \"\"\"\n #io.rprint('\\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg\n\n # if the cursor position isn't given, the only sane assumption we can\n # make is that it's at the end of the line (the common case)\n if cursor_pos is None:\n cursor_pos = len(line_buffer) if text is None else len(text)\n\n # if text is either None or an empty string, rely on the line buffer\n if not text:\n text = self.splitter.split_line(line_buffer, cursor_pos)\n\n # If no line buffer is given, assume the input text is all there was\n if line_buffer is None:\n line_buffer = text\n\n self.line_buffer = line_buffer\n self.text_until_cursor = self.line_buffer[:cursor_pos]\n #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg\n\n # Start with a clean slate of completions\n self.matches[:] = []\n custom_res = self.dispatch_custom_completer(text)\n if custom_res is not None:\n # did custom completers produce something?\n self.matches = custom_res\n else:\n # Extend the list of completions with the results of each\n # matcher, so we return results to the user from all\n # namespaces.\n if self.merge_completions:\n self.matches = []\n for matcher in self.matchers:\n try:\n self.matches.extend(matcher(text))\n except:\n # Show the ugly traceback if the matcher causes an\n # exception, but do NOT crash the kernel!\n sys.excepthook(*sys.exc_info())\n else:\n for matcher in self.matchers:\n self.matches = matcher(text)\n if self.matches:\n break\n # FIXME: we should extend our api to return a dict with completions for\n # different types of objects. The rlcomplete() method could then\n # simply collapse the dict into a list for readline, but we'd have\n # richer completion semantics in other evironments.\n self.matches = sorted(set(self.matches))\n #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg\n return text, self.matches"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the state - th possible completion for text.", "response": "def rlcomplete(self, text, state):\n \"\"\"Return the state-th possible completion for 'text'.\n\n This is called successively with state == 0, 1, 2, ... until it\n returns None. The completion should begin with 'text'.\n\n Parameters\n ----------\n text : string\n Text to perform the completion on.\n\n state : int\n Counter used by readline.\n \"\"\"\n if state==0:\n\n self.line_buffer = line_buffer = self.readline.get_line_buffer()\n cursor_pos = self.readline.get_endidx()\n\n #io.rprint(\"\\nRLCOMPLETE: %r %r %r\" %\n # (text, line_buffer, cursor_pos) ) # dbg\n\n # if there is only a tab on a line with only whitespace, instead of\n # the mostly useless 'do you want to see all million completions'\n # message, just do the right thing and give the user his tab!\n # Incidentally, this enables pasting of tabbed text from an editor\n # (as long as autoindent is off).\n\n # It should be noted that at least pyreadline still shows file\n # completions - is there a way around it?\n\n # don't apply this on 'dumb' terminals, such as emacs buffers, so\n # we don't interfere with their own tab-completion mechanism.\n if not (self.dumb_terminal or line_buffer.strip()):\n self.readline.insert_text('\\t')\n sys.stdout.flush()\n return None\n\n # Note: debugging exceptions that may occur in completion is very\n # tricky, because readline unconditionally silences them. So if\n # during development you suspect a bug in the completion code, turn\n # this flag on temporarily by uncommenting the second form (don't\n # flip the value in the first line, as the '# dbg' marker can be\n # automatically detected and is used elsewhere).\n DEBUG = False\n #DEBUG = True # dbg\n if DEBUG:\n try:\n self.complete(text, line_buffer, cursor_pos)\n except:\n import traceback; traceback.print_exc()\n else:\n # The normal production version is here\n\n # This method computes the self.matches array\n self.complete(text, line_buffer, cursor_pos)\n\n try:\n return self.matches[state]\n except IndexError:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking if a specific record matches tests.", "response": "def _match_one(self, rec, tests):\n \"\"\"Check if a specific record matches tests.\"\"\"\n for key,test in tests.iteritems():\n if not test(rec.get(key, None)):\n return False\n return True"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _match(self, check):\n matches = []\n tests = {}\n for k,v in check.iteritems():\n if isinstance(v, dict):\n tests[k] = CompositeFilter(v)\n else:\n tests[k] = lambda o: o==v\n\n for rec in self._records.itervalues():\n if self._match_one(rec, tests):\n matches.append(copy(rec))\n return matches", "response": "Find all the matches for a check dict."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nextract subdict of keys", "response": "def _extract_subdict(self, rec, keys):\n \"\"\"extract subdict of keys\"\"\"\n d = {}\n d['msg_id'] = rec['msg_id']\n for key in keys:\n d[key] = rec[key]\n return copy(d)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds a new Task Record by msg_id.", "response": "def add_record(self, msg_id, rec):\n \"\"\"Add a new Task Record, by msg_id.\"\"\"\n if self._records.has_key(msg_id):\n raise KeyError(\"Already have msg_id %r\"%(msg_id))\n self._records[msg_id] = rec"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets a specific Task Record by msg_id.", "response": "def get_record(self, msg_id):\n \"\"\"Get a specific Task Record, by msg_id.\"\"\"\n if not msg_id in self._records:\n raise KeyError(\"No such msg_id %r\"%(msg_id))\n return copy(self._records[msg_id])"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef drop_matching_records(self, check):\n matches = self._match(check)\n for m in matches:\n del self._records[m['msg_id']]", "response": "Remove a record from the DB."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef find_records(self, check, keys=None):\n matches = self._match(check)\n if keys:\n return [ self._extract_subdict(rec, keys) for rec in matches ]\n else:\n return matches", "response": "Find records matching a query dict optionally extracting subset of keys."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_history(self):\n msg_ids = self._records.keys()\n return sorted(msg_ids, key=lambda m: self._records[m]['submitted'])", "response": "get all msg_ids ordered by time submitted."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef quiet(self):\n # do not print output if input ends in ';'\n try:\n cell = self.shell.history_manager.input_hist_parsed[self.prompt_count]\n if cell.rstrip().endswith(';'):\n return True\n except IndexError:\n # some uses of ipshellembed may fail here\n pass\n return False", "response": "Should we silence the display hook because of ';'?"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write_output_prompt(self):\n # Use write, not print which adds an extra space.\n io.stdout.write(self.shell.separate_out)\n outprompt = self.shell.prompt_manager.render('out')\n if self.do_full_cache:\n io.stdout.write(outprompt)", "response": "Write the output prompt."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting the format data dict to the frontend.", "response": "def write_format_data(self, format_dict):\n \"\"\"Write the format data dict to the frontend.\n\n This default version of this method simply writes the plain text\n representation of the object to ``io.stdout``. Subclasses should\n override this method to send the entire `format_dict` to the\n frontends.\n\n Parameters\n ----------\n format_dict : dict\n The format dict for the object passed to `sys.displayhook`.\n \"\"\"\n # We want to print because we want to always make sure we have a\n # newline, even if all the prompt separators are ''. This is the\n # standard IPython behavior.\n result_repr = format_dict['text/plain']\n if '\\n' in result_repr:\n # So that multi-line strings line up with the left column of\n # the screen, instead of having the output prompt mess up\n # their first line.\n # We use the prompt template instead of the expanded prompt\n # because the expansion may add ANSI escapes that will interfere\n # with our ability to determine whether or not we should add\n # a newline.\n prompt_template = self.shell.prompt_manager.out_template\n if prompt_template and not prompt_template.endswith('\\n'):\n # But avoid extraneous empty lines.\n result_repr = '\\n' + result_repr\n\n print >>io.stdout, result_repr"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates user_ns with the current result.", "response": "def update_user_ns(self, result):\n \"\"\"Update user_ns with various things like _, __, _1, etc.\"\"\"\n\n # Avoid recursive reference when displaying _oh/Out\n if result is not self.shell.user_ns['_oh']:\n if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:\n warn('Output cache limit (currently '+\n `self.cache_size`+' entries) hit.\\n'\n 'Flushing cache and resetting history counter...\\n'\n 'The only history variables available will be _,__,___ and _1\\n'\n 'with the current result.')\n\n self.flush()\n # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise\n # we cause buggy behavior for things like gettext).\n\n if '_' not in __builtin__.__dict__:\n self.___ = self.__\n self.__ = self._\n self._ = result\n self.shell.push({'_':self._,\n '__':self.__,\n '___':self.___}, interactive=False)\n\n # hackish access to top-level namespace to create _1,_2... dynamically\n to_main = {}\n if self.do_full_cache:\n new_result = '_'+`self.prompt_count`\n to_main[new_result] = result\n self.shell.push(to_main, interactive=False)\n self.shell.user_ns['_oh'][self.prompt_count] = result"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlogs the output of the current process.", "response": "def log_output(self, format_dict):\n \"\"\"Log the output.\"\"\"\n if self.shell.logger.log_output:\n self.shell.logger.log_write(format_dict['text/plain'], 'output')\n self.shell.history_manager.output_hist_reprs[self.prompt_count] = \\\n format_dict['text/plain']"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef finish_displayhook(self):\n io.stdout.write(self.shell.separate_out2)\n io.stdout.flush()", "response": "Finish up all displayhook activities."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads the extension in IPython.", "response": "def load_ipython_extension(ip):\n \"\"\"Load the extension in IPython.\"\"\"\n global _loaded\n if not _loaded:\n plugin = StoreMagic(shell=ip, config=ip.config)\n ip.plugin_manager.register_plugin('storemagic', plugin)\n _loaded = True"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef raise_if_freezed(self):\n '''raise `InvalidOperationException` if is freezed.'''\n if self.is_freezed:\n name = type(self).__name__\n raise InvalidOperationException('obj {name} is freezed.'.format(name=name))", "response": "raise InvalidOperationException if is freezed."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert a MySQL TIMESTAMP to a Timestamp object.", "response": "def mysql_timestamp_converter(s):\n \"\"\"Convert a MySQL TIMESTAMP to a Timestamp object.\"\"\"\n # MySQL>4.1 returns TIMESTAMP in the same format as DATETIME\n if s[4] == '-': return DateTime_or_None(s)\n s = s + \"0\"*(14-len(s)) # padding\n parts = map(int, filter(None, (s[:4],s[4:6],s[6:8],\n s[8:10],s[10:12],s[12:14])))\n try:\n return Timestamp(*parts)\n except (SystemExit, KeyboardInterrupt):\n raise\n except:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a SOAP envelope for the current object.", "response": "def make_envelope(self, body_elements=None):\n \"\"\"\n body_elements: of etree.Elements or \n \"\"\"\n soap = self.get_soap_el_factory()\n body_elements = body_elements or []\n body = soap.Body(*body_elements)\n header = self.get_soap_header()\n if header is not None:\n elements = [header, body]\n else:\n elements = [body]\n return soap.Envelope(\n #{\n # '{%s}encodingStyle' % namespaces.SOAP_1_2: (\n # 'http://www.w3.org/2001/12/soap-encoding')\n #},\n *elements\n )"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef embed_kernel(module=None, local_ns=None, **kwargs):\n # get the app if it exists, or set it up if it doesn't\n if IPKernelApp.initialized():\n app = IPKernelApp.instance()\n else:\n app = IPKernelApp.instance(**kwargs)\n app.initialize([])\n # Undo unnecessary sys module mangling from init_sys_modules.\n # This would not be necessary if we could prevent it\n # in the first place by using a different InteractiveShell\n # subclass, as in the regular embed case.\n main = app.kernel.shell._orig_sys_modules_main_mod\n if main is not None:\n sys.modules[app.kernel.shell._orig_sys_modules_main_name] = main\n\n # load the calling scope if not given\n (caller_module, caller_locals) = extract_module_locals(1)\n if module is None:\n module = caller_module\n if local_ns is None:\n local_ns = caller_locals\n \n app.kernel.user_module = module\n app.kernel.user_ns = local_ns\n app.shell.set_completer_frame()\n app.start()", "response": "Embed and start an IPython kernel in a given scope."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _eventloop_changed(self, name, old, new):\n loop = ioloop.IOLoop.instance()\n loop.add_timeout(time.time()+0.1, self.enter_eventloop)", "response": "schedule call to eventloop from IOLoop"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nregistering dispatchers for streams", "response": "def start(self):\n \"\"\"register dispatchers for streams\"\"\"\n self.shell.exit_now = False\n if self.control_stream:\n self.control_stream.on_recv(self.dispatch_control, copy=False)\n\n def make_dispatcher(stream):\n def dispatcher(msg):\n return self.dispatch_shell(stream, msg)\n return dispatcher\n\n for s in self.shell_streams:\n s.on_recv(make_dispatcher(s), copy=False)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstep eventloop just once", "response": "def do_one_iteration(self):\n \"\"\"step eventloop just once\"\"\"\n if self.control_stream:\n self.control_stream.flush()\n for stream in self.shell_streams:\n # handle at most one request per iteration\n stream.flush(zmq.POLLIN, 1)\n stream.flush(zmq.POLLOUT)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _publish_pyin(self, code, parent, execution_count):\n\n self.session.send(self.iopub_socket, u'pyin',\n {u'code':code, u'execution_count': execution_count},\n parent=parent, ident=self._topic('pyin')\n )", "response": "Publish the code request on the pyin stream."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsend status on IOPub", "response": "def _publish_status(self, status, parent=None):\n \"\"\"send status (busy/idle) on IOPub\"\"\"\n self.session.send(self.iopub_socket,\n u'status',\n {u'execution_state': status},\n parent=parent,\n ident=self._topic('status'),\n )"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef abort_request(self, stream, ident, parent):\n msg_ids = parent['content'].get('msg_ids', None)\n if isinstance(msg_ids, basestring):\n msg_ids = [msg_ids]\n if not msg_ids:\n self.abort_queues()\n for mid in msg_ids:\n self.aborted.add(str(mid))\n\n content = dict(status='ok')\n reply_msg = self.session.send(stream, 'abort_reply', content=content,\n parent=parent, ident=ident)\n self.log.debug(\"%s\", reply_msg)", "response": "abort a specifig msg by id"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nprefixing topic for IOPub messages", "response": "def _topic(self, topic):\n \"\"\"prefixed topic for IOPub messages\"\"\"\n if self.int_id >= 0:\n base = \"engine.%i\" % self.int_id\n else:\n base = \"kernel.%s\" % self.ident\n \n return py3compat.cast_bytes(\"%s.%s\" % (base, topic))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _at_shutdown(self):\n # io.rprint(\"Kernel at_shutdown\") # dbg\n if self._shutdown_message is not None:\n self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown'))\n self.log.debug(\"%s\", self._shutdown_message)\n [ s.flush(zmq.POLLOUT) for s in self.shell_streams ]", "response": "Called by the kernel when the kernel is at shutdown."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nenable GUI event loop integration taking pylab into account.", "response": "def init_gui_pylab(self):\n \"\"\"Enable GUI event loop integration, taking pylab into account.\"\"\"\n\n # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab`\n # to ensure that any exception is printed straight to stderr.\n # Normally _showtraceback associates the reply with an execution,\n # which means frontends will never draw it, as this exception\n # is not associated with any execute request.\n\n shell = self.shell\n _showtraceback = shell._showtraceback\n try:\n # replace pyerr-sending traceback with stderr\n def print_tb(etype, evalue, stb):\n print (\"GUI event loop or pylab initialization failed\",\n file=io.stderr)\n print (shell.InteractiveTB.stb2text(stb), file=io.stderr)\n shell._showtraceback = print_tb\n InteractiveShellApp.init_gui_pylab(self)\n finally:\n shell._showtraceback = _showtraceback"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef slices(self, size, n=3):\r\n '''\r\n Create equal sized slices of the file. The last slice may be larger than\r\n the others.\r\n \r\n args:\r\n * size (int): The full size to be sliced.\r\n * n (int): The number of slices to return.\r\n returns:\r\n A list of `FileSlice` objects of length (n).\r\n '''\r\n if n <= 0:\r\n raise ValueError('n must be greater than 0')\r\n if size < n:\r\n raise ValueError('size argument cannot be less than n argument')\r\n slice_size = size // n\r\n last_slice_size = size - (n-1) * slice_size\r\n t = [self(c, slice_size) for c in range(0, (n-1)*slice_size, slice_size)]\r\n t.append(self((n-1)*slice_size, last_slice_size))\r\n return t", "response": "Create equal sized slices of the file."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a new array of size from the current position.", "response": "def read(self, size=-1):\r\n '''\r\n Same as `file.read()` but for the slice. Does not read beyond\r\n `self.end`.\r\n '''\r\n if self.closed:\r\n raise ValueError('I/O operation on closed file.')\r\n \r\n if size == -1 or size > self.size - self.pos:\r\n size = self.size - self.pos\r\n \r\n with self.lock:\r\n self.f.seek(self.start + self.pos)\r\n result = self.f.read(size)\r\n self.seek(self.pos + size)\r\n \r\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites a list of lines to the file.", "response": "def writelines(self, lines):\r\n '''\r\n Same as `file.writelines()` but for the slice.\r\n \r\n raises:\r\n EOFError if the new seek position is > `self.size`.\r\n '''\r\n lines = b''.join(lines)\r\n self.write(lines)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef configure(self, options, conf):\n Plugin.configure(self, options, conf)\n self._mod_stack = []", "response": "Configure the object with the given options and conf."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef beforeContext(self):\n mods = sys.modules.copy()\n self._mod_stack.append(mods)", "response": "Copy sys. modules onto my mod stack and add them to the stack"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef afterContext(self):\n mods = self._mod_stack.pop()\n to_del = [ m for m in sys.modules.keys() if m not in mods ]\n if to_del:\n log.debug('removing sys modules entries: %s', to_del)\n for mod in to_del:\n del sys.modules[mod]\n sys.modules.update(mods)", "response": "Called after the context has been pushed and restore sys. modules to the state\n it was in."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef absdir(path):\n if not os.path.isabs(path):\n path = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(),\n path)))\n if path is None or not os.path.isdir(path):\n return None\n return path", "response": "Return absolute normalized path to directory if it exists ; None otherwise."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef absfile(path, where=None):\n orig = path\n if where is None:\n where = os.getcwd()\n if isinstance(where, list) or isinstance(where, tuple):\n for maybe_path in where:\n maybe_abs = absfile(path, maybe_path)\n if maybe_abs is not None:\n return maybe_abs\n return None\n if not os.path.isabs(path):\n path = os.path.normpath(os.path.abspath(os.path.join(where, path)))\n if path is None or not os.path.exists(path):\n if where != os.getcwd():\n # try the cwd instead\n path = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(),\n orig)))\n if path is None or not os.path.exists(path):\n return None\n if os.path.isdir(path):\n # might want an __init__.py from pacakge\n init = os.path.join(path,'__init__.py')\n if os.path.isfile(init):\n return init\n elif os.path.isfile(path):\n return path\n return None", "response": "Return absolute path to file optionally in directory\n where or None if the file can t be found either in directory\n or the current directory\n where."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef file_like(name):\n return (os.path.exists(name)\n or os.path.dirname(name)\n or name.endswith('.py')\n or not ident_re.match(os.path.splitext(name)[0]))", "response": "A name is file - like if it exists a directory part of a\n or a file - like identifier."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nbeing obj a class? Inspect s isclass is too liberal and returns True for objects that can t be subclasses of anything.", "response": "def isclass(obj):\n \"\"\"Is obj a class? Inspect's isclass is too liberal and returns True\n for objects that can't be subclasses of anything.\n \"\"\"\n obj_type = type(obj)\n return obj_type in class_types or issubclass(obj_type, type)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ispackage(path):\n if os.path.isdir(path):\n # at least the end of the path must be a legal python identifier\n # and __init__.py[co] must exist\n end = os.path.basename(path)\n if ident_re.match(end):\n for init in ('__init__.py', '__init__.pyc', '__init__.pyo'):\n if os.path.isfile(os.path.join(path, init)):\n return True\n if sys.platform.startswith('java') and \\\n os.path.isfile(os.path.join(path, '__init__$py.class')):\n return True\n return False", "response": "Check if path is a package directory."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getfilename(package, relativeTo=None):\n if relativeTo is None:\n relativeTo = os.getcwd()\n path = os.path.join(relativeTo, os.sep.join(package.split('.')))\n suffixes = ('/__init__.py', '.py')\n for suffix in suffixes:\n filename = path + suffix\n if os.path.exists(filename):\n return filename\n return None", "response": "Find the python source file for a package relative to the current working directory."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getpackage(filename):\n src_file = src(filename)\n if not src_file.endswith('.py') and not ispackage(src_file):\n return None\n base, ext = os.path.splitext(os.path.basename(src_file))\n if base == '__init__':\n mod_parts = []\n else:\n mod_parts = [base]\n path, part = os.path.split(os.path.split(src_file)[0])\n while part:\n if ispackage(os.path.join(path, part)):\n mod_parts.append(part)\n else:\n break\n path, part = os.path.split(path)\n mod_parts.reverse()\n return '.'.join(mod_parts)", "response": "Find the full dotted package name for a given python source file. Returns None if the file is not a python source file."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ln(label):\n label_len = len(label) + 2\n chunk = (70 - label_len) // 2\n out = '%s %s %s' % ('-' * chunk, label, '-' * chunk)\n pad = 70 - len(out)\n if pad > 0:\n out = out + ('-' * pad)\n return out", "response": "Draw a 70 - char - wide divider with label in the middle."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntrying to run a list of possible method names with the object obj.", "response": "def try_run(obj, names):\n \"\"\"Given a list of possible method names, try to run them with the\n provided object. Keep going until something works. Used to run\n setup/teardown methods for module, package, and function tests.\n \"\"\"\n for name in names:\n func = getattr(obj, name, None)\n if func is not None:\n if type(obj) == types.ModuleType:\n # py.test compatibility\n try:\n args, varargs, varkw, defaults = inspect.getargspec(func)\n except TypeError:\n # Not a function. If it's callable, call it anyway\n if hasattr(func, '__call__'):\n func = func.__call__\n try:\n args, varargs, varkw, defaults = \\\n inspect.getargspec(func)\n args.pop(0) # pop the self off\n except TypeError:\n raise TypeError(\"Attribute %s of %r is not a python \"\n \"function. Only functions or callables\"\n \" may be used as fixtures.\" %\n (name, obj))\n if len(args):\n log.debug(\"call fixture %s.%s(%s)\", obj, name, obj)\n return func(obj)\n log.debug(\"call fixture %s.%s\", obj, name)\n return func()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef regex_last_key(regex):\n def k(obj):\n if regex.search(obj):\n return (1, obj)\n return (0, obj)\n return k", "response": "Sort list of items that match a\n regular expression last."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts a value that may be a list or a ( possibly comma - separated ) string into a list. The exception is returned as None not [ None ] not [ None ]", "response": "def tolist(val):\n \"\"\"Convert a value that may be a list or a (possibly comma-separated)\n string into a list. The exception: None is returned as None, not [None].\n\n >>> tolist([\"one\", \"two\"])\n ['one', 'two']\n >>> tolist(\"hello\")\n ['hello']\n >>> tolist(\"separate,values, with, commas, spaces , are ,ok\")\n ['separate', 'values', 'with', 'commas', 'spaces', 'are', 'ok']\n \"\"\"\n if val is None:\n return None\n try:\n # might already be a list\n val.extend([])\n return val\n except AttributeError:\n pass\n # might be a string\n try:\n return re.split(r'\\s*,\\s*', val)\n except TypeError:\n # who knows...\n return list(val)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntransplant a function in module A to module B.", "response": "def transplant_func(func, module):\n \"\"\"\n Make a function imported from module A appear as if it is located\n in module B.\n\n >>> from pprint import pprint\n >>> pprint.__module__\n 'pprint'\n >>> pp = transplant_func(pprint, __name__)\n >>> pp.__module__\n 'nose.util'\n\n The original function is not modified.\n\n >>> pprint.__module__\n 'pprint'\n\n Calling the transplanted function calls the original.\n\n >>> pp([1, 2])\n [1, 2]\n >>> pprint([1,2])\n [1, 2]\n\n \"\"\"\n from nose.tools import make_decorator\n def newfunc(*arg, **kw):\n return func(*arg, **kw)\n\n newfunc = make_decorator(func)(newfunc)\n newfunc.__module__ = module\n return newfunc"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef transplant_class(cls, module):\n class C(cls):\n pass\n C.__module__ = module\n C.__name__ = cls.__name__\n return C", "response": "Create a class that resides in module rather than the module in which\n it is actually defined."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the current system memory usage as a tuple.", "response": "def swap_memory():\n \"\"\"Swap system memory as a (total, used, free, sin, sout) tuple.\"\"\"\n total, used, free, sin, sout = _psutil_osx.get_swap_mem()\n percent = usage_percent(used, total, _round=1)\n return nt_swapmeminfo(total, used, free, percent, sin, sout)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_system_cpu_times():\n user, nice, system, idle = _psutil_osx.get_system_cpu_times()\n return _cputimes_ntuple(user, nice, system, idle)", "response": "Return system CPU times as a namedtuple."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn system CPU times as a named tuple", "response": "def get_system_per_cpu_times():\n \"\"\"Return system CPU times as a named tuple\"\"\"\n ret = []\n for cpu_t in _psutil_osx.get_system_per_cpu_times():\n user, nice, system, idle = cpu_t\n item = _cputimes_ntuple(user, nice, system, idle)\n ret.append(item)\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn process cmdline as a list of arguments.", "response": "def get_process_cmdline(self):\n \"\"\"Return process cmdline as a list of arguments.\"\"\"\n if not pid_exists(self.pid):\n raise NoSuchProcess(self.pid, self._process_name)\n return _psutil_osx.get_process_cmdline(self.pid)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a tuple with the process RSS and VMS size.", "response": "def get_memory_info(self):\n \"\"\"Return a tuple with the process' RSS and VMS size.\"\"\"\n rss, vms = _psutil_osx.get_process_memory_info(self.pid)[:2]\n return nt_meminfo(rss, vms)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a tuple with the process RSS VMS size and the process s memory size.", "response": "def get_ext_memory_info(self):\n \"\"\"Return a tuple with the process' RSS and VMS size.\"\"\"\n rss, vms, pfaults, pageins = _psutil_osx.get_process_memory_info(self.pid)\n return self._nt_ext_mem(rss, vms,\n pfaults * _PAGESIZE,\n pageins * _PAGESIZE)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning list of open files.", "response": "def get_open_files(self):\n \"\"\"Return files opened by process.\"\"\"\n if self.pid == 0:\n return []\n files = []\n rawlist = _psutil_osx.get_process_open_files(self.pid)\n for path, fd in rawlist:\n if isfile_strict(path):\n ntuple = nt_openfile(path, fd)\n files.append(ntuple)\n return files"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a list of etwork connections opened by a process as a list of namedtuples.", "response": "def get_connections(self, kind='inet'):\n \"\"\"Return etwork connections opened by a process as a list of\n namedtuples.\n \"\"\"\n if kind not in conn_tmap:\n raise ValueError(\"invalid %r kind argument; choose between %s\"\n % (kind, ', '.join([repr(x) for x in conn_tmap])))\n families, types = conn_tmap[kind]\n ret = _psutil_osx.get_process_connections(self.pid, families, types)\n return [nt_connection(*conn) for conn in ret]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef user_has_group(user, group, superuser_skip=True):\n\n if user.is_superuser and superuser_skip:\n return True\n\n return user.groups.filter(name=group).exists()", "response": "Check if a user is in a certaing group."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef resolve_class(class_path):\n\n modulepath, classname = class_path.rsplit('.', 1)\n module = __import__(modulepath, fromlist=[classname])\n return getattr(module, classname)", "response": "Load a class by a fully qualified class_path eg. myapp. models. ModelName"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncalculate percentage usage of used against total.", "response": "def usage_percent(used, total, _round=None):\n \"\"\"Calculate percentage usage of 'used' against 'total'.\"\"\"\n try:\n ret = (used / total) * 100\n except ZeroDivisionError:\n ret = 0\n if _round is not None:\n return round(ret, _round)\n else:\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef deprecated(replacement=None):\n def outer(fun):\n msg = \"psutil.%s is deprecated\" % fun.__name__\n if replacement is not None:\n msg += \"; use %s instead\" % replacement\n if fun.__doc__ is None:\n fun.__doc__ = msg\n\n @wraps(fun)\n def inner(*args, **kwargs):\n warnings.warn(msg, category=DeprecationWarning, stacklevel=2)\n return fun(*args, **kwargs)\n\n return inner\n return outer", "response": "A decorator which can be used to mark functions as deprecated."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\npush specified directory to git remote", "response": "def git_push(git_message=None, git_repository=None, git_branch=None,\n locale_root=None):\n \"\"\"\n Pushes specified directory to git remote\n :param git_message: commit message\n :param git_repository: repository address\n :param git_branch: git branch\n :param locale_root: path to locale root folder containing directories\n with languages\n :return: tuple stdout, stderr of completed command\n \"\"\"\n if git_message is None:\n git_message = settings.GIT_MESSAGE\n if git_repository is None:\n git_repository = settings.GIT_REPOSITORY\n if git_branch is None:\n git_branch = settings.GIT_BRANCH\n if locale_root is None:\n locale_root = settings.LOCALE_ROOT\n\n try:\n subprocess.check_call(['git', 'checkout', git_branch])\n except subprocess.CalledProcessError:\n try:\n subprocess.check_call(['git', 'checkout', '-b', git_branch])\n except subprocess.CalledProcessError as e:\n raise PODocsError(e)\n\n try:\n subprocess.check_call(['git', 'ls-remote', git_repository])\n except subprocess.CalledProcessError:\n try:\n subprocess.check_call(['git', 'remote', 'add', 'po_translator',\n git_repository])\n except subprocess.CalledProcessError as e:\n raise PODocsError(e)\n\n commands = 'git add ' + locale_root + \\\n ' && git commit -m \"' + git_message + '\"' + \\\n ' && git push po_translator ' + git_branch + ':' + git_branch\n proc = Popen(commands, shell=True, stdout=PIPE, stderr=PIPE)\n stdout, stderr = proc.communicate()\n\n return stdout, stderr"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef git_checkout(git_branch=None, locale_root=None):\n if git_branch is None:\n git_branch = settings.GIT_BRANCH\n if locale_root is None:\n locale_root = settings.LOCALE_ROOT\n\n proc = Popen('git checkout ' + git_branch + ' -- ' + locale_root,\n shell=True, stdout=PIPE, stderr=PIPE)\n stdout, stderr = proc.communicate()\n\n return stdout, stderr", "response": "Checkout the last commit of a node in a git repository"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _login(self):\n try:\n self.gd_client = gdata.docs.client.DocsClient()\n self.gd_client.ClientLogin(self.email, self.password, self.source)\n except RequestError as e:\n raise PODocsError(e)", "response": "Login into Google Docs with user authentication info."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse GDocs key from Spreadsheet url.", "response": "def _get_gdocs_key(self):\n \"\"\"\n Parse GDocs key from Spreadsheet url.\n \"\"\"\n try:\n args = urlparse.parse_qs(urlparse.urlparse(self.url).query)\n self.key = args['key'][0]\n except KeyError as e:\n raise PODocsError(e)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nensures that temp directory exists and create one if it does not.", "response": "def _ensure_temp_path_exists(self):\n \"\"\"\n Make sure temp directory exists and create one if it does not.\n \"\"\"\n try:\n if not os.path.exists(self.temp_path):\n os.mkdir(self.temp_path)\n except OSError as e:\n raise PODocsError(e)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _clear_temp(self):\n temp_files = [LOCAL_ODS, GDOCS_TRANS_CSV, GDOCS_META_CSV,\n LOCAL_TRANS_CSV, LOCAL_META_CSV]\n for temp_file in temp_files:\n file_path = os.path.join(self.temp_path, temp_file)\n if os.path.exists(file_path):\n os.remove(file_path)", "response": "Clear temp directory from created csv and ods files during communicator operations."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndownload csv from GDoc.", "response": "def _download_csv_from_gdocs(self, trans_csv_path, meta_csv_path):\n \"\"\"\n Download csv from GDoc.\n :return: returns resource if worksheets are present\n :except: raises PODocsError with info if communication\n with GDocs lead to any errors\n \"\"\"\n try:\n entry = self.gd_client.GetResourceById(self.key)\n self.gd_client.DownloadResource(\n entry, trans_csv_path,\n extra_params={'gid': 0, 'exportFormat': 'csv'}\n )\n self.gd_client.DownloadResource(\n entry, meta_csv_path,\n extra_params={'gid': 1, 'exportFormat': 'csv'}\n )\n except (RequestError, IOError) as e:\n raise PODocsError(e)\n return entry"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _upload_file_to_gdoc(\n self, file_path,\n content_type='application/x-vnd.oasis.opendocument.spreadsheet'):\n \"\"\"\n Uploads file to GDocs spreadsheet.\n Content type can be provided as argument, default is ods.\n \"\"\"\n try:\n entry = self.gd_client.GetResourceById(self.key)\n media = gdata.data.MediaSource(\n file_path=file_path, content_type=content_type)\n self.gd_client.UpdateResource(\n entry, media=media, update_metadata=True)\n except (RequestError, IOError) as e:\n raise PODocsError(e)", "response": "Uploads file to GDocs spreadsheet."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndownload csv from GDoc and update resource if worksheets are present.", "response": "def _merge_local_and_gdoc(self, entry, local_trans_csv,\n local_meta_csv, gdocs_trans_csv, gdocs_meta_csv):\n \"\"\"\n Download csv from GDoc.\n :return: returns resource if worksheets are present\n :except: raises PODocsError with info if communication with GDocs\n lead to any errors\n \"\"\"\n try:\n new_translations = po_to_csv_merge(\n self.languages, self.locale_root, self.po_files_path,\n local_trans_csv, local_meta_csv,\n gdocs_trans_csv, gdocs_meta_csv)\n if new_translations:\n local_ods = os.path.join(self.temp_path, LOCAL_ODS)\n csv_to_ods(local_trans_csv, local_meta_csv, local_ods)\n media = gdata.data.MediaSource(\n file_path=local_ods, content_type=\n 'application/x-vnd.oasis.opendocument.spreadsheet'\n )\n self.gd_client.UpdateResource(entry, media=media,\n update_metadata=True)\n except (IOError, OSError, RequestError) as e:\n raise PODocsError(e)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef synchronize(self):\n gdocs_trans_csv = os.path.join(self.temp_path, GDOCS_TRANS_CSV)\n gdocs_meta_csv = os.path.join(self.temp_path, GDOCS_META_CSV)\n local_trans_csv = os.path.join(self.temp_path, LOCAL_TRANS_CSV)\n local_meta_csv = os.path.join(self.temp_path, LOCAL_META_CSV)\n\n try:\n entry = self._download_csv_from_gdocs(gdocs_trans_csv,\n gdocs_meta_csv)\n except PODocsError as e:\n if 'Sheet 1 not found' in str(e) \\\n or 'Conversion failed unexpectedly' in str(e):\n self.upload()\n else:\n raise PODocsError(e)\n else:\n self._merge_local_and_gdoc(entry, local_trans_csv, local_meta_csv,\n gdocs_trans_csv, gdocs_meta_csv)\n\n try:\n csv_to_po(local_trans_csv, local_meta_csv,\n self.locale_root, self.po_files_path, self.header)\n except IOError as e:\n raise PODocsError(e)\n\n self._clear_temp()", "response": "Synchronize local po files with GDocs Spreadsheet."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndownloads csv files from GDocs and convert them into PO files structure.", "response": "def download(self):\n \"\"\"\n Download csv files from GDocs and convert them into po files structure.\n \"\"\"\n trans_csv_path = os.path.realpath(\n os.path.join(self.temp_path, GDOCS_TRANS_CSV))\n meta_csv_path = os.path.realpath(\n os.path.join(self.temp_path, GDOCS_META_CSV))\n\n self._download_csv_from_gdocs(trans_csv_path, meta_csv_path)\n\n try:\n csv_to_po(trans_csv_path, meta_csv_path,\n self.locale_root, self.po_files_path, header=self.header)\n except IOError as e:\n raise PODocsError(e)\n\n self._clear_temp()"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nuploads all po files to GDocs ignoring conflicts. This method looks for all msgids in po_files and sends them as ods to GDocs Spreadsheet.", "response": "def upload(self):\n \"\"\"\n Upload all po files to GDocs ignoring conflicts.\n This method looks for all msgids in po_files and sends them\n as ods to GDocs Spreadsheet.\n \"\"\"\n local_ods_path = os.path.join(self.temp_path, LOCAL_ODS)\n try:\n po_to_ods(self.languages, self.locale_root,\n self.po_files_path, local_ods_path)\n except (IOError, OSError) as e:\n raise PODocsError(e)\n\n self._upload_file_to_gdoc(local_ods_path)\n\n self._clear_temp()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef clear(self):\n empty_file_path = os.path.join(self.temp_path, 'empty.csv')\n try:\n empty_file = open(empty_file_path, 'w')\n empty_file.write(',')\n empty_file.close()\n except IOError as e:\n raise PODocsError(e)\n\n self._upload_file_to_gdoc(empty_file_path, content_type='text/csv')\n\n os.remove(empty_file_path)", "response": "Clear the GDoc Spreadsheet by sending empty. csv file to the temp directory."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstarting a new qtconsole connected to our kernel", "response": "def new_qt_console(self, evt=None):\n \"\"\"start a new qtconsole connected to our kernel\"\"\"\n return connect_qtconsole(self.ipkernel.connection_file, profile=self.ipkernel.profile)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck whether the URL is accessible and returns HTTP 200 OK or raises ValidationError", "response": "def check_url_accessibility(url, timeout=10): \n '''\n Check whether the URL accessible and returns HTTP 200 OK or not\n if not raises ValidationError\n '''\n if(url=='localhost'):\n url = 'http://127.0.0.1'\n try:\n req = urllib2.urlopen(url, timeout=timeout)\n if (req.getcode()==200):\n return True\n except Exception:\n pass\n fail(\"URL '%s' is not accessible from this machine\" % url)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef url_has_contents(url, contents, case_sensitive=False, timeout=10):\n ''' \n Check whether the HTML page contains the content or not and return boolean\n '''\n try: \n req = urllib2.urlopen(url, timeout=timeout)\n except Exception, _:\n False\n else:\n rep = req.read()\n if (not case_sensitive and rep.lower().find(contents.lower()) >= 0) or (case_sensitive and rep.find(contents) >= 0):\n return True\n else:\n return False", "response": "Check whether the URL contains the contents or not and return boolean\n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the HTTP response code of the given URL.", "response": "def get_response_code(url, timeout=10):\n '''\n Visit the URL and return the HTTP response code in 'int'\n '''\n try: \n req = urllib2.urlopen(url, timeout=timeout)\n except HTTPError, e:\n return e.getcode()\n except Exception, _:\n fail(\"Couldn't reach the URL '%s'\" % url)\n else:\n return req.getcode()"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncompares the content type header of url param with content_type param and returns boolean", "response": "def compare_content_type(url, content_type):\n '''\n Compare the content type header of url param with content_type param and returns boolean \n @param url -> string e.g. http://127.0.0.1/index\n @param content_type -> string e.g. text/html\n '''\n try: \n response = urllib2.urlopen(url)\n except:\n return False\n\n return response.headers.type == content_type"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomparing the response code of url param with code param and returns boolean", "response": "def compare_response_code(url, code):\n '''\n Compare the response code of url param with code param and returns boolean \n @param url -> string e.g. http://127.0.0.1/index\n @param content_type -> int e.g. 404, 500, 400 ..etc\n '''\n try:\n response = urllib2.urlopen(url)\n \n except HTTPError as e:\n return e.code == code\n \n except:\n return False\n\n return response.code == code"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npublish data and metadata to all frontends.", "response": "def publish_display_data(source, data, metadata=None):\n \"\"\"Publish data and metadata to all frontends.\n\n See the ``display_data`` message in the messaging documentation for\n more details about this message type.\n\n The following MIME types are currently implemented:\n\n * text/plain\n * text/html\n * text/latex\n * application/json\n * application/javascript\n * image/png\n * image/jpeg\n * image/svg+xml\n\n Parameters\n ----------\n source : str\n A string that give the function or method that created the data,\n such as 'IPython.core.page'.\n data : dict\n A dictionary having keys that are valid MIME types (like\n 'text/plain' or 'image/svg+xml') and values that are the data for\n that MIME type. The data itself must be a JSON'able data\n structure. Minimally all data should have the 'text/plain' data,\n which can be displayed by all frontends. If more than the plain\n text is given, it is up to the frontend to decide which\n representation to use.\n metadata : dict\n A dictionary for metadata related to the data. This can contain\n arbitrary key, value pairs that frontends can use to interpret\n the data.\n \"\"\"\n from IPython.core.interactiveshell import InteractiveShell\n InteractiveShell.instance().display_pub.publish(\n source,\n data,\n metadata\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _validate_data(self, source, data, metadata=None):\n\n if not isinstance(source, basestring):\n raise TypeError('source must be a str, got: %r' % source)\n if not isinstance(data, dict):\n raise TypeError('data must be a dict, got: %r' % data)\n if metadata is not None:\n if not isinstance(metadata, dict):\n raise TypeError('metadata must be a dict, got: %r' % data)", "response": "Validate the display data."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef publish(self, source, data, metadata=None):\n\n # The default is to simply write the plain text data using io.stdout.\n if data.has_key('text/plain'):\n print(data['text/plain'], file=io.stdout)", "response": "Publish data and metadata to all frontends."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef clear_output(self, stdout=True, stderr=True, other=True):\n if stdout:\n print('\\033[2K\\r', file=io.stdout, end='')\n io.stdout.flush()\n if stderr:\n print('\\033[2K\\r', file=io.stderr, end='')\n io.stderr.flush()", "response": "Clear the output of the cell receiving output."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn numerical gradient function of given input function f", "response": "def grad(f, delta=DELTA):\n \"\"\"\n Returns numerical gradient function of given input function\n Input: f, scalar function of one or two variables\n delta(optional), finite difference step\n Output: gradient function object\n \"\"\"\n def grad_f(*args, **kwargs):\n if len(args) == 1:\n x, = args\n gradf_x = (\n f(x+delta/2) - f(x-delta/2)\n )/delta\n return gradf_x\n elif len(args) == 2:\n x, y = args\n if type(x) in [float, int] and type(y) in [float, int]:\n gradf_x = (f(x + delta/2, y) - f(x - delta/2, y))/delta\n gradf_y = (f(x, y + delta/2) - f(x, y - delta/2))/delta\n return gradf_x, gradf_y\n return grad_f"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns numerical hessian of a given function f on the object", "response": "def hessian(f, delta=DELTA):\n \"\"\"\n Returns numerical hessian function of given input function\n Input: f, scalar function of one or two variables\n delta(optional), finite difference step\n Output: hessian function object\n \"\"\"\n def hessian_f(*args, **kwargs):\n if len(args) == 1:\n x, = args\n hessianf_x = (\n f(x+delta) + f(x-delta) - 2*f(x)\n )/delta**2\n return hessianf_x\n elif len(args) == 2:\n x, y = args\n if type(x) in [float, int] and type(y) in [float, int]:\n hess_xx = (\n f(x + delta, y) + f(x - delta, y) - 2*f(x, y)\n )/delta**2\n hess_yy = (\n f(x, y + delta) + f(x, y - delta) - 2*f(x, y)\n )/delta**2\n hess_xy = (\n + f(x+delta/2, y+delta/2)\n + f(x-delta/2, y-delta/2)\n - f(x+delta/2, y-delta/2)\n - f(x-delta/2, y+delta/2)\n )/delta**2\n return hess_xx, hess_xy, hess_yy\n return hessian_f"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns numerical gradient function of given input function f delta", "response": "def ndgrad(f, delta=DELTA):\n \"\"\"\n Returns numerical gradient function of given input function\n Input: f, scalar function of an numpy array object\n delta(optional), finite difference step\n Output: gradient function object\n \"\"\"\n def grad_f(*args, **kwargs):\n x = args[0]\n grad_val = numpy.zeros(x.shape)\n it = numpy.nditer(x, op_flags=['readwrite'], flags=['multi_index'])\n for xi in it:\n i = it.multi_index\n xi += delta/2\n fp = f(*args, **kwargs)\n xi -= delta\n fm = f(*args, **kwargs)\n xi += delta/2\n grad_val[i] = (fp - fm)/delta\n return grad_val\n return grad_f"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ndhess(f, delta=DELTA):\n def hess_f(*args, **kwargs):\n x = args[0]\n hess_val = numpy.zeros(x.shape + x.shape)\n it = numpy.nditer(x, op_flags=['readwrite'], flags=['multi_index'])\n for xi in it:\n i = it.multi_index\n jt = numpy.nditer(x, op_flags=['readwrite'], flags=['multi_index'])\n for xj in jt:\n j = jt.multi_index\n xi += delta/2\n xj += delta/2\n fpp = f(x)\n xj -= delta\n fpm = f(x)\n xi -= delta\n fmm = f(x)\n xj += delta\n fmp = f(x)\n xi += delta/2\n xj -= delta/2\n hess_val[i + j] = (fpp + fmm - fpm - fmp)/delta**2\n return hess_val\n return hess_f", "response": "Returns numerical hessian of given input function f"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns numerical gradient function of given class method and attribute with respect to a class attribute.", "response": "def clgrad(obj, exe, arg, delta=DELTA):\n \"\"\"\n Returns numerical gradient function of given class method\n with respect to a class attribute\n Input: obj, general object\n exe (str), name of object method\n arg (str), name of object atribute\n delta(float, optional), finite difference step\n Output: gradient function object\n \"\"\"\n f, x = get_method_and_copy_of_attribute(obj, exe, arg)\n def grad_f(*args, **kwargs):\n grad_val = numpy.zeros(x.shape)\n it = numpy.nditer(x, op_flags=['readwrite'], flags=['multi_index'])\n for xi in it:\n i = it.multi_index\n xi += delta/2\n fp = f(*args, **kwargs)\n xi -= delta\n fm = f(*args, **kwargs)\n xi += delta/2\n grad_val[i] = (fp - fm)/delta\n return grad_val\n return grad_f"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns numerical mixed Hessian function of given class method and two class attributes.", "response": "def clmixhess(obj, exe, arg1, arg2, delta=DELTA):\n \"\"\"\n Returns numerical mixed Hessian function of given class method\n with respect to two class attributes\n Input: obj, general object\n exe (str), name of object method\n arg1(str), name of object attribute\n arg2(str), name of object attribute\n delta(float, optional), finite difference step\n Output: Hessian function object\n \"\"\"\n f, x = get_method_and_copy_of_attribute(obj, exe, arg1)\n _, y = get_method_and_copy_of_attribute(obj, exe, arg2)\n def hess_f(*args, **kwargs):\n hess_val = numpy.zeros(x.shape + y.shape)\n it = numpy.nditer(x, op_flags=['readwrite'], flags=['multi_index'])\n for xi in it:\n i = it.multi_index\n jt = numpy.nditer(y, op_flags=['readwrite'], flags=['multi_index'])\n for yj in jt:\n j = jt.multi_index\n xi += delta/2\n yj += delta/2\n fpp = f(*args, **kwargs)\n yj -= delta\n fpm = f(*args, **kwargs)\n xi -= delta\n fmm = f(*args, **kwargs)\n yj += delta\n fmp = f(*args, **kwargs)\n xi += delta/2\n yj -= delta/2\n hess_val[i + j] = (fpp + fmm - fpm - fmp)/delta**2\n return hess_val\n return hess_f"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_cmd(cmd):\n if cmd == 'python':\n return os.path.abspath(sys.executable)\n try:\n path = _find_cmd(cmd).rstrip()\n except OSError:\n raise FindCmdError('command could not be found: %s' % cmd)\n # which returns empty if not found\n if path == '':\n raise FindCmdError('command could not be found: %s' % cmd)\n return os.path.abspath(path)", "response": "Find absolute path to executable cmd in a cross platform manner."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pycmd2argv(cmd):\n ext = os.path.splitext(cmd)[1]\n if ext in ['.exe', '.com', '.bat']:\n return [cmd]\n else:\n return [sys.executable, cmd]", "response": "Take the path of a python command and return a list of argv - style strings."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn abbreviated version of cwd e. g. d mydir", "response": "def abbrev_cwd():\n \"\"\" Return abbreviated version of cwd, e.g. d:mydir \"\"\"\n cwd = os.getcwdu().replace('\\\\','/')\n drivepart = ''\n tail = cwd\n if sys.platform == 'win32':\n if len(cwd) < 4:\n return cwd\n drivepart,tail = os.path.splitdrive(cwd)\n\n\n parts = tail.split('/')\n if len(parts) > 2:\n tail = '/'.join(parts[-2:])\n\n return (drivepart + (\n cwd == '/' and '/' or tail))"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef code_unit_factory(morfs, file_locator):\n # Be sure we have a list.\n if not isinstance(morfs, (list, tuple)):\n morfs = [morfs]\n\n # On Windows, the shell doesn't expand wildcards. Do it here.\n globbed = []\n for morf in morfs:\n if isinstance(morf, string_class) and ('?' in morf or '*' in morf):\n globbed.extend(glob.glob(morf))\n else:\n globbed.append(morf)\n morfs = globbed\n\n code_units = [CodeUnit(morf, file_locator) for morf in morfs]\n\n return code_units", "response": "Construct a list of CodeUnits from a list of polymorphic inputs."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef source_file(self):\n if os.path.exists(self.filename):\n # A regular text file: open it.\n return open_source(self.filename)\n\n # Maybe it's in a zip file?\n source = self.file_locator.get_zip_data(self.filename)\n if source is not None:\n return StringIO(source)\n\n # Couldn't find source.\n raise CoverageException(\n \"No source for code '%s'.\" % self.filename\n )", "response": "Return an open file for reading the source of the code unit."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef should_be_python(self):\n # Get the file extension.\n _, ext = os.path.splitext(self.filename)\n\n # Anything named *.py* should be Python.\n if ext.startswith('.py'):\n return True\n # A file with no extension should be Python.\n if not ext:\n return True\n # Everything else is probably not Python.\n return False", "response": "Does this file seem like this file contains Python?"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalls the method f with the current object as its first argument.", "response": "def check_ready(f, self, *args, **kwargs):\n \"\"\"Call spin() to sync state prior to calling the method.\"\"\"\n self.wait(0)\n if not self._ready:\n raise error.TimeoutError(\"result not ready\")\n return f(self, *args, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the result when it arrives within timeout seconds.", "response": "def get(self, timeout=-1):\n \"\"\"Return the result when it arrives.\n\n If `timeout` is not ``None`` and the result does not arrive within\n `timeout` seconds then ``TimeoutError`` is raised. If the\n remote call raised an exception then that exception will be reraised\n by get() inside a `RemoteError`.\n \"\"\"\n if not self.ready():\n self.wait(timeout)\n\n if self._ready:\n if self._success:\n return self._result\n else:\n raise self._exception\n else:\n raise error.TimeoutError(\"Result not ready.\")"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wait(self, timeout=-1):\n if self._ready:\n return\n self._ready = self._client.wait(self.msg_ids, timeout)\n if self._ready:\n try:\n results = map(self._client.results.get, self.msg_ids)\n self._result = results\n if self._single_result:\n r = results[0]\n if isinstance(r, Exception):\n raise r\n else:\n results = error.collect_exceptions(results, self._fname)\n self._result = self._reconstruct_result(results)\n except Exception, e:\n self._exception = e\n self._success = False\n else:\n self._success = True\n finally:\n self._metadata = map(self._client.metadata.get, self.msg_ids)\n self._wait_for_outputs(10)", "response": "Wait until the result is available or until timeout seconds pass."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the results as a dict keyed by engine_id.", "response": "def get_dict(self, timeout=-1):\n \"\"\"Get the results as a dict, keyed by engine_id.\n\n timeout behavior is described in `get()`.\n \"\"\"\n\n results = self.get(timeout)\n engine_ids = [ md['engine_id'] for md in self._metadata ]\n bycount = sorted(engine_ids, key=lambda k: engine_ids.count(k))\n maxcount = bycount.count(bycount[-1])\n if maxcount > 1:\n raise ValueError(\"Cannot build dict, %i jobs ran on engine #%i\"%(\n maxcount, bycount[-1]))\n\n return dict(zip(engine_ids,results))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef timedelta(self, start, end, start_key=min, end_key=max):\n if not isinstance(start, datetime):\n # handle single_result AsyncResults, where ar.stamp is single object,\n # not a list\n start = start_key(start)\n if not isinstance(end, datetime):\n # handle single_result AsyncResults, where ar.stamp is single object,\n # not a list\n end = end_key(end)\n return _total_seconds(end - start)", "response": "compute the difference between two set of timestamps"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef progress(self):\n self.wait(0)\n return len(self) - len(set(self.msg_ids).intersection(self._client.outstanding))", "response": "Returns the number of tasks which have been completed at this point."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef elapsed(self):\n if self.ready():\n return self.wall_time\n \n now = submitted = datetime.now()\n for msg_id in self.msg_ids:\n if msg_id in self._client.metadata:\n stamp = self._client.metadata[msg_id]['submitted']\n if stamp and stamp < submitted:\n submitted = stamp\n return _total_seconds(now-submitted)", "response": "elapsed time since initial submission"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef serial_time(self):\n t = 0\n for md in self._metadata:\n t += _total_seconds(md['completed'] - md['started'])\n return t", "response": "Computes the total serial time of a parallel calculation\n \n Computed as the sum of completed - started of each task\n "} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wait_interactive(self, interval=1., timeout=None):\n N = len(self)\n tic = time.time()\n while not self.ready() and (timeout is None or time.time() - tic <= timeout):\n self.wait(interval)\n clear_output()\n print(\"%4i/%i tasks finished after %4i s\" % (self.progress, N, self.elapsed), end=\"\")\n sys.stdout.flush()\n print()\n print(\"done\")", "response": "wait for all tasks to finish"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nrepublishes individual displaypub content dicts", "response": "def _republish_displaypub(self, content, eid):\n \"\"\"republish individual displaypub content dicts\"\"\"\n try:\n ip = get_ipython()\n except NameError:\n # displaypub is meaningless outside IPython\n return\n md = content['metadata'] or {}\n md['engine'] = eid\n ip.display_pub.publish(content['source'], content['data'], md)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwaits for all outputs to be ready", "response": "def _wait_for_outputs(self, timeout=-1):\n \"\"\"wait for the 'status=idle' message that indicates we have all outputs\n \"\"\"\n if not self._success:\n # don't wait on errors\n return\n tic = time.time()\n while not all(md['outputs_ready'] for md in self._metadata):\n time.sleep(0.01)\n self._client._flush_iopub(self._client._iopub_socket)\n if timeout >= 0 and time.time() > tic + timeout:\n break"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef display_outputs(self, groupby=\"type\"):\n if self._single_result:\n self._display_single_result()\n return\n \n stdouts = self.stdout\n stderrs = self.stderr\n pyouts = self.pyout\n output_lists = self.outputs\n results = self.get()\n \n targets = self.engine_id\n \n if groupby == \"engine\":\n for eid,stdout,stderr,outputs,r,pyout in zip(\n targets, stdouts, stderrs, output_lists, results, pyouts\n ):\n self._display_stream(stdout, '[stdout:%i] ' % eid)\n self._display_stream(stderr, '[stderr:%i] ' % eid, file=sys.stderr)\n \n try:\n get_ipython()\n except NameError:\n # displaypub is meaningless outside IPython\n return \n \n if outputs or pyout is not None:\n _raw_text('[output:%i]' % eid)\n \n for output in outputs:\n self._republish_displaypub(output, eid)\n \n if pyout is not None:\n display(r)\n \n elif groupby in ('type', 'order'):\n # republish stdout:\n for eid,stdout in zip(targets, stdouts):\n self._display_stream(stdout, '[stdout:%i] ' % eid)\n \n # republish stderr:\n for eid,stderr in zip(targets, stderrs):\n self._display_stream(stderr, '[stderr:%i] ' % eid, file=sys.stderr)\n \n try:\n get_ipython()\n except NameError:\n # displaypub is meaningless outside IPython\n return\n \n if groupby == 'order':\n output_dict = dict((eid, outputs) for eid,outputs in zip(targets, output_lists))\n N = max(len(outputs) for outputs in output_lists)\n for i in range(N):\n for eid in targets:\n outputs = output_dict[eid]\n if len(outputs) >= N:\n _raw_text('[output:%i]' % eid)\n self._republish_displaypub(outputs[i], eid)\n else:\n # republish displaypub output\n for eid,outputs in zip(targets, output_lists):\n if outputs:\n _raw_text('[output:%i]' % eid)\n for output in outputs:\n self._republish_displaypub(output, eid)\n \n # finally, add pyout:\n for eid,r,pyout in zip(targets, results, pyouts):\n if pyout is not None:\n display(r)\n \n else:\n raise ValueError(\"groupby must be one of 'type', 'engine', 'collate', not %r\" % groupby)", "response": "display the outputs of the computation\n "} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _unordered_iter(self):\n try:\n rlist = self.get(0)\n except error.TimeoutError:\n pending = set(self.msg_ids)\n while pending:\n try:\n self._client.wait(pending, 1e-3)\n except error.TimeoutError:\n # ignore timeout error, because that only means\n # *some* jobs are outstanding\n pass\n # update ready set with those no longer outstanding:\n ready = pending.difference(self._client.outstanding)\n # update pending to exclude those that are finished\n pending = pending.difference(ready)\n while ready:\n msg_id = ready.pop()\n ar = AsyncResult(self._client, msg_id, self._fname)\n rlist = ar.get()\n try:\n for r in rlist:\n yield r\n except TypeError:\n # flattened, not a list\n # this could get broken by flattened data that returns iterables\n # but most calls to map do not expose the `flatten` argument\n yield rlist\n else:\n # already done\n for r in rlist:\n yield r", "response": "iterator for results as they arrive on FCFS basis ignoring submission order."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef wait(self, timeout=-1):\n start = time.time()\n if self._ready:\n return\n local_ids = filter(lambda msg_id: msg_id in self._client.outstanding, self.msg_ids)\n local_ready = self._client.wait(local_ids, timeout)\n if local_ready:\n remote_ids = filter(lambda msg_id: msg_id not in self._client.results, self.msg_ids)\n if not remote_ids:\n self._ready = True\n else:\n rdict = self._client.result_status(remote_ids, status_only=False)\n pending = rdict['pending']\n while pending and (timeout < 0 or time.time() < start+timeout):\n rdict = self._client.result_status(remote_ids, status_only=False)\n pending = rdict['pending']\n if pending:\n time.sleep(0.1)\n if not pending:\n self._ready = True\n if self._ready:\n try:\n results = map(self._client.results.get, self.msg_ids)\n self._result = results\n if self._single_result:\n r = results[0]\n if isinstance(r, Exception):\n raise r\n else:\n results = error.collect_exceptions(results, self._fname)\n self._result = self._reconstruct_result(results)\n except Exception, e:\n self._exception = e\n self._success = False\n else:\n self._success = True\n finally:\n self._metadata = map(self._client.metadata.get, self.msg_ids)", "response": "wait for result to complete."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the absolute normalized form of filename.", "response": "def abs_file(filename):\n \"\"\"Return the absolute normalized form of `filename`.\"\"\"\n path = os.path.expandvars(os.path.expanduser(filename))\n path = os.path.abspath(os.path.realpath(path))\n path = actual_path(path)\n return path"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npreparing the file patterns for use in a FnmatchMatcher.", "response": "def prep_patterns(patterns):\n \"\"\"Prepare the file patterns for use in a `FnmatchMatcher`.\n\n If a pattern starts with a wildcard, it is used as a pattern\n as-is. If it does not start with a wildcard, then it is made\n absolute with the current directory.\n\n If `patterns` is None, an empty list is returned.\n\n \"\"\"\n prepped = []\n for p in patterns or []:\n if p.startswith(\"*\") or p.startswith(\"?\"):\n prepped.append(p)\n else:\n prepped.append(abs_file(p))\n return prepped"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sep(s):\n sep_match = re.search(r\"[\\\\/]\", s)\n if sep_match:\n the_sep = sep_match.group(0)\n else:\n the_sep = os.sep\n return the_sep", "response": "Find the path separator used in this string or os. sep if none is found."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef find_python_files(dirname):\n for i, (dirpath, dirnames, filenames) in enumerate(os.walk(dirname)):\n if i > 0 and '__init__.py' not in filenames:\n # If a directory doesn't have __init__.py, then it isn't\n # importable and neither are its files\n del dirnames[:]\n continue\n for filename in filenames:\n # We're only interested in files that look like reasonable Python\n # files: Must end with .py or .pyw, and must not have certain funny\n # characters that probably mean they are editor junk.\n if re.match(r\"^[^.#~!$@%^&*()+=,]+\\.pyw?$\", filename):\n yield os.path.join(dirpath, filename)", "response": "Yield all of the importable Python files in a directory recursively."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef relative_filename(self, filename):\n fnorm = os.path.normcase(filename)\n if fnorm.startswith(self.relative_dir):\n filename = filename[len(self.relative_dir):]\n return filename", "response": "Return the relative form of filename."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef canonical_filename(self, filename):\n if filename not in self.canonical_filename_cache:\n if not os.path.isabs(filename):\n for path in [os.curdir] + sys.path:\n if path is None:\n continue\n f = os.path.join(path, filename)\n if os.path.exists(f):\n filename = f\n break\n cf = abs_file(filename)\n self.canonical_filename_cache[filename] = cf\n return self.canonical_filename_cache[filename]", "response": "Return a canonical filename for filename."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_zip_data(self, filename):\n import zipimport\n markers = ['.zip'+os.sep, '.egg'+os.sep]\n for marker in markers:\n if marker in filename:\n parts = filename.split(marker)\n try:\n zi = zipimport.zipimporter(parts[0]+marker[:-1])\n except zipimport.ZipImportError:\n continue\n try:\n data = zi.get_data(parts[1])\n except IOError:\n continue\n return to_string(data)\n return None", "response": "Get data from filename if it is a zip file path."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndo fpath indicate a file in one of our trees?", "response": "def match(self, fpath):\n \"\"\"Does `fpath` indicate a file in one of our trees?\"\"\"\n for d in self.dirs:\n if fpath.startswith(d):\n if fpath == d:\n # This is the same file!\n return True\n if fpath[len(d)] == os.sep:\n # This is a file in the directory\n return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndoing fpath match one of our filename patterns?", "response": "def match(self, fpath):\n \"\"\"Does `fpath` match one of our filename patterns?\"\"\"\n for pat in self.pats:\n if fnmatch.fnmatch(fpath, pat):\n return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add(self, pattern, result):\n # The pattern can't end with a wildcard component.\n pattern = pattern.rstrip(r\"\\/\")\n if pattern.endswith(\"*\"):\n raise CoverageException(\"Pattern must not end with wildcards.\")\n pattern_sep = sep(pattern)\n\n # The pattern is meant to match a filepath. Let's make it absolute\n # unless it already is, or is meant to match any prefix.\n if not pattern.startswith('*') and not isabs_anywhere(pattern):\n pattern = abs_file(pattern)\n pattern += pattern_sep\n\n # Make a regex from the pattern. fnmatch always adds a \\Z or $ to\n # match the whole string, which we don't want.\n regex_pat = fnmatch.translate(pattern).replace(r'\\Z(', '(')\n if regex_pat.endswith(\"$\"):\n regex_pat = regex_pat[:-1]\n # We want */a/b.py to match on Windows too, so change slash to match\n # either separator.\n regex_pat = regex_pat.replace(r\"\\/\", r\"[\\\\/]\")\n # We want case-insensitive matching, so add that flag.\n regex = re.compile(r\"(?i)\" + regex_pat)\n\n # Normalize the result: it must end with a path separator.\n result_sep = sep(result)\n result = result.rstrip(r\"\\/\") + result_sep\n self.aliases.append((regex, result, pattern_sep, result_sep))", "response": "Add the pattern and result pair to the list of aliases."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef map(self, path):\n for regex, result, pattern_sep, result_sep in self.aliases:\n m = regex.match(path)\n if m:\n new = path.replace(m.group(0), result)\n if pattern_sep != result_sep:\n new = new.replace(pattern_sep, result_sep)\n if self.locator:\n new = self.locator.canonical_filename(new)\n return new\n return path", "response": "Map a path through the aliases."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef loop_qt4(kernel):\n\n from IPython.external.qt_for_kernel import QtCore\n from IPython.lib.guisupport import get_app_qt4, start_event_loop_qt4\n\n kernel.app = get_app_qt4([\" \"])\n kernel.app.setQuitOnLastWindowClosed(False)\n kernel.timer = QtCore.QTimer()\n kernel.timer.timeout.connect(kernel.do_one_iteration)\n # Units for the timer are in milliseconds\n kernel.timer.start(1000*kernel._poll_interval)\n start_event_loop_qt4(kernel.app)", "response": "Start a kernel with PyQt4 event loop integration."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstarting a kernel with wx event loop support.", "response": "def loop_wx(kernel):\n \"\"\"Start a kernel with wx event loop support.\"\"\"\n\n import wx\n from IPython.lib.guisupport import start_event_loop_wx\n\n doi = kernel.do_one_iteration\n # Wx uses milliseconds\n poll_interval = int(1000*kernel._poll_interval)\n\n # We have to put the wx.Timer in a wx.Frame for it to fire properly.\n # We make the Frame hidden when we create it in the main app below.\n class TimerFrame(wx.Frame):\n def __init__(self, func):\n wx.Frame.__init__(self, None, -1)\n self.timer = wx.Timer(self)\n # Units for the timer are in milliseconds\n self.timer.Start(poll_interval)\n self.Bind(wx.EVT_TIMER, self.on_timer)\n self.func = func\n\n def on_timer(self, event):\n self.func()\n\n # We need a custom wx.App to create our Frame subclass that has the\n # wx.Timer to drive the ZMQ event loop.\n class IPWxApp(wx.App):\n def OnInit(self):\n self.frame = TimerFrame(doi)\n self.frame.Show(False)\n return True\n\n # The redirect=False here makes sure that wx doesn't replace\n # sys.stdout/stderr with its own classes.\n kernel.app = IPWxApp(redirect=False)\n\n # The import of wx on Linux sets the handler for signal.SIGINT\n # to 0. This is a bug in wx or gtk. We fix by just setting it\n # back to the Python default.\n import signal\n if not callable(signal.getsignal(signal.SIGINT)):\n signal.signal(signal.SIGINT, signal.default_int_handler)\n\n start_event_loop_wx(kernel.app)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstarts a kernel with the Tk event loop.", "response": "def loop_tk(kernel):\n \"\"\"Start a kernel with the Tk event loop.\"\"\"\n\n import Tkinter\n doi = kernel.do_one_iteration\n # Tk uses milliseconds\n poll_interval = int(1000*kernel._poll_interval)\n # For Tkinter, we create a Tk object and call its withdraw method.\n class Timer(object):\n def __init__(self, func):\n self.app = Tkinter.Tk()\n self.app.withdraw()\n self.func = func\n\n def on_timer(self):\n self.func()\n self.app.after(poll_interval, self.on_timer)\n\n def start(self):\n self.on_timer() # Call it once to get things going.\n self.app.mainloop()\n\n kernel.timer = Timer(doi)\n kernel.timer.start()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef loop_gtk(kernel):\n from .gui.gtkembed import GTKEmbed\n\n gtk_kernel = GTKEmbed(kernel)\n gtk_kernel.start()", "response": "Start the kernel coordinating with the GTK event loop"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting the kernel and return a CFRunLoop event loop for the Cocoa event loop.", "response": "def loop_cocoa(kernel):\n \"\"\"Start the kernel, coordinating with the Cocoa CFRunLoop event loop\n via the matplotlib MacOSX backend.\n \"\"\"\n import matplotlib\n if matplotlib.__version__ < '1.1.0':\n kernel.log.warn(\n \"MacOSX backend in matplotlib %s doesn't have a Timer, \"\n \"falling back on Tk for CFRunLoop integration. Note that \"\n \"even this won't work if Tk is linked against X11 instead of \"\n \"Cocoa (e.g. EPD). To use the MacOSX backend in the kernel, \"\n \"you must use matplotlib >= 1.1.0, or a native libtk.\"\n )\n return loop_tk(kernel)\n \n from matplotlib.backends.backend_macosx import TimerMac, show\n\n # scale interval for sec->ms\n poll_interval = int(1000*kernel._poll_interval)\n\n real_excepthook = sys.excepthook\n def handle_int(etype, value, tb):\n \"\"\"don't let KeyboardInterrupts look like crashes\"\"\"\n if etype is KeyboardInterrupt:\n io.raw_print(\"KeyboardInterrupt caught in CFRunLoop\")\n else:\n real_excepthook(etype, value, tb)\n\n # add doi() as a Timer to the CFRunLoop\n def doi():\n # restore excepthook during IPython code\n sys.excepthook = real_excepthook\n kernel.do_one_iteration()\n # and back:\n sys.excepthook = handle_int\n\n t = TimerMac(poll_interval)\n t.add_callback(doi)\n t.start()\n\n # but still need a Poller for when there are no active windows,\n # during which time mainloop() returns immediately\n poller = zmq.Poller()\n if kernel.control_stream:\n poller.register(kernel.control_stream.socket, zmq.POLLIN)\n for stream in kernel.shell_streams:\n poller.register(stream.socket, zmq.POLLIN)\n\n while True:\n try:\n # double nested try/except, to properly catch KeyboardInterrupt\n # due to pyzmq Issue #130\n try:\n # don't let interrupts during mainloop invoke crash_handler:\n sys.excepthook = handle_int\n show.mainloop()\n sys.excepthook = real_excepthook\n # use poller if mainloop returned (no windows)\n # scale by extra factor of 10, since it's a real poll\n poller.poll(10*poll_interval)\n kernel.do_one_iteration()\n except:\n raise\n except KeyboardInterrupt:\n # Ctrl-C shouldn't crash the kernel\n io.raw_print(\"KeyboardInterrupt caught in kernel\")\n finally:\n # ensure excepthook is restored\n sys.excepthook = real_excepthook"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef enable_gui(gui, kernel=None):\n if gui not in loop_map:\n raise ValueError(\"GUI %r not supported\" % gui)\n if kernel is None:\n if Application.initialized():\n kernel = getattr(Application.instance(), 'kernel', None)\n if kernel is None:\n raise RuntimeError(\"You didn't specify a kernel,\"\n \" and no IPython Application with a kernel appears to be running.\"\n )\n loop = loop_map[gui]\n if kernel.eventloop is not None and kernel.eventloop is not loop:\n raise RuntimeError(\"Cannot activate multiple GUI eventloops\")\n kernel.eventloop = loop", "response": "Enable integration with a given GUI"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GOE(N):\n m = ra.standard_normal((N,N))\n m += m.T\n return m/2", "response": "Creates an NxN element of the Gaussian Orthogonal Ensemble"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef center_eigenvalue_diff(mat):\n N = len(mat)\n evals = np.sort(la.eigvals(mat))\n diff = np.abs(evals[N/2] - evals[N/2-1])\n return diff", "response": "Compute the eigvals of mat and then find the center eigval difference."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ensemble_diffs(num, N):\n diffs = np.empty(num)\n for i in xrange(num):\n mat = GOE(N)\n diffs[i] = center_eigenvalue_diff(mat)\n return diffs", "response": "Return num eigenvalue diffs for the NxN GOE ensemble."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing the item. This calls the class constructor with the appropriate arguments and returns the initialized object.", "response": "def init(self, ctxt, step_addr):\n \"\"\"\n Initialize the item. This calls the class constructor with the\n appropriate arguments and returns the initialized object.\n\n :param ctxt: The context object.\n :param step_addr: The address of the step in the test\n configuration.\n \"\"\"\n\n return self.cls(ctxt, self.name, self.conf, step_addr)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse_file(cls, ctxt, fname, key=None, step_addr=None):\n\n # Load the YAML file\n try:\n with open(fname) as f:\n step_data = yaml.load(f)\n except Exception as exc:\n raise ConfigError(\n 'Failed to read file \"%s\": %s' % (fname, exc),\n step_addr,\n )\n\n # Do we have a key?\n if key is not None:\n if (not isinstance(step_data, collections.Mapping) or\n key not in step_data):\n raise ConfigError(\n 'Bad step configuration file \"%s\": expecting dictionary '\n 'with key \"%s\"' % (fname, key),\n step_addr,\n )\n\n # Extract just the step data\n step_data = step_data[key]\n\n # Validate that it's a sequence\n if not isinstance(step_data, collections.Sequence):\n addr = ('%s[%s]' % (fname, key)) if key is not None else fname\n raise ConfigError(\n 'Bad step configuration sequence at %s: expecting list, '\n 'not \"%s\"' % (addr, step_data.__class__.__name__),\n step_addr,\n )\n\n # OK, assemble the step list and return it\n steps = []\n for idx, step_conf in enumerate(step_data):\n steps.extend(cls.parse_step(\n ctxt, StepAddress(fname, idx, key), step_conf))\n\n return steps", "response": "Parse a YAML file containing test steps."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse a step dictionary into a list of steps.", "response": "def parse_step(cls, ctxt, step_addr, step_conf):\n \"\"\"\n Parse a step dictionary.\n\n :param ctxt: The context object.\n :param step_addr: The address of the step in the test\n configuration.\n :param step_conf: The description of the step. This may be a\n scalar string or a dictionary.\n\n :returns: A list of steps.\n \"\"\"\n\n # Make sure the step makes sense\n if isinstance(step_conf, six.string_types):\n # Convert string to a dict for uniformity of processing\n step_conf = {step_conf: None}\n elif not isinstance(step_conf, collections.Mapping):\n raise ConfigError(\n 'Unable to parse step configuration: expecting string or '\n 'dictionary, not \"%s\"' % step_conf.__class__.__name__,\n step_addr,\n )\n\n # Parse the configuration into the action and modifier classes\n # and the configuration to apply to each\n action_item = None\n mod_items = {}\n kwargs = {} # extra args for Step.__init__()\n for key, key_conf in step_conf.items():\n # Handle special keys first\n if key in cls.schemas:\n # Validate the key\n utils.schema_validate(key_conf, cls.schemas[key], ConfigError,\n key, step_addr=step_addr)\n\n # Save the value\n kwargs[key] = key_conf\n\n # Is it an action?\n elif key in entry.points[NAMESPACE_ACTION]:\n if action_item is not None:\n raise ConfigError(\n 'Bad step configuration: action \"%s\" specified, '\n 'but action \"%s\" already processed' %\n (key, action_item.name),\n step_addr,\n )\n\n action_item = StepItem(\n entry.points[NAMESPACE_ACTION][key], key, key_conf)\n\n # OK, is it a modifier?\n elif key in entry.points[NAMESPACE_MODIFIER]:\n mod_class = entry.points[NAMESPACE_MODIFIER][key]\n\n # Store it in priority order\n mod_items.setdefault(mod_class.priority, [])\n mod_items[mod_class.priority].append(StepItem(\n mod_class, key, key_conf))\n\n # Couldn't resolve it\n else:\n raise ConfigError(\n 'Bad step configuration: unable to resolve action '\n '\"%s\"' % key,\n step_addr,\n )\n\n # Make sure we have an action\n if action_item is None:\n raise ConfigError(\n 'Bad step configuration: no action specified',\n step_addr,\n )\n\n # What is the action type?\n action_type = (Modifier.STEP if action_item.cls.step_action\n else Modifier.NORMAL)\n\n # OK, build our modifiers list and preprocess the action\n # configuration\n modifiers = []\n for mod_item in utils.iter_prio_dict(mod_items):\n # Verify that the modifier is compatible with the\n # action\n if mod_item.cls.restriction & action_type == 0:\n raise ConfigError(\n 'Bad step configuration: modifier \"%s\" is '\n 'incompatible with the action \"%s\"' %\n (mod_item.name, action_item.name),\n step_addr,\n )\n\n # Initialize the modifier\n modifier = mod_item.init(ctxt, step_addr)\n\n # Add it to the list of modifiers\n modifiers.append(modifier)\n\n # Apply the modifier's configuration processing\n action_item.conf = modifier.action_conf(\n ctxt, action_item.cls, action_item.name, action_item.conf,\n step_addr)\n\n # Now we can initialize the action\n action = action_item.init(ctxt, step_addr)\n\n # Create the step\n step = cls(step_addr, action, modifiers, **kwargs)\n\n # If the final_action is a StepAction, invoke it now and\n # return the list of steps. We do this after creating the\n # Step object so that we can take advantage of its handling of\n # modifiers.\n if action_item.cls.step_action:\n return step(ctxt)\n\n # Not a step action, return the step as a list of one element\n return [step]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef init_crash_handler(self):\n self.crash_handler = self.crash_handler_class(self)\n sys.excepthook = self.excepthook\n def unset_crashhandler():\n sys.excepthook = sys.__excepthook__\n atexit.register(unset_crashhandler)", "response": "Create a crash handler typically setting sys. excepthook to it."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading the config file.", "response": "def load_config_file(self, suppress_errors=True):\n \"\"\"Load the config file.\n\n By default, errors in loading config are handled, and a warning\n printed on screen. For testing, the suppress_errors option is set\n to False, so errors will make tests fail.\n \"\"\"\n self.log.debug(\"Searching path %s for config files\", self.config_file_paths)\n base_config = 'ipython_config.py'\n self.log.debug(\"Attempting to load config file: %s\" %\n base_config)\n try:\n Application.load_config_file(\n self,\n base_config,\n path=self.config_file_paths\n )\n except ConfigFileNotFound:\n # ignore errors loading parent\n self.log.debug(\"Config file %s not found\", base_config)\n pass\n if self.config_file_name == base_config:\n # don't load secondary config\n return\n self.log.debug(\"Attempting to load config file: %s\" %\n self.config_file_name)\n try:\n Application.load_config_file(\n self,\n self.config_file_name,\n path=self.config_file_paths\n )\n except ConfigFileNotFound:\n # Only warn if the default config file was NOT being used.\n if self.config_file_specified:\n msg = self.log.warn\n else:\n msg = self.log.debug\n msg(\"Config file not found, skipping: %s\", self.config_file_name)\n except:\n # For testing purposes.\n if not suppress_errors:\n raise\n self.log.warn(\"Error loading config file: %s\" %\n self.config_file_name, exc_info=True)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef init_profile_dir(self):\n try:\n # location explicitly specified:\n location = self.config.ProfileDir.location\n except AttributeError:\n # location not specified, find by profile name\n try:\n p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)\n except ProfileDirError:\n # not found, maybe create it (always create default profile)\n if self.auto_create or self.profile=='default':\n try:\n p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)\n except ProfileDirError:\n self.log.fatal(\"Could not create profile: %r\"%self.profile)\n self.exit(1)\n else:\n self.log.info(\"Created profile dir: %r\"%p.location)\n else:\n self.log.fatal(\"Profile %r not found.\"%self.profile)\n self.exit(1)\n else:\n self.log.info(\"Using existing profile dir: %r\"%p.location)\n else:\n # location is fully specified\n try:\n p = ProfileDir.find_profile_dir(location, self.config)\n except ProfileDirError:\n # not found, maybe create it\n if self.auto_create:\n try:\n p = ProfileDir.create_profile_dir(location, self.config)\n except ProfileDirError:\n self.log.fatal(\"Could not create profile directory: %r\"%location)\n self.exit(1)\n else:\n self.log.info(\"Creating new profile dir: %r\"%location)\n else:\n self.log.fatal(\"Profile directory %r not found.\"%location)\n self.exit(1)\n else:\n self.log.info(\"Using existing profile dir: %r\"%location)\n\n self.profile_dir = p\n self.config_file_paths.append(p.location)", "response": "initialize the profile dir"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef init_config_files(self):\n # copy config files\n path = self.builtin_profile_dir\n if self.copy_config_files:\n src = self.profile\n\n cfg = self.config_file_name\n if path and os.path.exists(os.path.join(path, cfg)):\n self.log.warn(\"Staging %r from %s into %r [overwrite=%s]\"%(\n cfg, src, self.profile_dir.location, self.overwrite)\n )\n self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)\n else:\n self.stage_default_config_file()\n else:\n # Still stage *bundled* config files, but not generated ones\n # This is necessary for `ipython profile=sympy` to load the profile\n # on the first go\n files = glob.glob(os.path.join(path, '*.py'))\n for fullpath in files:\n cfg = os.path.basename(fullpath)\n if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):\n # file was copied\n self.log.warn(\"Staging bundled %s from %s into %r\"%(\n cfg, self.profile, self.profile_dir.location)\n )", "response": "Copy default config files into profile dir."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads coverage data from the coverage data file.", "response": "def read(self):\n \"\"\"Read coverage data from the coverage data file (if it exists).\"\"\"\n if self.use_file:\n self.lines, self.arcs = self._read_file(self.filename)\n else:\n self.lines, self.arcs = {}, {}"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrite the collected coverage data to a file.", "response": "def write(self, suffix=None):\n \"\"\"Write the collected coverage data to a file.\n\n `suffix` is a suffix to append to the base file name. This can be used\n for multiple or parallel execution, so that many coverage data files\n can exist simultaneously. A dot will be used to join the base name and\n the suffix.\n\n \"\"\"\n if self.use_file:\n filename = self.filename\n if suffix:\n filename += \".\" + suffix\n self.write_file(filename)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef erase(self):\n if self.use_file:\n if self.filename:\n file_be_gone(self.filename)\n self.lines = {}\n self.arcs = {}", "response": "Erase the data both in this object and from its file storage."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef line_data(self):\n return dict(\n [(f, sorted(lmap.keys())) for f, lmap in iitems(self.lines)]\n )", "response": "Return the map from filenames to lists of line numbers executed."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the map from filenames to lists of line number pairs.", "response": "def arc_data(self):\n \"\"\"Return the map from filenames to lists of line number pairs.\"\"\"\n return dict(\n [(f, sorted(amap.keys())) for f, amap in iitems(self.arcs)]\n )"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite the coverage data to filename.", "response": "def write_file(self, filename):\n \"\"\"Write the coverage data to `filename`.\"\"\"\n\n # Create the file data.\n data = {}\n\n data['lines'] = self.line_data()\n arcs = self.arc_data()\n if arcs:\n data['arcs'] = arcs\n\n if self.collector:\n data['collector'] = self.collector\n\n if self.debug and self.debug.should('dataio'):\n self.debug.write(\"Writing data to %r\" % (filename,))\n\n # Write the pickle to the file.\n fdata = open(filename, 'wb')\n try:\n pickle.dump(data, fdata, 2)\n finally:\n fdata.close()"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread the coverage data from filename.", "response": "def read_file(self, filename):\n \"\"\"Read the coverage data from `filename`.\"\"\"\n self.lines, self.arcs = self._read_file(filename)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the raw pickled data from filename.", "response": "def raw_data(self, filename):\n \"\"\"Return the raw pickled data from `filename`.\"\"\"\n if self.debug and self.debug.should('dataio'):\n self.debug.write(\"Reading data from %r\" % (filename,))\n fdata = open(filename, 'rb')\n try:\n data = pickle.load(fdata)\n finally:\n fdata.close()\n return data"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _read_file(self, filename):\n lines = {}\n arcs = {}\n try:\n data = self.raw_data(filename)\n if isinstance(data, dict):\n # Unpack the 'lines' item.\n lines = dict([\n (f, dict.fromkeys(linenos, None))\n for f, linenos in iitems(data.get('lines', {}))\n ])\n # Unpack the 'arcs' item.\n arcs = dict([\n (f, dict.fromkeys(arcpairs, None))\n for f, arcpairs in iitems(data.get('arcs', {}))\n ])\n except Exception:\n pass\n return lines, arcs", "response": "Read the stored coverage data from the given file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef combine_parallel_data(self, aliases=None):\n aliases = aliases or PathAliases()\n data_dir, local = os.path.split(self.filename)\n localdot = local + '.'\n for f in os.listdir(data_dir or '.'):\n if f.startswith(localdot):\n full_path = os.path.join(data_dir, f)\n new_lines, new_arcs = self._read_file(full_path)\n for filename, file_data in iitems(new_lines):\n filename = aliases.map(filename)\n self.lines.setdefault(filename, {}).update(file_data)\n for filename, file_data in iitems(new_arcs):\n filename = aliases.map(filename)\n self.arcs.setdefault(filename, {}).update(file_data)\n if f != local:\n os.remove(full_path)", "response": "Combine a number of data files together."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_line_data(self, line_data):\n for filename, linenos in iitems(line_data):\n self.lines.setdefault(filename, {}).update(linenos)", "response": "Add executed line data."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_arc_data(self, arc_data):\n for filename, arcs in iitems(arc_data):\n self.arcs.setdefault(filename, {}).update(arcs)", "response": "Add measured arc data. Arc data is a dictionary of filename - > arcs."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncontributing filename s data to the Md5Hash hasher.", "response": "def add_to_hash(self, filename, hasher):\n \"\"\"Contribute `filename`'s data to the Md5Hash `hasher`.\"\"\"\n hasher.update(self.executed_lines(filename))\n hasher.update(self.executed_arcs(filename))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef summary(self, fullpath=False):\n summ = {}\n if fullpath:\n filename_fn = lambda f: f\n else:\n filename_fn = os.path.basename\n for filename, lines in iitems(self.lines):\n summ[filename_fn(filename)] = len(lines)\n return summ", "response": "Return a dict summarizing the coverage data."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_pasted_lines(sentinel, l_input=py3compat.input):\n print \"Pasting code; enter '%s' alone on the line to stop or use Ctrl-D.\" \\\n % sentinel\n while True:\n try:\n l = l_input(':')\n if l == sentinel:\n return\n else:\n yield l\n except EOFError:\n print ''\n return", "response": "Yield pasted lines until the user enters the given sentinel value."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstarting the mainloop. If an optional banner argument is given, it will override the internally created default banner.", "response": "def mainloop(self, display_banner=None):\n \"\"\"Start the mainloop.\n\n If an optional banner argument is given, it will override the\n internally created default banner.\n \"\"\"\n\n with nested(self.builtin_trap, self.display_trap):\n\n while 1:\n try:\n self.interact(display_banner=display_banner)\n #self.interact_with_readline()\n # XXX for testing of a readline-decoupled repl loop, call\n # interact_with_readline above\n break\n except KeyboardInterrupt:\n # this should not be necessary, but KeyboardInterrupt\n # handling seems rather unpredictable...\n self.write(\"\\nKeyboardInterrupt in interact()\\n\")"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _replace_rlhist_multiline(self, source_raw, hlen_before_cell):\n\n # do nothing without readline or disabled multiline\n if not self.has_readline or not self.multiline_history:\n return hlen_before_cell\n\n # windows rl has no remove_history_item\n if not hasattr(self.readline, \"remove_history_item\"):\n return hlen_before_cell\n\n # skip empty cells\n if not source_raw.rstrip():\n return hlen_before_cell\n\n # nothing changed do nothing, e.g. when rl removes consecutive dups\n hlen = self.readline.get_current_history_length()\n if hlen == hlen_before_cell:\n return hlen_before_cell\n\n for i in range(hlen - hlen_before_cell):\n self.readline.remove_history_item(hlen - i - 1)\n stdin_encoding = get_stream_enc(sys.stdin, 'utf-8')\n self.readline.add_history(py3compat.unicode_to_str(source_raw.rstrip(),\n stdin_encoding))\n return self.readline.get_current_history_length()", "response": "Replace the history of a single line with a single entry in history"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef raw_input(self, prompt=''):\n # Code run by the user may have modified the readline completer state.\n # We must ensure that our completer is back in place.\n\n if self.has_readline:\n self.set_readline_completer()\n \n # raw_input expects str, but we pass it unicode sometimes\n prompt = py3compat.cast_bytes_py2(prompt)\n\n try:\n line = py3compat.str_to_unicode(self.raw_input_original(prompt))\n except ValueError:\n warn(\"\\n********\\nYou or a %run:ed script called sys.stdin.close()\"\n \" or sys.stdout.close()!\\nExiting IPython!\\n\")\n self.ask_exit()\n return \"\"\n\n # Try to be reasonably smart about not re-indenting pasted input more\n # than necessary. We do this by trimming out the auto-indent initial\n # spaces, if the user's actual input started itself with whitespace.\n if self.autoindent:\n if num_ini_spaces(line) > self.indent_current_nsp:\n line = line[self.indent_current_nsp:]\n self.indent_current_nsp = 0\n\n return line", "response": "Write a prompt and read a line."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _should_recompile(self,e):\n\n if e.filename in ('','','',\n '','',\n None):\n\n return False\n try:\n if (self.autoedit_syntax and\n not self.ask_yes_no('Return to editor to correct syntax error? '\n '[Y/n] ','y')):\n return False\n except EOFError:\n return False\n\n def int0(x):\n try:\n return int(x)\n except TypeError:\n return 0\n # always pass integer line and offset values to editor hook\n try:\n self.hooks.fix_error_editor(e.filename,\n int0(e.lineno),int0(e.offset),e.msg)\n except TryNext:\n warn('Could not open editor')\n return False\n return True", "response": "Utility routine for edit_syntax_error"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef exit(self):\n if self.confirm_exit:\n if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'):\n self.ask_exit()\n else:\n self.ask_exit()", "response": "Handle interactive exit.\n This method calls the ask_exit callback."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the correct repository URL and revision by parsing the given repository URL", "response": "def get_url_rev(self):\n \"\"\"\n Returns the correct repository URL and revision by parsing the given\n repository URL\n \"\"\"\n error_message= (\n \"Sorry, '%s' is a malformed VCS url. \"\n \"Ihe format is +://, \"\n \"e.g. svn+http://myrepo/svn/MyApp#egg=MyApp\")\n assert '+' in self.url, error_message % self.url\n url = self.url.split('+', 1)[1]\n scheme, netloc, path, query, frag = urlparse.urlsplit(url)\n rev = None\n if '@' in path:\n path, rev = path.rsplit('@', 1)\n url = urlparse.urlunsplit((scheme, netloc, path, query, ''))\n return url, rev"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndisposes of the exitable object.", "response": "def dispose_at_exit(exitable):\n '''\n register `exitable.__exit__()` into `atexit` module.\n\n return the `exitable` itself.\n '''\n @atexit.register\n def callback():\n exitable.__exit__(*sys.exc_info())\n return exitable"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates and return a new frontend master.", "response": "def new_frontend_master(self):\n \"\"\" Create and return new frontend attached to new kernel, launched on localhost.\n \"\"\"\n ip = self.ip if self.ip in LOCAL_IPS else LOCALHOST\n kernel_manager = self.kernel_manager_class(\n ip=ip,\n connection_file=self._new_connection_file(),\n config=self.config,\n )\n # start the kernel\n kwargs = dict()\n kwargs['extra_arguments'] = self.kernel_argv\n kernel_manager.start_kernel(**kwargs)\n kernel_manager.start_channels()\n widget = self.widget_factory(config=self.config,\n local_kernel=True)\n self.init_colors(widget)\n widget.kernel_manager = kernel_manager\n widget._existing = False\n widget._may_close = True\n widget._confirm_exit = self.confirm_exit\n return widget"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef new_frontend_slave(self, current_widget):\n kernel_manager = self.kernel_manager_class(\n connection_file=current_widget.kernel_manager.connection_file,\n config = self.config,\n )\n kernel_manager.load_connection_file()\n kernel_manager.start_channels()\n widget = self.widget_factory(config=self.config,\n local_kernel=False)\n self.init_colors(widget)\n widget._existing = True\n widget._may_close = False\n widget._confirm_exit = False\n widget.kernel_manager = kernel_manager\n return widget", "response": "Create and return a new frontend attached to an existing kernel."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconfiguring the coloring of the given widget.", "response": "def init_colors(self, widget):\n \"\"\"Configure the coloring of the widget\"\"\"\n # Note: This will be dramatically simplified when colors\n # are removed from the backend.\n\n # parse the colors arg down to current known labels\n try:\n colors = self.config.ZMQInteractiveShell.colors\n except AttributeError:\n colors = None\n try:\n style = self.config.IPythonWidget.syntax_style\n except AttributeError:\n style = None\n try:\n sheet = self.config.IPythonWidget.style_sheet\n except AttributeError:\n sheet = None\n\n # find the value for colors:\n if colors:\n colors=colors.lower()\n if colors in ('lightbg', 'light'):\n colors='lightbg'\n elif colors in ('dark', 'linux'):\n colors='linux'\n else:\n colors='nocolor'\n elif style:\n if style=='bw':\n colors='nocolor'\n elif styles.dark_style(style):\n colors='linux'\n else:\n colors='lightbg'\n else:\n colors=None\n\n # Configure the style\n if style:\n widget.style_sheet = styles.sheet_from_template(style, colors)\n widget.syntax_style = style\n widget._syntax_style_changed()\n widget._style_sheet_changed()\n elif colors:\n # use a default dark/light/bw style\n widget.set_default_style(colors=colors)\n\n if self.stylesheet:\n # we got an explicit stylesheet\n if os.path.isfile(self.stylesheet):\n with open(self.stylesheet) as f:\n sheet = f.read()\n else:\n raise IOError(\"Stylesheet %r not found.\" % self.stylesheet)\n if sheet:\n widget.style_sheet = sheet\n widget._style_sheet_changed()"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef info(self):\n return (self.identity, self.url, self.pub_url, self.location)", "response": "return the connection info for this object s sockets."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef connect(self, peers):\n for peer, (ident, url, pub_url, location) in peers.items():\n self.peers[peer] = ident\n if ident != self.identity:\n self.sub.connect(disambiguate_url(pub_url, location))\n if ident > self.identity:\n # prevent duplicate xrep, by only connecting\n # engines to engines with higher IDENTITY\n # a doubly-connected pair will crash\n self.socket.connect(disambiguate_url(url, location))", "response": "connect to peers. `peers` will be a dict of 4-tuples, keyed by name.\n {peer : (ident, addr, pub_addr, location)}\n where peer is the name, ident is the XREP identity, addr,pub_addr are the"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting an object in R s namespace to one suitable for ipython s namespace.", "response": "def Rconverter(Robj, dataframe=False):\n \"\"\"\n Convert an object in R's namespace to one suitable\n for ipython's namespace.\n\n For a data.frame, it tries to return a structured array.\n It first checks for colnames, then names.\n If all are NULL, it returns np.asarray(Robj), else\n it tries to construct a recarray\n\n Parameters\n ----------\n\n Robj: an R object returned from rpy2\n \"\"\"\n is_data_frame = ro.r('is.data.frame')\n colnames = ro.r('colnames')\n rownames = ro.r('rownames') # with pandas, these could be used for the index\n names = ro.r('names')\n\n if dataframe:\n as_data_frame = ro.r('as.data.frame')\n cols = colnames(Robj)\n _names = names(Robj)\n if cols != ri.NULL:\n Robj = as_data_frame(Robj)\n names = tuple(np.array(cols))\n elif _names != ri.NULL:\n names = tuple(np.array(_names))\n else: # failed to find names\n return np.asarray(Robj)\n Robj = np.rec.fromarrays(Robj, names = names)\n return np.asarray(Robj)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef findsource(object):\n\n file = getsourcefile(object) or getfile(object)\n # If the object is a frame, then trying to get the globals dict from its\n # module won't work. Instead, the frame object itself has the globals\n # dictionary.\n globals_dict = None\n if inspect.isframe(object):\n # XXX: can this ever be false?\n globals_dict = object.f_globals\n else:\n module = getmodule(object, file)\n if module:\n globals_dict = module.__dict__\n lines = linecache.getlines(file, globals_dict)\n if not lines:\n raise IOError('could not get source code')\n\n if ismodule(object):\n return lines, 0\n\n if isclass(object):\n name = object.__name__\n pat = re.compile(r'^(\\s*)class\\s*' + name + r'\\b')\n # make some effort to find the best matching class definition:\n # use the one with the least indentation, which is the one\n # that's most probably not inside a function definition.\n candidates = []\n for i in range(len(lines)):\n match = pat.match(lines[i])\n if match:\n # if it's at toplevel, it's already the best one\n if lines[i][0] == 'c':\n return lines, i\n # else add whitespace to candidate list\n candidates.append((match.group(1), i))\n if candidates:\n # this will sort by whitespace, and by line number,\n # less whitespace first\n candidates.sort()\n return lines, candidates[0][1]\n else:\n raise IOError('could not find class definition')\n\n if ismethod(object):\n object = object.im_func\n if isfunction(object):\n object = object.func_code\n if istraceback(object):\n object = object.tb_frame\n if isframe(object):\n object = object.f_code\n if iscode(object):\n if not hasattr(object, 'co_firstlineno'):\n raise IOError('could not find function definition')\n pat = re.compile(r'^(\\s*def\\s)|(.*(? 0:\n if pmatch(lines[lnum]): break\n lnum -= 1\n\n return lines, lnum\n raise IOError('could not find code object')", "response": "Return the entire source file and starting line number for an object."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntrying to fix the filenames in each record from inspect. getinnerframes.", "response": "def fix_frame_records_filenames(records):\n \"\"\"Try to fix the filenames in each record from inspect.getinnerframes().\n\n Particularly, modules loaded from within zip files have useless filenames\n attached to their code object, and inspect.getinnerframes() just uses it.\n \"\"\"\n fixed_records = []\n for frame, filename, line_no, func_name, lines, index in records:\n # Look inside the frame's globals dictionary for __file__, which should\n # be better.\n better_fn = frame.f_globals.get('__file__', None)\n if isinstance(better_fn, str):\n # Check the type just in case someone did something weird with\n # __file__. It might also be None if the error occurred during\n # import.\n filename = better_fn\n fixed_records.append((frame, filename, line_no, func_name, lines, index))\n return fixed_records"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef color_toggle(self):\n\n if self.color_scheme_table.active_scheme_name == 'NoColor':\n self.color_scheme_table.set_active_scheme(self.old_scheme)\n self.Colors = self.color_scheme_table.active_colors\n else:\n self.old_scheme = self.color_scheme_table.active_scheme_name\n self.color_scheme_table.set_active_scheme('NoColor')\n self.Colors = self.color_scheme_table.active_colors", "response": "Toggle between the currently active color scheme and NoColor."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef text(self, etype, value, tb, tb_offset=None, context=5):\n tb_list = self.structured_traceback(etype, value, tb,\n tb_offset, context)\n return self.stb2text(tb_list)", "response": "Return formatted traceback.\n\n Subclasses may override this if they add extra arguments."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a color formatted string with the traceback info.", "response": "def structured_traceback(self, etype, value, elist, tb_offset=None,\n context=5):\n \"\"\"Return a color formatted string with the traceback info.\n\n Parameters\n ----------\n etype : exception type\n Type of the exception raised.\n\n value : object\n Data stored in the exception\n\n elist : list\n List of frames, see class docstring for details.\n\n tb_offset : int, optional\n Number of frames in the traceback to skip. If not given, the\n instance value is used (set in constructor).\n\n context : int, optional\n Number of lines of context information to print.\n\n Returns\n -------\n String with formatted exception.\n \"\"\"\n tb_offset = self.tb_offset if tb_offset is None else tb_offset\n Colors = self.Colors\n out_list = []\n if elist:\n\n if tb_offset and len(elist) > tb_offset:\n elist = elist[tb_offset:]\n\n out_list.append('Traceback %s(most recent call last)%s:' %\n (Colors.normalEm, Colors.Normal) + '\\n')\n out_list.extend(self._format_list(elist))\n # The exception info should be a single entry in the list.\n lines = ''.join(self._format_exception_only(etype, value))\n out_list.append(lines)\n\n # Note: this code originally read:\n\n ## for line in lines[:-1]:\n ## out_list.append(\" \"+line)\n ## out_list.append(lines[-1])\n\n # This means it was indenting everything but the last line by a little\n # bit. I've disabled this for now, but if we see ugliness somewhre we\n # can restore it.\n\n return out_list"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _format_list(self, extracted_list):\n\n Colors = self.Colors\n list = []\n for filename, lineno, name, line in extracted_list[:-1]:\n item = ' File %s\"%s\"%s, line %s%d%s, in %s%s%s\\n' % \\\n (Colors.filename, filename, Colors.Normal,\n Colors.lineno, lineno, Colors.Normal,\n Colors.name, name, Colors.Normal)\n if line:\n item += ' %s\\n' % line.strip()\n list.append(item)\n # Emphasize the last entry\n filename, lineno, name, line = extracted_list[-1]\n item = '%s File %s\"%s\"%s, line %s%d%s, in %s%s%s%s\\n' % \\\n (Colors.normalEm,\n Colors.filenameEm, filename, Colors.normalEm,\n Colors.linenoEm, lineno, Colors.normalEm,\n Colors.nameEm, name, Colors.normalEm,\n Colors.Normal)\n if line:\n item += '%s %s%s\\n' % (Colors.line, line.strip(),\n Colors.Normal)\n list.append(item)\n #from pprint import pformat; print 'LISTTB', pformat(list) # dbg\n return list", "response": "Format a list of traceback entries for printing."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _format_exception_only(self, etype, value):\n\n have_filedata = False\n Colors = self.Colors\n list = []\n stype = Colors.excName + etype.__name__ + Colors.Normal\n if value is None:\n # Not sure if this can still happen in Python 2.6 and above\n list.append( str(stype) + '\\n')\n else:\n if etype is SyntaxError:\n have_filedata = True\n #print 'filename is',filename # dbg\n if not value.filename: value.filename = \"\"\n list.append('%s File %s\"%s\"%s, line %s%d%s\\n' % \\\n (Colors.normalEm,\n Colors.filenameEm, value.filename, Colors.normalEm,\n Colors.linenoEm, value.lineno, Colors.Normal ))\n if value.text is not None:\n i = 0\n while i < len(value.text) and value.text[i].isspace():\n i += 1\n list.append('%s %s%s\\n' % (Colors.line,\n value.text.strip(),\n Colors.Normal))\n if value.offset is not None:\n s = ' '\n for c in value.text[i:value.offset-1]:\n if c.isspace():\n s += c\n else:\n s += ' '\n list.append('%s%s^%s\\n' % (Colors.caret, s,\n Colors.Normal) )\n\n try:\n s = value.msg\n except Exception:\n s = self._some_str(value)\n if s:\n list.append('%s%s:%s %s\\n' % (str(stype), Colors.excName,\n Colors.Normal, s))\n else:\n list.append('%s\\n' % str(stype))\n\n # sync with user hooks\n if have_filedata:\n ipinst = ipapi.get()\n if ipinst is not None:\n ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)\n\n return list", "response": "Format the exception part of a traceback."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef structured_traceback(self, etype, evalue, etb, tb_offset=None,\n context=5):\n \"\"\"Return a nice text document describing the traceback.\"\"\"\n\n tb_offset = self.tb_offset if tb_offset is None else tb_offset\n\n # some locals\n try:\n etype = etype.__name__\n except AttributeError:\n pass\n Colors = self.Colors # just a shorthand + quicker name lookup\n ColorsNormal = Colors.Normal # used a lot\n col_scheme = self.color_scheme_table.active_scheme_name\n indent = ' '*INDENT_SIZE\n em_normal = '%s\\n%s%s' % (Colors.valEm, indent,ColorsNormal)\n undefined = '%sundefined%s' % (Colors.em, ColorsNormal)\n exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal)\n\n # some internal-use functions\n def text_repr(value):\n \"\"\"Hopefully pretty robust repr equivalent.\"\"\"\n # this is pretty horrible but should always return *something*\n try:\n return pydoc.text.repr(value)\n except KeyboardInterrupt:\n raise\n except:\n try:\n return repr(value)\n except KeyboardInterrupt:\n raise\n except:\n try:\n # all still in an except block so we catch\n # getattr raising\n name = getattr(value, '__name__', None)\n if name:\n # ick, recursion\n return text_repr(name)\n klass = getattr(value, '__class__', None)\n if klass:\n return '%s instance' % text_repr(klass)\n except KeyboardInterrupt:\n raise\n except:\n return 'UNRECOVERABLE REPR FAILURE'\n def eqrepr(value, repr=text_repr): return '=%s' % repr(value)\n def nullrepr(value, repr=text_repr): return ''\n\n # meat of the code begins\n try:\n etype = etype.__name__\n except AttributeError:\n pass\n\n if self.long_header:\n # Header with the exception type, python version, and date\n pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable\n date = time.ctime(time.time())\n\n head = '%s%s%s\\n%s%s%s\\n%s' % (Colors.topline, '-'*75, ColorsNormal,\n exc, ' '*(75-len(str(etype))-len(pyver)),\n pyver, date.rjust(75) )\n head += \"\\nA problem occured executing Python code. Here is the sequence of function\"\\\n \"\\ncalls leading up to the error, with the most recent (innermost) call last.\"\n else:\n # Simplified header\n head = '%s%s%s\\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc,\n 'Traceback (most recent call last)'.\\\n rjust(75 - len(str(etype)) ) )\n frames = []\n # Flush cache before calling inspect. This helps alleviate some of the\n # problems with python 2.3's inspect.py.\n ##self.check_cache()\n # Drop topmost frames if requested\n try:\n # Try the default getinnerframes and Alex's: Alex's fixes some\n # problems, but it generates empty tracebacks for console errors\n # (5 blanks lines) where none should be returned.\n #records = inspect.getinnerframes(etb, context)[tb_offset:]\n #print 'python records:', records # dbg\n records = _fixed_getinnerframes(etb, context, tb_offset)\n #print 'alex records:', records # dbg\n except:\n\n # FIXME: I've been getting many crash reports from python 2.3\n # users, traceable to inspect.py. If I can find a small test-case\n # to reproduce this, I should either write a better workaround or\n # file a bug report against inspect (if that's the real problem).\n # So far, I haven't been able to find an isolated example to\n # reproduce the problem.\n inspect_error()\n traceback.print_exc(file=self.ostream)\n info('\\nUnfortunately, your original traceback can not be constructed.\\n')\n return ''\n\n # build some color string templates outside these nested loops\n tpl_link = '%s%%s%s' % (Colors.filenameEm,ColorsNormal)\n tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,\n ColorsNormal)\n tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \\\n (Colors.vName, Colors.valEm, ColorsNormal)\n tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal)\n tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,\n Colors.vName, ColorsNormal)\n tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)\n tpl_line = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)\n tpl_line_em = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line,\n ColorsNormal)\n\n # now, loop over all records printing context and info\n abspath = os.path.abspath\n for frame, file, lnum, func, lines, index in records:\n #print '*** record:',file,lnum,func,lines,index # dbg\n\n if not file:\n file = '?'\n elif not(file.startswith(\"<\") and file.endswith(\">\")):\n # Guess that filenames like aren't real filenames, so\n # don't call abspath on them. \n try:\n file = abspath(file)\n except OSError:\n # Not sure if this can still happen: abspath now works with\n # file names like \n pass\n\n link = tpl_link % file\n args, varargs, varkw, locals = inspect.getargvalues(frame)\n\n if func == '?':\n call = ''\n else:\n # Decide whether to include variable details or not\n var_repr = self.include_vars and eqrepr or nullrepr\n try:\n call = tpl_call % (func,inspect.formatargvalues(args,\n varargs, varkw,\n locals,formatvalue=var_repr))\n except KeyError:\n # This happens in situations like errors inside generator\n # expressions, where local variables are listed in the\n # line, but can't be extracted from the frame. I'm not\n # 100% sure this isn't actually a bug in inspect itself,\n # but since there's no info for us to compute with, the\n # best we can do is report the failure and move on. Here\n # we must *not* call any traceback construction again,\n # because that would mess up use of %debug later on. So we\n # simply report the failure and move on. The only\n # limitation will be that this frame won't have locals\n # listed in the call signature. Quite subtle problem...\n # I can't think of a good way to validate this in a unit\n # test, but running a script consisting of:\n # dict( (k,v.strip()) for (k,v) in range(10) )\n # will illustrate the error, if this exception catch is\n # disabled.\n call = tpl_call_fail % func\n \n # Don't attempt to tokenize binary files.\n if file.endswith(('.so', '.pyd', '.dll')):\n frames.append('%s %s\\n' % (link,call))\n continue\n elif file.endswith(('.pyc','.pyo')):\n # Look up the corresponding source file.\n file = pyfile.source_from_cache(file)\n\n def linereader(file=file, lnum=[lnum], getline=linecache.getline):\n line = getline(file, lnum[0])\n lnum[0] += 1\n return line\n\n # Build the list of names on this line of code where the exception\n # occurred.\n try:\n names = []\n name_cont = False\n \n for token_type, token, start, end, line in generate_tokens(linereader):\n # build composite names\n if token_type == tokenize.NAME and token not in keyword.kwlist:\n if name_cont:\n # Continuation of a dotted name\n try:\n names[-1].append(token)\n except IndexError:\n names.append([token])\n name_cont = False\n else:\n # Regular new names. We append everything, the caller\n # will be responsible for pruning the list later. It's\n # very tricky to try to prune as we go, b/c composite\n # names can fool us. The pruning at the end is easy\n # to do (or the caller can print a list with repeated\n # names if so desired.\n names.append([token])\n elif token == '.':\n name_cont = True\n elif token_type == tokenize.NEWLINE:\n break\n \n except (IndexError, UnicodeDecodeError):\n # signals exit of tokenizer\n pass\n except tokenize.TokenError,msg:\n _m = (\"An unexpected error occurred while tokenizing input\\n\"\n \"The following traceback may be corrupted or invalid\\n\"\n \"The error message is: %s\\n\" % msg)\n error(_m)\n\n # Join composite names (e.g. \"dict.fromkeys\")\n names = ['.'.join(n) for n in names]\n # prune names list of duplicates, but keep the right order\n unique_names = uniq_stable(names)\n\n # Start loop over vars\n lvals = []\n if self.include_vars:\n for name_full in unique_names:\n name_base = name_full.split('.',1)[0]\n if name_base in frame.f_code.co_varnames:\n if locals.has_key(name_base):\n try:\n value = repr(eval(name_full,locals))\n except:\n value = undefined\n else:\n value = undefined\n name = tpl_local_var % name_full\n else:\n if frame.f_globals.has_key(name_base):\n try:\n value = repr(eval(name_full,frame.f_globals))\n except:\n value = undefined\n else:\n value = undefined\n name = tpl_global_var % name_full\n lvals.append(tpl_name_val % (name,value))\n if lvals:\n lvals = '%s%s' % (indent,em_normal.join(lvals))\n else:\n lvals = ''\n\n level = '%s %s\\n' % (link,call)\n\n if index is None:\n frames.append(level)\n else:\n frames.append('%s%s' % (level,''.join(\n _format_traceback_lines(lnum,index,lines,Colors,lvals,\n col_scheme))))\n\n # Get (safely) a string form of the exception info\n try:\n etype_str,evalue_str = map(str,(etype,evalue))\n except:\n # User exception is improperly defined.\n etype,evalue = str,sys.exc_info()[:2]\n etype_str,evalue_str = map(str,(etype,evalue))\n # ... and format it\n exception = ['%s%s%s: %s' % (Colors.excName, etype_str,\n ColorsNormal, evalue_str)]\n if (not py3compat.PY3) and type(evalue) is types.InstanceType:\n try:\n names = [w for w in dir(evalue) if isinstance(w, basestring)]\n except:\n # Every now and then, an object with funny inernals blows up\n # when dir() is called on it. We do the best we can to report\n # the problem and continue\n _m = '%sException reporting error (object with broken dir())%s:'\n exception.append(_m % (Colors.excName,ColorsNormal))\n etype_str,evalue_str = map(str,sys.exc_info()[:2])\n exception.append('%s%s%s: %s' % (Colors.excName,etype_str,\n ColorsNormal, evalue_str))\n names = []\n for name in names:\n value = text_repr(getattr(evalue, name))\n exception.append('\\n%s%s = %s' % (indent, name, value))\n\n # vds: >>\n if records:\n filepath, lnum = records[-1][1:3]\n #print \"file:\", str(file), \"linenb\", str(lnum) # dbg\n filepath = os.path.abspath(filepath)\n ipinst = ipapi.get()\n if ipinst is not None:\n ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)\n # vds: <<\n\n # return all our info assembled as a single string\n # return '%s\\n\\n%s\\n%s' % (head,'\\n'.join(frames),''.join(exception[0]) )\n return [head] + frames + [''.join(exception[0])]", "response": "Return a nice text document describing the traceback."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef debugger(self,force=False):\n\n if force or self.call_pdb:\n if self.pdb is None:\n self.pdb = debugger.Pdb(\n self.color_scheme_table.active_scheme_name)\n # the system displayhook may have changed, restore the original\n # for pdb\n display_trap = DisplayTrap(hook=sys.__displayhook__)\n with display_trap:\n self.pdb.reset()\n # Find the right frame so we don't pop up inside ipython itself\n if hasattr(self,'tb') and self.tb is not None:\n etb = self.tb\n else:\n etb = self.tb = sys.last_traceback\n while self.tb is not None and self.tb.tb_next is not None:\n self.tb = self.tb.tb_next\n if etb and etb.tb_next:\n etb = etb.tb_next\n self.pdb.botframe = etb.tb_frame\n self.pdb.interaction(self.tb.tb_frame, self.tb)\n\n if hasattr(self,'tb'):\n del self.tb", "response": "Call up the debugger if desired."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nswitch to the desired mode.", "response": "def set_mode(self,mode=None):\n \"\"\"Switch to the desired mode.\n\n If mode is not specified, cycles through the available modes.\"\"\"\n\n if not mode:\n new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \\\n len(self.valid_modes)\n self.mode = self.valid_modes[new_idx]\n elif mode not in self.valid_modes:\n raise ValueError, 'Unrecognized mode in FormattedTB: <'+mode+'>\\n'\\\n 'Valid modes: '+str(self.valid_modes)\n else:\n self.mode = mode\n # include variable details only in 'Verbose' mode\n self.include_vars = (self.mode == self.valid_modes[2])\n # Set the join character for generating text tracebacks\n self.tb_join_char = self._join_chars[self.mode]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef group_required(group,\r\n login_url=None,\r\n redirect_field_name=REDIRECT_FIELD_NAME,\r\n skip_superuser=True):\r\n \"\"\"\r\n View decorator for requiring a user group.\r\n \"\"\"\r\n def decorator(view_func):\r\n @login_required(redirect_field_name=redirect_field_name,\r\n login_url=login_url)\r\n def _wrapped_view(request, *args, **kwargs):\r\n\r\n if not (request.user.is_superuser and skip_superuser):\r\n if request.user.groups.filter(name=group).count() == 0:\r\n raise PermissionDenied\r\n\r\n return view_func(request, *args, **kwargs)\r\n return _wrapped_view\r\n\r\n return decorator", "response": "Decorator for making views that require a user group."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the parent of the current module.", "response": "def get_parent(globals, level):\n \"\"\"\n parent, name = get_parent(globals, level)\n\n Return the package that an import is being performed in. If globals comes\n from the module foo.bar.bat (not itself a package), this returns the\n sys.modules entry for foo.bar. If globals is from a package's __init__.py,\n the package's entry in sys.modules is returned.\n\n If globals doesn't come from a package or a module in a package, or a\n corresponding entry is not found in sys.modules, None is returned.\n \"\"\"\n orig_level = level\n\n if not level or not isinstance(globals, dict):\n return None, ''\n\n pkgname = globals.get('__package__', None)\n\n if pkgname is not None:\n # __package__ is set, so use it\n if not hasattr(pkgname, 'rindex'):\n raise ValueError('__package__ set to non-string')\n if len(pkgname) == 0:\n if level > 0:\n raise ValueError('Attempted relative import in non-package')\n return None, ''\n name = pkgname\n else:\n # __package__ not set, so figure it out and set it\n if '__name__' not in globals:\n return None, ''\n modname = globals['__name__']\n\n if '__path__' in globals:\n # __path__ is set, so modname is already the package name\n globals['__package__'] = name = modname\n else:\n # Normal module, so work out the package name if any\n lastdot = modname.rfind('.')\n if lastdot < 0 and level > 0:\n raise ValueError(\"Attempted relative import in non-package\")\n if lastdot < 0:\n globals['__package__'] = None\n return None, ''\n globals['__package__'] = name = modname[:lastdot]\n\n dot = len(name)\n for x in xrange(level, 1, -1):\n try:\n dot = name.rindex('.', 0, dot)\n except ValueError:\n raise ValueError(\"attempted relative import beyond top-level \"\n \"package\")\n name = name[:dot]\n\n try:\n parent = sys.modules[name]\n except:\n if orig_level < 1:\n warn(\"Parent module '%.200s' not found while handling absolute \"\n \"import\" % name)\n parent = None\n else:\n raise SystemError(\"Parent module '%.200s' not loaded, cannot \"\n \"perform relative import\" % name)\n\n # We expect, but can't guarantee, if parent != None, that:\n # - parent.__name__ == name\n # - parent.__dict__ is globals\n # If this is violated... Who cares?\n return parent, name"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads next module in the sequence of modules.", "response": "def load_next(mod, altmod, name, buf):\n \"\"\"\n mod, name, buf = load_next(mod, altmod, name, buf)\n\n altmod is either None or same as mod\n \"\"\"\n\n if len(name) == 0:\n # completely empty module name should only happen in\n # 'from . import' (or '__import__(\"\")')\n return mod, None, buf\n\n dot = name.find('.')\n if dot == 0:\n raise ValueError('Empty module name')\n\n if dot < 0:\n subname = name\n next = None\n else:\n subname = name[:dot]\n next = name[dot+1:]\n\n if buf != '':\n buf += '.'\n buf += subname\n\n result = import_submodule(mod, subname, buf)\n if result is None and mod != altmod:\n result = import_submodule(altmod, subname, subname)\n if result is not None:\n buf = subname\n\n if result is None:\n raise ImportError(\"No module named %.200s\" % name)\n\n return result, next, buf"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef import_submodule(mod, subname, fullname):\n # Require:\n # if mod == None: subname == fullname\n # else: mod.__name__ + \".\" + subname == fullname\n\n global found_now\n if fullname in found_now and fullname in sys.modules:\n m = sys.modules[fullname]\n else:\n print 'Reloading', fullname\n found_now[fullname] = 1\n oldm = sys.modules.get(fullname, None)\n\n if mod is None:\n path = None\n elif hasattr(mod, '__path__'):\n path = mod.__path__\n else:\n return None\n\n try:\n # This appears to be necessary on Python 3, because imp.find_module()\n # tries to import standard libraries (like io) itself, and we don't\n # want them to be processed by our deep_import_hook.\n with replace_import_hook(original_import):\n fp, filename, stuff = imp.find_module(subname, path)\n except ImportError:\n return None\n\n try:\n m = imp.load_module(fullname, fp, filename, stuff)\n except:\n # load_module probably removed name from modules because of\n # the error. Put back the original module object.\n if oldm:\n sys.modules[fullname] = oldm\n raise\n finally:\n if fp: fp.close()\n\n add_submodule(mod, m, fullname, subname)\n\n return m", "response": "Import a module and return the module object."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd a submodule to the module", "response": "def add_submodule(mod, submod, fullname, subname):\n \"\"\"mod.{subname} = submod\"\"\"\n if mod is None:\n return #Nothing to do here.\n\n if submod is None:\n submod = sys.modules[fullname]\n\n setattr(mod, subname, submod)\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ensure_fromlist(mod, fromlist, buf, recursive):\n if not hasattr(mod, '__path__'):\n return\n for item in fromlist:\n if not hasattr(item, 'rindex'):\n raise TypeError(\"Item in ``from list'' not a string\")\n if item == '*':\n if recursive:\n continue # avoid endless recursion\n try:\n all = mod.__all__\n except AttributeError:\n pass\n else:\n ret = ensure_fromlist(mod, all, buf, 1)\n if not ret:\n return 0\n elif not hasattr(mod, item):\n import_submodule(mod, item, buf + '.' + item)", "response": "Handle from module import a b c imports."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef code_name(code, number=0):\n hash_digest = hashlib.md5(code.encode(\"utf-8\")).hexdigest()\n # Include the number and 12 characters of the hash in the name. It's\n # pretty much impossible that in a single session we'll have collisions\n # even with truncated hashes, and the full one makes tracebacks too long\n return ''.format(number, hash_digest[:12])", "response": "Compute a unique name for a code."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse code to an AST with the current compiler flags active.", "response": "def ast_parse(self, source, filename='', symbol='exec'):\n \"\"\"Parse code to an AST with the current compiler flags active.\n \n Arguments are exactly the same as ast.parse (in the standard library),\n and are passed to the built-in compile function.\"\"\"\n return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmaking a name for a block of code, and cache the code. Parameters ---------- code : str The Python source code to cache. number : int A number which forms part of the code's name. Used for the execution counter. Returns ------- The name of the cached code (as a string). Pass this as the filename argument to compilation, so that tracebacks are correctly hooked up.", "response": "def cache(self, code, number=0):\n \"\"\"Make a name for a block of code, and cache the code.\n \n Parameters\n ----------\n code : str\n The Python source code to cache.\n number : int\n A number which forms part of the code's name. Used for the execution\n counter.\n \n Returns\n -------\n The name of the cached code (as a string). Pass this as the filename\n argument to compilation, so that tracebacks are correctly hooked up.\n \"\"\"\n name = code_name(code, number)\n entry = (len(code), time.time(),\n [line+'\\n' for line in code.splitlines()], name)\n linecache.cache[name] = entry\n linecache._ipython_cache[name] = entry\n return name"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncall the orignal checkcache as intended and update the cache with our data.", "response": "def check_cache(self, *args):\n \"\"\"Call linecache.checkcache() safely protecting our cached values.\n \"\"\"\n # First call the orignal checkcache as intended\n linecache._checkcache_ori(*args)\n # Then, update back the cache with our data, so that tracebacks related\n # to our compiled codes can be produced.\n linecache.cache.update(linecache._ipython_cache)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a line of source to the code.", "response": "def add_line(self, line):\n \"\"\"Add a line of source to the code.\n\n Don't include indentations or newlines.\n\n \"\"\"\n self.code.append(\" \" * self.indent_amount)\n self.code.append(line)\n self.code.append(\"\\n\")"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd a section a sub - CodeBuilder.", "response": "def add_section(self):\n \"\"\"Add a section, a sub-CodeBuilder.\"\"\"\n sect = CodeBuilder(self.indent_amount)\n self.code.append(sect)\n return sect"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_function(self, fn_name):\n assert self.indent_amount == 0\n g = {}\n code_text = str(self)\n exec(code_text, g)\n return g[fn_name]", "response": "Compile the code and return the function fn_name."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates a Python expression for expr.", "response": "def expr_code(self, expr):\n \"\"\"Generate a Python expression for `expr`.\"\"\"\n if \"|\" in expr:\n pipes = expr.split(\"|\")\n code = self.expr_code(pipes[0])\n for func in pipes[1:]:\n self.all_vars.add(func)\n code = \"c_%s(%s)\" % (func, code)\n elif \".\" in expr:\n dots = expr.split(\".\")\n code = self.expr_code(dots[0])\n args = [repr(d) for d in dots[1:]]\n code = \"dot(%s, %s)\" % (code, \", \".join(args))\n else:\n self.all_vars.add(expr)\n code = \"c_%s\" % expr\n return code"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrendering this template by applying it to context.", "response": "def render(self, context=None):\n \"\"\"Render this template by applying it to `context`.\n\n `context` is a dictionary of values to use in this rendering.\n\n \"\"\"\n # Make the complete context we'll use.\n ctx = dict(self.context)\n if context:\n ctx.update(context)\n return self.render_function(ctx, self.do_dots)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef do_dots(self, value, *dots):\n for dot in dots:\n try:\n value = getattr(value, dot)\n except AttributeError:\n value = value[dot]\n if hasattr(value, '__call__'):\n value = value()\n return value", "response": "Evaluate dotted expressions at runtime."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef render_template(tpl, context):\r\n '''\r\n A shortcut function to render a partial template with context and return\r\n the output.\r\n '''\r\n\r\n templates = [tpl] if type(tpl) != list else tpl\r\n tpl_instance = None\r\n\r\n for tpl in templates:\r\n try:\r\n tpl_instance = template.loader.get_template(tpl)\r\n break\r\n except template.TemplateDoesNotExist:\r\n pass\r\n\r\n if not tpl_instance:\r\n raise Exception('Template does not exist: ' + templates[-1])\r\n\r\n return tpl_instance.render(template.Context(context))", "response": "A shortcut function to render a partial template with context and return\r\n the output."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a format data dict for an object.", "response": "def format_display_data(obj, include=None, exclude=None):\n \"\"\"Return a format data dict for an object.\n\n By default all format types will be computed.\n\n The following MIME types are currently implemented:\n\n * text/plain\n * text/html\n * text/latex\n * application/json\n * application/javascript\n * image/png\n * image/jpeg\n * image/svg+xml\n\n Parameters\n ----------\n obj : object\n The Python object whose format data will be computed.\n\n Returns\n -------\n format_dict : dict\n A dictionary of key/value pairs, one or each format that was\n generated for the object. The keys are the format types, which\n will usually be MIME type strings and the values and JSON'able\n data structure containing the raw data for the representation in\n that format.\n include : list or tuple, optional\n A list of format type strings (MIME types) to include in the\n format data dict. If this is set *only* the format types included\n in this list will be computed.\n exclude : list or tuple, optional\n A list of format type string (MIME types) to exclue in the format\n data dict. If this is set all format types will be computed,\n except for those included in this argument.\n \"\"\"\n from IPython.core.interactiveshell import InteractiveShell\n\n InteractiveShell.instance().display_formatter.format(\n obj,\n include,\n exclude\n )"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _formatters_default(self):\n formatter_classes = [\n PlainTextFormatter,\n HTMLFormatter,\n SVGFormatter,\n PNGFormatter,\n JPEGFormatter,\n LatexFormatter,\n JSONFormatter,\n JavascriptFormatter\n ]\n d = {}\n for cls in formatter_classes:\n f = cls(config=self.config)\n d[f.format_type] = f\n return d", "response": "Activate the default formatters."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef format(self, obj, include=None, exclude=None):\n format_dict = {}\n\n # If plain text only is active\n if self.plain_text_only:\n formatter = self.formatters['text/plain']\n try:\n data = formatter(obj)\n except:\n # FIXME: log the exception\n raise\n if data is not None:\n format_dict['text/plain'] = data\n return format_dict\n\n for format_type, formatter in self.formatters.items():\n if include is not None:\n if format_type not in include:\n continue\n if exclude is not None:\n if format_type in exclude:\n continue\n try:\n data = formatter(obj)\n except:\n # FIXME: log the exception\n raise\n if data is not None:\n format_dict[format_type] = data\n return format_dict", "response": "Return a format data dict for an object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a format function for a given type.", "response": "def for_type(self, typ, func):\n \"\"\"Add a format function for a given type.\n\n Parameters\n -----------\n typ : class\n The class of the object that will be formatted using `func`.\n func : callable\n The callable that will be called to compute the format data. The\n call signature of this function is simple, it must take the\n object to be formatted and return the raw data for the given\n format. Subclasses may use a different call signature for the\n `func` argument.\n \"\"\"\n oldfunc = self.type_printers.get(typ, None)\n if func is not None:\n # To support easy restoration of old printers, we need to ignore\n # Nones.\n self.type_printers[typ] = func\n return oldfunc"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a format function for a type specified by the full dotted name of the module and the name of the object.", "response": "def for_type_by_name(self, type_module, type_name, func):\n \"\"\"Add a format function for a type specified by the full dotted\n module and name of the type, rather than the type of the object.\n\n Parameters\n ----------\n type_module : str\n The full dotted name of the module the type is defined in, like\n ``numpy``.\n type_name : str\n The name of the type (the class name), like ``dtype``\n func : callable\n The callable that will be called to compute the format data. The\n call signature of this function is simple, it must take the\n object to be formatted and return the raw data for the given\n format. Subclasses may use a different call signature for the\n `func` argument.\n \"\"\"\n key = (type_module, type_name)\n oldfunc = self.deferred_printers.get(key, None)\n if func is not None:\n # To support easy restoration of old printers, we need to ignore\n # Nones.\n self.deferred_printers[key] = func\n return oldfunc"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _in_deferred_types(self, cls):\n mod = getattr(cls, '__module__', None)\n name = getattr(cls, '__name__', None)\n key = (mod, name)\n printer = None\n if key in self.deferred_printers:\n # Move the printer over to the regular registry.\n printer = self.deferred_printers.pop(key)\n self.type_printers[cls] = printer\n return printer", "response": "Check if the given class is specified in the deferred type registry. If it does return the printer that is registered under the given class. If it does return None."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _float_precision_changed(self, name, old, new):\n\n if '%' in new:\n # got explicit format string\n fmt = new\n try:\n fmt%3.14159\n except Exception:\n raise ValueError(\"Precision must be int or format string, not %r\"%new)\n elif new:\n # otherwise, should be an int\n try:\n i = int(new)\n assert i >= 0\n except ValueError:\n raise ValueError(\"Precision must be int or format string, not %r\"%new)\n except AssertionError:\n raise ValueError(\"int precision must be non-negative, not %r\"%i)\n\n fmt = '%%.%if'%i\n if 'numpy' in sys.modules:\n # set numpy precision if it has been imported\n import numpy\n numpy.set_printoptions(precision=i)\n else:\n # default back to repr\n fmt = '%r'\n if 'numpy' in sys.modules:\n import numpy\n # numpy default is 8\n numpy.set_printoptions(precision=8)\n self.float_format = fmt", "response": "This function is called when the float precision of a single entry in a log file has changed."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn path to any existing user config files", "response": "def user_config_files():\n \"\"\"Return path to any existing user config files\n \"\"\"\n return filter(os.path.exists,\n map(os.path.expanduser, config_files))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndo the value look like an on flag?", "response": "def flag(val):\n \"\"\"Does the value look like an on/off flag?\"\"\"\n if val == 1:\n return True\n elif val == 0:\n return False\n val = str(val)\n if len(val) > 5:\n return False\n return val.upper() in ('1', '0', 'F', 'T', 'TRUE', 'FALSE', 'ON', 'OFF')"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef configure(self, argv=None, doc=None):\n env = self.env\n if argv is None:\n argv = sys.argv\n\n cfg_files = getattr(self, 'files', [])\n options, args = self._parseArgs(argv, cfg_files)\n # If -c --config has been specified on command line,\n # load those config files and reparse\n if getattr(options, 'files', []):\n options, args = self._parseArgs(argv, options.files)\n\n self.options = options\n if args:\n self.testNames = args\n if options.testNames is not None:\n self.testNames.extend(tolist(options.testNames))\n\n if options.py3where is not None:\n if sys.version_info >= (3,):\n options.where = options.py3where\n\n # `where` is an append action, so it can't have a default value\n # in the parser, or that default will always be in the list\n if not options.where:\n options.where = env.get('NOSE_WHERE', None)\n\n # include and exclude also\n if not options.ignoreFiles:\n options.ignoreFiles = env.get('NOSE_IGNORE_FILES', [])\n if not options.include:\n options.include = env.get('NOSE_INCLUDE', [])\n if not options.exclude:\n options.exclude = env.get('NOSE_EXCLUDE', [])\n\n self.addPaths = options.addPaths\n self.stopOnError = options.stopOnError\n self.verbosity = options.verbosity\n self.includeExe = options.includeExe\n self.traverseNamespace = options.traverseNamespace\n self.debug = options.debug\n self.debugLog = options.debugLog\n self.loggingConfig = options.loggingConfig\n self.firstPackageWins = options.firstPackageWins\n self.configureLogging()\n\n if options.where is not None:\n self.configureWhere(options.where)\n\n if options.testMatch:\n self.testMatch = re.compile(options.testMatch)\n\n if options.ignoreFiles:\n self.ignoreFiles = map(re.compile, tolist(options.ignoreFiles))\n log.info(\"Ignoring files matching %s\", options.ignoreFiles)\n else:\n log.info(\"Ignoring files matching %s\", self.ignoreFilesDefaultStrings)\n\n if options.include:\n self.include = map(re.compile, tolist(options.include))\n log.info(\"Including tests matching %s\", options.include)\n\n if options.exclude:\n self.exclude = map(re.compile, tolist(options.exclude))\n log.info(\"Excluding tests matching %s\", options.exclude)\n\n # When listing plugins we don't want to run them\n if not options.showPlugins:\n self.plugins.configure(options, self)\n self.plugins.begin()", "response": "Execute configure before\n collecting tests with nose. TestCollector to enable output capture and other features."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconfigures logging for nose or optionally other packages.", "response": "def configureLogging(self):\n \"\"\"Configure logging for nose, or optionally other packages. Any logger\n name may be set with the debug option, and that logger will be set to\n debug level and be assigned the same handler as the nose loggers, unless\n it already has a handler.\n \"\"\"\n if self.loggingConfig:\n from logging.config import fileConfig\n fileConfig(self.loggingConfig)\n return\n\n format = logging.Formatter('%(name)s: %(levelname)s: %(message)s')\n if self.debugLog:\n handler = logging.FileHandler(self.debugLog)\n else:\n handler = logging.StreamHandler(self.logStream)\n handler.setFormatter(format)\n\n logger = logging.getLogger('nose')\n logger.propagate = 0\n\n # only add our default handler if there isn't already one there\n # this avoids annoying duplicate log messages.\n if handler not in logger.handlers:\n logger.addHandler(handler)\n\n # default level\n lvl = logging.WARNING\n if self.verbosity >= 5:\n lvl = 0\n elif self.verbosity >= 4:\n lvl = logging.DEBUG\n elif self.verbosity >= 3:\n lvl = logging.INFO\n logger.setLevel(lvl)\n\n # individual overrides\n if self.debug:\n # no blanks\n debug_loggers = [ name for name in self.debug.split(',')\n if name ]\n for logger_name in debug_loggers:\n l = logging.getLogger(logger_name)\n l.setLevel(logging.DEBUG)\n if not l.handlers and not logger_name.startswith('nose'):\n l.addHandler(handler)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef configureWhere(self, where):\n from nose.importer import add_path\n self.workingDir = None\n where = tolist(where)\n warned = False\n for path in where:\n if not self.workingDir:\n abs_path = absdir(path)\n if abs_path is None:\n raise ValueError(\"Working directory %s not found, or \"\n \"not a directory\" % path)\n log.info(\"Set working dir to %s\", abs_path)\n self.workingDir = abs_path\n if self.addPaths and \\\n os.path.exists(os.path.join(abs_path, '__init__.py')):\n log.info(\"Working directory %s is a package; \"\n \"adding to sys.path\" % abs_path)\n add_path(abs_path)\n continue\n if not warned:\n warn(\"Use of multiple -w arguments is deprecated and \"\n \"support may be removed in a future release. You can \"\n \"get the same behavior by passing directories without \"\n \"the -w argument on the command line, or by using the \"\n \"--tests argument in a configuration file.\",\n DeprecationWarning)\n self.testNames.append(path)", "response": "Configure the working directory or directories for the test run."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getParser(self, doc=None):\n if self.parser:\n return self.parser\n env = self.env\n parser = self.parserClass(doc)\n parser.add_option(\n \"-V\",\"--version\", action=\"store_true\",\n dest=\"version\", default=False,\n help=\"Output nose version and exit\")\n parser.add_option(\n \"-p\", \"--plugins\", action=\"store_true\",\n dest=\"showPlugins\", default=False,\n help=\"Output list of available plugins and exit. Combine with \"\n \"higher verbosity for greater detail\")\n parser.add_option(\n \"-v\", \"--verbose\",\n action=\"count\", dest=\"verbosity\",\n default=self.verbosity,\n help=\"Be more verbose. [NOSE_VERBOSE]\")\n parser.add_option(\n \"--verbosity\", action=\"store\", dest=\"verbosity\",\n metavar='VERBOSITY',\n type=\"int\", help=\"Set verbosity; --verbosity=2 is \"\n \"the same as -v\")\n parser.add_option(\n \"-q\", \"--quiet\", action=\"store_const\", const=0, dest=\"verbosity\",\n help=\"Be less verbose\")\n parser.add_option(\n \"-c\", \"--config\", action=\"append\", dest=\"files\",\n metavar=\"FILES\",\n help=\"Load configuration from config file(s). May be specified \"\n \"multiple times; in that case, all config files will be \"\n \"loaded and combined\")\n parser.add_option(\n \"-w\", \"--where\", action=\"append\", dest=\"where\",\n metavar=\"WHERE\",\n help=\"Look for tests in this directory. \"\n \"May be specified multiple times. The first directory passed \"\n \"will be used as the working directory, in place of the current \"\n \"working directory, which is the default. Others will be added \"\n \"to the list of tests to execute. [NOSE_WHERE]\"\n )\n parser.add_option(\n \"--py3where\", action=\"append\", dest=\"py3where\",\n metavar=\"PY3WHERE\",\n help=\"Look for tests in this directory under Python 3.x. \"\n \"Functions the same as 'where', but only applies if running under \"\n \"Python 3.x or above. Note that, if present under 3.x, this \"\n \"option completely replaces any directories specified with \"\n \"'where', so the 'where' option becomes ineffective. \"\n \"[NOSE_PY3WHERE]\"\n )\n parser.add_option(\n \"-m\", \"--match\", \"--testmatch\", action=\"store\",\n dest=\"testMatch\", metavar=\"REGEX\",\n help=\"Files, directories, function names, and class names \"\n \"that match this regular expression are considered tests. \"\n \"Default: %s [NOSE_TESTMATCH]\" % self.testMatchPat,\n default=self.testMatchPat)\n parser.add_option(\n \"--tests\", action=\"store\", dest=\"testNames\", default=None,\n metavar='NAMES',\n help=\"Run these tests (comma-separated list). This argument is \"\n \"useful mainly from configuration files; on the command line, \"\n \"just pass the tests to run as additional arguments with no \"\n \"switch.\")\n parser.add_option(\n \"-l\", \"--debug\", action=\"store\",\n dest=\"debug\", default=self.debug,\n help=\"Activate debug logging for one or more systems. \"\n \"Available debug loggers: nose, nose.importer, \"\n \"nose.inspector, nose.plugins, nose.result and \"\n \"nose.selector. Separate multiple names with a comma.\")\n parser.add_option(\n \"--debug-log\", dest=\"debugLog\", action=\"store\",\n default=self.debugLog, metavar=\"FILE\",\n help=\"Log debug messages to this file \"\n \"(default: sys.stderr)\")\n parser.add_option(\n \"--logging-config\", \"--log-config\",\n dest=\"loggingConfig\", action=\"store\",\n default=self.loggingConfig, metavar=\"FILE\",\n help=\"Load logging config from this file -- bypasses all other\"\n \" logging config settings.\")\n parser.add_option(\n \"-I\", \"--ignore-files\", action=\"append\", dest=\"ignoreFiles\",\n metavar=\"REGEX\",\n help=\"Completely ignore any file that matches this regular \"\n \"expression. Takes precedence over any other settings or \"\n \"plugins. \"\n \"Specifying this option will replace the default setting. \"\n \"Specify this option multiple times \"\n \"to add more regular expressions [NOSE_IGNORE_FILES]\")\n parser.add_option(\n \"-e\", \"--exclude\", action=\"append\", dest=\"exclude\",\n metavar=\"REGEX\",\n help=\"Don't run tests that match regular \"\n \"expression [NOSE_EXCLUDE]\")\n parser.add_option(\n \"-i\", \"--include\", action=\"append\", dest=\"include\",\n metavar=\"REGEX\",\n help=\"This regular expression will be applied to files, \"\n \"directories, function names, and class names for a chance \"\n \"to include additional tests that do not match TESTMATCH. \"\n \"Specify this option multiple times \"\n \"to add more regular expressions [NOSE_INCLUDE]\")\n parser.add_option(\n \"-x\", \"--stop\", action=\"store_true\", dest=\"stopOnError\",\n default=self.stopOnError,\n help=\"Stop running tests after the first error or failure\")\n parser.add_option(\n \"-P\", \"--no-path-adjustment\", action=\"store_false\",\n dest=\"addPaths\",\n default=self.addPaths,\n help=\"Don't make any changes to sys.path when \"\n \"loading tests [NOSE_NOPATH]\")\n parser.add_option(\n \"--exe\", action=\"store_true\", dest=\"includeExe\",\n default=self.includeExe,\n help=\"Look for tests in python modules that are \"\n \"executable. Normal behavior is to exclude executable \"\n \"modules, since they may not be import-safe \"\n \"[NOSE_INCLUDE_EXE]\")\n parser.add_option(\n \"--noexe\", action=\"store_false\", dest=\"includeExe\",\n help=\"DO NOT look for tests in python modules that are \"\n \"executable. (The default on the windows platform is to \"\n \"do so.)\")\n parser.add_option(\n \"--traverse-namespace\", action=\"store_true\",\n default=self.traverseNamespace, dest=\"traverseNamespace\",\n help=\"Traverse through all path entries of a namespace package\")\n parser.add_option(\n \"--first-package-wins\", \"--first-pkg-wins\", \"--1st-pkg-wins\",\n action=\"store_true\", default=False, dest=\"firstPackageWins\",\n help=\"nose's importer will normally evict a package from sys.\"\n \"modules if it sees a package with the same name in a different \"\n \"location. Set this option to disable that behavior.\")\n\n self.plugins.loadPlugins()\n self.pluginOpts(parser)\n\n self.parser = parser\n return parser", "response": "Get the command line option parser."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef page_dumb(strng, start=0, screen_lines=25):\n\n out_ln = strng.splitlines()[start:]\n screens = chop(out_ln,screen_lines-1)\n if len(screens) == 1:\n print >>io.stdout, os.linesep.join(screens[0])\n else:\n last_escape = \"\"\n for scr in screens[0:-1]:\n hunk = os.linesep.join(scr)\n print >>io.stdout, last_escape + hunk\n if not page_more():\n return\n esc_list = esc_re.findall(hunk)\n if len(esc_list) > 0:\n last_escape = esc_list[-1]\n print >>io.stdout, last_escape + os.linesep.join(screens[-1])", "response": "Very dumb pager in Python for when nothing else works."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndetect the screen size of the current terminal.", "response": "def _detect_screen_size(use_curses, screen_lines_def):\n \"\"\"Attempt to work out the number of lines on the screen.\n \n This is called by page(). It can raise an error (e.g. when run in the\n test suite), so it's separated out so it can easily be called in a try block.\n \"\"\"\n TERM = os.environ.get('TERM',None)\n if (TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5':\n local_use_curses = use_curses\n else:\n # curses causes problems on many terminals other than xterm, and\n # some termios calls lock up on Sun OS5.\n local_use_curses = False\n if local_use_curses:\n import termios\n import curses\n # There is a bug in curses, where *sometimes* it fails to properly\n # initialize, and then after the endwin() call is made, the\n # terminal is left in an unusable state. Rather than trying to\n # check everytime for this (by requesting and comparing termios\n # flags each time), we just save the initial terminal state and\n # unconditionally reset it every time. It's cheaper than making\n # the checks.\n term_flags = termios.tcgetattr(sys.stdout)\n\n # Curses modifies the stdout buffer size by default, which messes\n # up Python's normal stdout buffering. This would manifest itself\n # to IPython users as delayed printing on stdout after having used\n # the pager.\n #\n # We can prevent this by manually setting the NCURSES_NO_SETBUF\n # environment variable. For more details, see:\n # http://bugs.python.org/issue10144\n NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)\n os.environ['NCURSES_NO_SETBUF'] = ''\n \n # Proceed with curses initialization\n scr = curses.initscr()\n screen_lines_real,screen_cols = scr.getmaxyx()\n curses.endwin()\n\n # Restore environment\n if NCURSES_NO_SETBUF is None:\n del os.environ['NCURSES_NO_SETBUF']\n else:\n os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF\n \n # Restore terminal state in case endwin() didn't.\n termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)\n # Now we have what we needed: the screen size in rows/columns\n return screen_lines_real\n #print '***Screen size:',screen_lines_real,'lines x',\\\n #screen_cols,'columns.' # dbg\n else:\n return screen_lines_def"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nprinting a string through a pager command.", "response": "def page(strng, start=0, screen_lines=0, pager_cmd=None):\n \"\"\"Print a string, piping through a pager after a certain length.\n\n The screen_lines parameter specifies the number of *usable* lines of your\n terminal screen (total lines minus lines you need to reserve to show other\n information).\n\n If you set screen_lines to a number <=0, page() will try to auto-determine\n your screen size and will only use up to (screen_size+screen_lines) for\n printing, paging after that. That is, if you want auto-detection but need\n to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for\n auto-detection without any lines reserved simply use screen_lines = 0.\n\n If a string won't fit in the allowed lines, it is sent through the\n specified pager command. If none given, look for PAGER in the environment,\n and ultimately default to less.\n\n If no system pager works, the string is sent through a 'dumb pager'\n written in python, very simplistic.\n \"\"\"\n\n # Some routines may auto-compute start offsets incorrectly and pass a\n # negative value. Offset to 0 for robustness.\n start = max(0, start)\n\n # first, try the hook\n ip = ipapi.get()\n if ip:\n try:\n ip.hooks.show_in_pager(strng)\n return\n except TryNext:\n pass\n\n # Ugly kludge, but calling curses.initscr() flat out crashes in emacs\n TERM = os.environ.get('TERM','dumb')\n if TERM in ['dumb','emacs'] and os.name != 'nt':\n print strng\n return\n # chop off the topmost part of the string we don't want to see\n str_lines = strng.splitlines()[start:]\n str_toprint = os.linesep.join(str_lines)\n num_newlines = len(str_lines)\n len_str = len(str_toprint)\n\n # Dumb heuristics to guesstimate number of on-screen lines the string\n # takes. Very basic, but good enough for docstrings in reasonable\n # terminals. If someone later feels like refining it, it's not hard.\n numlines = max(num_newlines,int(len_str/80)+1)\n\n screen_lines_def = get_terminal_size()[1]\n\n # auto-determine screen size\n if screen_lines <= 0:\n try:\n screen_lines += _detect_screen_size(use_curses, screen_lines_def)\n except (TypeError, UnsupportedOperation):\n print >>io.stdout, str_toprint\n return\n\n #print 'numlines',numlines,'screenlines',screen_lines # dbg\n if numlines <= screen_lines :\n #print '*** normal print' # dbg\n print >>io.stdout, str_toprint\n else:\n # Try to open pager and default to internal one if that fails.\n # All failure modes are tagged as 'retval=1', to match the return\n # value of a failed system command. If any intermediate attempt\n # sets retval to 1, at the end we resort to our own page_dumb() pager.\n pager_cmd = get_pager_cmd(pager_cmd)\n pager_cmd += ' ' + get_pager_start(pager_cmd,start)\n if os.name == 'nt':\n if pager_cmd.startswith('type'):\n # The default WinXP 'type' command is failing on complex strings.\n retval = 1\n else:\n tmpname = tempfile.mktemp('.txt')\n tmpfile = open(tmpname,'wt')\n tmpfile.write(strng)\n tmpfile.close()\n cmd = \"%s < %s\" % (pager_cmd,tmpname)\n if os.system(cmd):\n retval = 1\n else:\n retval = None\n os.remove(tmpname)\n else:\n try:\n retval = None\n # if I use popen4, things hang. No idea why.\n #pager,shell_out = os.popen4(pager_cmd)\n pager = os.popen(pager_cmd,'w')\n pager.write(strng)\n pager.close()\n retval = pager.close() # success returns None\n except IOError,msg: # broken pipe when user quits\n if msg.args == (32,'Broken pipe'):\n retval = None\n else:\n retval = 1\n except OSError:\n # Other strange problems, sometimes seen in Win2k/cygwin\n retval = 1\n if retval is not None:\n page_dumb(strng,screen_lines=screen_lines)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npages a file using a pager command and starting line.", "response": "def page_file(fname, start=0, pager_cmd=None):\n \"\"\"Page a file, using an optional pager command and starting line.\n \"\"\"\n\n pager_cmd = get_pager_cmd(pager_cmd)\n pager_cmd += ' ' + get_pager_start(pager_cmd,start)\n\n try:\n if os.environ['TERM'] in ['emacs','dumb']:\n raise EnvironmentError\n system(pager_cmd + ' ' + fname)\n except:\n try:\n if start > 0:\n start -= 1\n page(open(fname).read(),start)\n except:\n print 'Unable to show file',`fname`"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a pager command.", "response": "def get_pager_cmd(pager_cmd=None):\n \"\"\"Return a pager command.\n\n Makes some attempts at finding an OS-correct one.\n \"\"\"\n if os.name == 'posix':\n default_pager_cmd = 'less -r' # -r for color control sequences\n elif os.name in ['nt','dos']:\n default_pager_cmd = 'type'\n\n if pager_cmd is None:\n try:\n pager_cmd = os.environ['PAGER']\n except:\n pager_cmd = default_pager_cmd\n return pager_cmd"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the string for paging files with an offset.", "response": "def get_pager_start(pager, start):\n \"\"\"Return the string for paging files with an offset.\n\n This is the '+N' argument which less and more (under Unix) accept.\n \"\"\"\n\n if pager in ['less','more']:\n if start:\n start_string = '+' + str(start)\n else:\n start_string = ''\n else:\n start_string = ''\n return start_string"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nprint a string snipping the midsection to fit in width.", "response": "def snip_print(str,width = 75,print_full = 0,header = ''):\n \"\"\"Print a string snipping the midsection to fit in width.\n\n print_full: mode control:\n - 0: only snip long strings\n - 1: send to page() directly.\n - 2: snip long strings and ask for full length viewing with page()\n Return 1 if snipping was necessary, 0 otherwise.\"\"\"\n\n if print_full == 1:\n page(header+str)\n return 0\n\n print header,\n if len(str) < width:\n print str\n snip = 0\n else:\n whalf = int((width -5)/2)\n print str[:whalf] + ' <...> ' + str[-whalf:]\n snip = 1\n if snip and print_full == 2:\n if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':\n page(str)\n return snip"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexecute a function reps times and return a tuple with the elapsed total CPU time in seconds the time per call and the output.", "response": "def timings_out(reps,func,*args,**kw):\n \"\"\"timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)\n\n Execute a function reps times, return a tuple with the elapsed total\n CPU time in seconds, the time per call and the function's output.\n\n Under Unix, the return value is the sum of user+system time consumed by\n the process, computed via the resource module. This prevents problems\n related to the wraparound effect which the time.clock() function has.\n\n Under Windows the return value is in wall clock seconds. See the\n documentation for the time module for more details.\"\"\"\n\n reps = int(reps)\n assert reps >=1, 'reps must be >= 1'\n if reps==1:\n start = clock()\n out = func(*args,**kw)\n tot_time = clock()-start\n else:\n rng = xrange(reps-1) # the last time is executed separately to store output\n start = clock()\n for dummy in rng: func(*args,**kw)\n out = func(*args,**kw) # one last time\n tot_time = clock()-start\n av_time = tot_time / reps\n return tot_time,av_time,out"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexecutes a function reps times and return a tuple with the elapsed total CPU time in seconds and the time per call.", "response": "def timings(reps,func,*args,**kw):\n \"\"\"timings(reps,func,*args,**kw) -> (t_total,t_per_call)\n\n Execute a function reps times, return a tuple with the elapsed total CPU\n time in seconds and the time per call. These are just the first two values\n in timings_out().\"\"\"\n\n return timings_out(reps,func,*args,**kw)[0:2]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef print_display_png(o):\n s = latex(o, mode='plain')\n s = s.strip('$')\n # As matplotlib does not support display style, dvipng backend is\n # used here.\n png = latex_to_png('$$%s$$' % s, backend='dvipng')\n return png", "response": "A function to display sympy expression using display style LaTeX in PNG."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef can_print_latex(o):\n import sympy\n if isinstance(o, (list, tuple, set, frozenset)):\n return all(can_print_latex(i) for i in o)\n elif isinstance(o, dict):\n return all((isinstance(i, basestring) or can_print_latex(i)) and can_print_latex(o[i]) for i in o)\n elif isinstance(o,(sympy.Basic, sympy.matrices.Matrix, int, long, float)):\n return True\n return False", "response": "Return True if LaTeX can print the given object."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef print_latex(o):\n if can_print_latex(o):\n s = latex(o, mode='plain')\n s = s.replace('\\\\dag','\\\\dagger')\n s = s.strip('$')\n return '$$%s$$' % s\n # Fallback to the string printer\n return None", "response": "A function to generate the latex representation of sympy\n expressions."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads the extension in IPython.", "response": "def load_ipython_extension(ip):\n \"\"\"Load the extension in IPython.\"\"\"\n import sympy\n\n # sympyprinting extension has been moved to SymPy as of 0.7.2, if it\n # exists there, warn the user and import it\n try:\n import sympy.interactive.ipythonprinting\n except ImportError:\n pass\n else:\n warnings.warn(\"The sympyprinting extension in IPython is deprecated, \"\n \"use sympy.interactive.ipythonprinting\")\n ip.extension_manager.load_extension('sympy.interactive.ipythonprinting')\n return\n\n global _loaded\n if not _loaded:\n plaintext_formatter = ip.display_formatter.formatters['text/plain']\n\n for cls in (object, str):\n plaintext_formatter.for_type(cls, print_basic_unicode)\n\n printable_containers = [list, tuple]\n\n # set and frozen set were broken with SymPy's latex() function, but\n # was fixed in the 0.7.1-git development version. See\n # http://code.google.com/p/sympy/issues/detail?id=3062.\n if sympy.__version__ > '0.7.1':\n printable_containers += [set, frozenset]\n else:\n plaintext_formatter.for_type(cls, print_basic_unicode)\n\n plaintext_formatter.for_type_by_name(\n 'sympy.core.basic', 'Basic', print_basic_unicode\n )\n plaintext_formatter.for_type_by_name(\n 'sympy.matrices.matrices', 'Matrix', print_basic_unicode\n )\n\n png_formatter = ip.display_formatter.formatters['image/png']\n\n png_formatter.for_type_by_name(\n 'sympy.core.basic', 'Basic', print_png\n )\n png_formatter.for_type_by_name(\n 'sympy.matrices.matrices', 'Matrix', print_display_png\n )\n for cls in [dict, int, long, float] + printable_containers:\n png_formatter.for_type(cls, print_png)\n\n latex_formatter = ip.display_formatter.formatters['text/latex']\n latex_formatter.for_type_by_name(\n 'sympy.core.basic', 'Basic', print_latex\n )\n latex_formatter.for_type_by_name(\n 'sympy.matrices.matrices', 'Matrix', print_latex\n )\n\n for cls in printable_containers:\n # Use LaTeX only if every element is printable by latex\n latex_formatter.for_type(cls, print_latex)\n\n _loaded = True"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef str2tokens(string, delimiter):\n\n token_list = [token.strip() for token in string.split(delimiter)]\n return token_list", "response": "Convert a string to a list of tokens"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef str2tokenstags(string, delimiter):\n\n token_list = [token.strip() for token in string.split(delimiter)]\n return token_list", "response": "Convert a string to a list of tokens."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_options(self, parser, env=None):\n # FIXME raise deprecation warning if wasn't called by wrapper\n if env is None:\n env = os.environ\n try:\n self.options(parser, env)\n self.can_configure = True\n except OptionConflictError, e:\n warn(\"Plugin %s has conflicting option string: %s and will \"\n \"be disabled\" % (self, e), RuntimeWarning)\n self.enabled = False\n self.can_configure = False", "response": "Add options to the object."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef options(self, parser, env):\n env_opt = 'NOSE_WITH_%s' % self.name.upper()\n env_opt = env_opt.replace('-', '_')\n parser.add_option(\"--with-%s\" % self.name,\n action=\"store_true\",\n dest=self.enableOpt,\n default=env.get(env_opt),\n help=\"Enable plugin %s: %s [%s]\" %\n (self.__class__.__name__, self.help(), env_opt))", "response": "Register commandline options.\n\n Implement this method for normal options behavior with protection from\n OptionConflictErrors. If you override this method and want the default\n --with-$name option to be registered, be sure to call super()."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconfigure the base plugin and system based on selected options.", "response": "def configure(self, options, conf):\n \"\"\"Configure the plugin and system, based on selected options.\n\n The base plugin class sets the plugin to enabled if the enable option\n for the plugin (self.enableOpt) is true.\n \"\"\"\n if not self.can_configure:\n return\n self.conf = conf\n if hasattr(options, self.enableOpt):\n self.enabled = getattr(options, self.enableOpt)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef validate_string_list(lst):\n if not isinstance(lst, list):\n raise ValueError('input %r must be a list' % lst)\n for x in lst:\n if not isinstance(x, basestring):\n raise ValueError('element %r in list must be a string' % x)", "response": "Validate that the input is a list of strings. Raises ValueError if not."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nvalidates that the input is a dict with string keys and values. Raises ValueError if not.", "response": "def validate_string_dict(dct):\n \"\"\"Validate that the input is a dict with string keys and values.\n\n Raises ValueError if not.\"\"\"\n for k,v in dct.iteritems():\n if not isinstance(k, basestring):\n raise ValueError('key %r in dict must be a string' % k)\n if not isinstance(v, basestring):\n raise ValueError('value %r in dict must be a string' % v)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _run_loop(self):\n while True:\n try:\n self.ioloop.start()\n except ZMQError as e:\n if e.errno == errno.EINTR:\n continue\n else:\n raise\n except Exception:\n if self._exiting:\n break\n else:\n raise\n else:\n break", "response": "Run the loop in the poller"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nqueuing a message to be sent from the IOLoop s thread.", "response": "def _queue_send(self, msg):\n \"\"\"Queue a message to be sent from the IOLoop's thread.\n \n Parameters\n ----------\n msg : message to send\n \n This is threadsafe, as it uses IOLoop.add_callback to give the loop's\n thread control of the action.\n \"\"\"\n def thread_send():\n self.session.send(self.stream, msg)\n self.ioloop.add_callback(thread_send)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _handle_recv(self, msg):\n ident,smsg = self.session.feed_identities(msg)\n self.call_handlers(self.session.unserialize(smsg))", "response": "callback for stream. on_recv\n \n unpacks message and calls handlers with it."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef run(self):\n self.socket = self.context.socket(zmq.DEALER)\n self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)\n self.socket.connect('tcp://%s:%i' % self.address)\n self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)\n self.stream.on_recv(self._handle_recv)\n self._run_loop()\n try:\n self.socket.close()\n except:\n pass", "response": "The main activity. Call start() instead."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexecute a Python code in the kernel and return the message_id.", "response": "def execute(self, code, silent=False,\n user_variables=None, user_expressions=None, allow_stdin=None):\n \"\"\"Execute code in the kernel.\n\n Parameters\n ----------\n code : str\n A string of Python code.\n\n silent : bool, optional (default False)\n If set, the kernel will execute the code as quietly possible.\n\n user_variables : list, optional\n A list of variable names to pull from the user's namespace. They\n will come back as a dict with these names as keys and their\n :func:`repr` as values.\n\n user_expressions : dict, optional\n A dict with string keys and to pull from the user's\n namespace. They will come back as a dict with these names as keys\n and their :func:`repr` as values.\n\n allow_stdin : bool, optional\n Flag for \n A dict with string keys and to pull from the user's\n namespace. They will come back as a dict with these names as keys\n and their :func:`repr` as values.\n\n Returns\n -------\n The msg_id of the message sent.\n \"\"\"\n if user_variables is None:\n user_variables = []\n if user_expressions is None:\n user_expressions = {}\n if allow_stdin is None:\n allow_stdin = self.allow_stdin\n \n \n # Don't waste network traffic if inputs are invalid\n if not isinstance(code, basestring):\n raise ValueError('code %r must be a string' % code)\n validate_string_list(user_variables)\n validate_string_dict(user_expressions)\n\n # Create class for content/msg creation. Related to, but possibly\n # not in Session.\n content = dict(code=code, silent=silent,\n user_variables=user_variables,\n user_expressions=user_expressions,\n allow_stdin=allow_stdin,\n )\n msg = self.session.msg('execute_request', content)\n self._queue_send(msg)\n return msg['header']['msg_id']"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef object_info(self, oname, detail_level=0):\n content = dict(oname=oname, detail_level=detail_level)\n msg = self.session.msg('object_info_request', content)\n self._queue_send(msg)\n return msg['header']['msg_id']", "response": "Get metadata about an object."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget entries from the history list.", "response": "def history(self, raw=True, output=False, hist_access_type='range', **kwargs):\n \"\"\"Get entries from the history list.\n\n Parameters\n ----------\n raw : bool\n If True, return the raw input.\n output : bool\n If True, then return the output as well.\n hist_access_type : str\n 'range' (fill in session, start and stop params), 'tail' (fill in n)\n or 'search' (fill in pattern param).\n\n session : int\n For a range request, the session from which to get lines. Session\n numbers are positive integers; negative ones count back from the\n current session.\n start : int\n The first line number of a history range.\n stop : int\n The final (excluded) line number of a history range.\n\n n : int\n The number of lines of history to get for a tail request.\n\n pattern : str\n The glob-syntax pattern for a search request.\n\n Returns\n -------\n The msg_id of the message sent.\n \"\"\"\n content = dict(raw=raw, output=output, hist_access_type=hist_access_type,\n **kwargs)\n msg = self.session.msg('history_request', content)\n self._queue_send(msg)\n return msg['header']['msg_id']"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrequesting an immediate kernel shutdown.", "response": "def shutdown(self, restart=False):\n \"\"\"Request an immediate kernel shutdown.\n\n Upon receipt of the (empty) reply, client code can safely assume that\n the kernel has shut down and it's safe to forcefully terminate it if\n it's still alive.\n\n The kernel will send the reply via a function registered with Python's\n atexit module, ensuring it's truly done as the kernel is done with all\n normal operation.\n \"\"\"\n # Send quit message to kernel. Once we implement kernel-side setattr,\n # this should probably be done that way, but for now this will do.\n msg = self.session.msg('shutdown_request', {'restart':restart})\n self._queue_send(msg)\n return msg['header']['msg_id']"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef flush(self, timeout=1.0):\n # We do the IOLoop callback process twice to ensure that the IOLoop\n # gets to perform at least one full poll.\n stop_time = time.time() + timeout\n for i in xrange(2):\n self._flushed = False\n self.ioloop.add_callback(self._flush)\n while not self._flushed and time.time() < stop_time:\n time.sleep(0.01)", "response": "Flushes all pending messages on the SUB channel."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsend a string of raw input to the kernel.", "response": "def input(self, string):\n \"\"\"Send a string of raw input to the kernel.\"\"\"\n content = dict(value=string)\n msg = self.session.msg('input_reply', content)\n self._queue_send(msg)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _poll(self, start_time):\n \n until_dead = self.time_to_dead - (time.time() - start_time)\n # ensure poll at least once\n until_dead = max(until_dead, 1e-3)\n events = []\n while True:\n try:\n events = self.poller.poll(1000 * until_dead)\n except ZMQError as e:\n if e.errno == errno.EINTR:\n # ignore interrupts during heartbeat\n # this may never actually happen\n until_dead = self.time_to_dead - (time.time() - start_time)\n until_dead = max(until_dead, 1e-3)\n pass\n else:\n raise\n except Exception:\n if self._exiting:\n break\n else:\n raise\n else:\n break\n return events", "response": "poll for heartbeat replies until we reach self. time_to_dead\nalid"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run(self):\n self._create_socket()\n self._running = True\n self._beating = True\n \n while self._running:\n if self._pause:\n # just sleep, and skip the rest of the loop\n time.sleep(self.time_to_dead)\n continue\n \n since_last_heartbeat = 0.0\n # io.rprint('Ping from HB channel') # dbg\n # no need to catch EFSM here, because the previous event was\n # either a recv or connect, which cannot be followed by EFSM\n self.socket.send(b'ping')\n request_time = time.time()\n ready = self._poll(request_time)\n if ready:\n self._beating = True\n # the poll above guarantees we have something to recv\n self.socket.recv()\n # sleep the remainder of the cycle\n remainder = self.time_to_dead - (time.time() - request_time)\n if remainder > 0:\n time.sleep(remainder)\n continue\n else:\n # nothing was received within the time limit, signal heart failure\n self._beating = False\n since_last_heartbeat = time.time() - request_time\n self.call_handlers(since_last_heartbeat)\n # and close/reopen the socket, because the REQ/REP cycle has been broken\n self._create_socket()\n continue\n try:\n self.socket.close()\n except:\n pass", "response": "The main activity. Call start() instead."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nam the heartbeat running and responsive and not paused?", "response": "def is_beating(self):\n \"\"\"Is the heartbeat running and responsive (and not paused).\"\"\"\n if self.is_alive() and not self._pause and self._beating:\n return True\n else:\n return False"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef start_channels(self, shell=True, sub=True, stdin=True, hb=True):\n if shell:\n self.shell_channel.start()\n if sub:\n self.sub_channel.start()\n if stdin:\n self.stdin_channel.start()\n self.shell_channel.allow_stdin = True\n else:\n self.shell_channel.allow_stdin = False\n if hb:\n self.hb_channel.start()", "response": "Starts the channels for this kernel."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstop all the running channels for this kernel.", "response": "def stop_channels(self):\n \"\"\"Stops all the running channels for this kernel.\n \"\"\"\n if self.shell_channel.is_alive():\n self.shell_channel.stop()\n if self.sub_channel.is_alive():\n self.sub_channel.stop()\n if self.stdin_channel.is_alive():\n self.stdin_channel.stop()\n if self.hb_channel.is_alive():\n self.hb_channel.stop()"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef channels_running(self):\n return (self.shell_channel.is_alive() or self.sub_channel.is_alive() or\n self.stdin_channel.is_alive() or self.hb_channel.is_alive())", "response": "Return True if any of the channels created and running?"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cleanup_connection_file(self):\n if self._connection_file_written:\n # cleanup connection files on full shutdown of kernel we started\n self._connection_file_written = False\n try:\n os.remove(self.connection_file)\n except OSError:\n pass", "response": "cleanup connection file if we wrote it *if we wrote it*"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_connection_file(self):\n with open(self.connection_file) as f:\n cfg = json.loads(f.read())\n \n self.ip = cfg['ip']\n self.shell_port = cfg['shell_port']\n self.stdin_port = cfg['stdin_port']\n self.iopub_port = cfg['iopub_port']\n self.hb_port = cfg['hb_port']\n self.session.key = str_to_bytes(cfg['key'])", "response": "load connection info from JSON dict in self. connection_file"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwrites the connection info to the connection file", "response": "def write_connection_file(self):\n \"\"\"write connection info to JSON dict in self.connection_file\"\"\"\n if self._connection_file_written:\n return\n self.connection_file,cfg = write_connection_file(self.connection_file,\n ip=self.ip, key=self.session.key,\n stdin_port=self.stdin_port, iopub_port=self.iopub_port,\n shell_port=self.shell_port, hb_port=self.hb_port)\n # write_connection_file also sets default ports:\n self.shell_port = cfg['shell_port']\n self.stdin_port = cfg['stdin_port']\n self.iopub_port = cfg['iopub_port']\n self.hb_port = cfg['hb_port']\n \n self._connection_file_written = True"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstarting a kernel process and configures the manager to use it.", "response": "def start_kernel(self, **kw):\n \"\"\"Starts a kernel process and configures the manager to use it.\n\n If random ports (port=0) are being used, this method must be called\n before the channels are created.\n\n Parameters:\n -----------\n launcher : callable, optional (default None)\n A custom function for launching the kernel process (generally a\n wrapper around ``entry_point.base_launch_kernel``). In most cases,\n it should not be necessary to use this parameter.\n\n **kw : optional\n See respective options for IPython and Python kernels.\n \"\"\"\n if self.ip not in LOCAL_IPS:\n raise RuntimeError(\"Can only launch a kernel on a local interface. \"\n \"Make sure that the '*_address' attributes are \"\n \"configured properly. \"\n \"Currently valid addresses are: %s\"%LOCAL_IPS\n )\n \n # write connection file / get default ports\n self.write_connection_file()\n\n self._launch_args = kw.copy()\n launch_kernel = kw.pop('launcher', None)\n if launch_kernel is None:\n from ipkernel import launch_kernel\n self.kernel = launch_kernel(fname=self.connection_file, **kw)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstop the kernel process and kills the kernel process if possible.", "response": "def shutdown_kernel(self, restart=False):\n \"\"\" Attempts to the stop the kernel process cleanly. If the kernel\n cannot be stopped, it is killed, if possible.\n \"\"\"\n # FIXME: Shutdown does not work on Windows due to ZMQ errors!\n if sys.platform == 'win32':\n self.kill_kernel()\n return\n\n # Pause the heart beat channel if it exists.\n if self._hb_channel is not None:\n self._hb_channel.pause()\n\n # Don't send any additional kernel kill messages immediately, to give\n # the kernel a chance to properly execute shutdown actions. Wait for at\n # most 1s, checking every 0.1s.\n self.shell_channel.shutdown(restart=restart)\n for i in range(10):\n if self.is_alive:\n time.sleep(0.1)\n else:\n break\n else:\n # OK, we've waited long enough.\n if self.has_kernel:\n self.kill_kernel()\n\n if not restart and self._connection_file_written:\n # cleanup connection files on full shutdown of kernel we started\n self._connection_file_written = False\n try:\n os.remove(self.connection_file)\n except IOError:\n pass"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrestart a kernel with the arguments that were used to launch it.", "response": "def restart_kernel(self, now=False, **kw):\n \"\"\"Restarts a kernel with the arguments that were used to launch it.\n\n If the old kernel was launched with random ports, the same ports will be\n used for the new kernel.\n\n Parameters\n ----------\n now : bool, optional\n If True, the kernel is forcefully restarted *immediately*, without\n having a chance to do any cleanup action. Otherwise the kernel is\n given 1s to clean up before a forceful restart is issued.\n\n In all cases the kernel is restarted, the only difference is whether\n it is given a chance to perform a clean shutdown or not.\n\n **kw : optional\n Any options specified here will replace those used to launch the\n kernel.\n \"\"\"\n if self._launch_args is None:\n raise RuntimeError(\"Cannot restart the kernel. \"\n \"No previous call to 'start_kernel'.\")\n else:\n # Stop currently running kernel.\n if self.has_kernel:\n if now:\n self.kill_kernel()\n else:\n self.shutdown_kernel(restart=True)\n\n # Start new kernel.\n self._launch_args.update(kw)\n self.start_kernel(**self._launch_args)\n\n # FIXME: Messages get dropped in Windows due to probable ZMQ bug\n # unless there is some delay here.\n if sys.platform == 'win32':\n time.sleep(0.2)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nkills the running kernel.", "response": "def kill_kernel(self):\n \"\"\" Kill the running kernel. \"\"\"\n if self.has_kernel:\n # Pause the heart beat channel if it exists.\n if self._hb_channel is not None:\n self._hb_channel.pause()\n\n # Attempt to kill the kernel.\n try:\n self.kernel.kill()\n except OSError, e:\n # In Windows, we will get an Access Denied error if the process\n # has already terminated. Ignore it.\n if sys.platform == 'win32':\n if e.winerror != 5:\n raise\n # On Unix, we may get an ESRCH error if the process has already\n # terminated. Ignore it.\n else:\n from errno import ESRCH\n if e.errno != ESRCH:\n raise\n self.kernel = None\n else:\n raise RuntimeError(\"Cannot kill kernel. No kernel is running!\")"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninterrupts the kernel. Unlike ``signal_kernel``, this operation is well supported on all platforms.", "response": "def interrupt_kernel(self):\n \"\"\" Interrupts the kernel. Unlike ``signal_kernel``, this operation is\n well supported on all platforms.\n \"\"\"\n if self.has_kernel:\n if sys.platform == 'win32':\n from parentpoller import ParentPollerWindows as Poller\n Poller.send_interrupt(self.kernel.win32_interrupt_event)\n else:\n self.kernel.send_signal(signal.SIGINT)\n else:\n raise RuntimeError(\"Cannot interrupt kernel. No kernel is running!\")"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending a signal to the kernel.", "response": "def signal_kernel(self, signum):\n \"\"\" Sends a signal to the kernel. Note that since only SIGTERM is\n supported on Windows, this function is only useful on Unix systems.\n \"\"\"\n if self.has_kernel:\n self.kernel.send_signal(signum)\n else:\n raise RuntimeError(\"Cannot signal kernel. No kernel is running!\")"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nam the kernel process still running?", "response": "def is_alive(self):\n \"\"\"Is the kernel process still running?\"\"\"\n if self.has_kernel:\n if self.kernel.poll() is None:\n return True\n else:\n return False\n elif self._hb_channel is not None:\n # We didn't start the kernel with this KernelManager so we\n # use the heartbeat.\n return self._hb_channel.is_beating()\n else:\n # no heartbeat and not local, we can't tell if it's running,\n # so naively return True\n return True"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the REQ socket channel object to make requests of the kernel.", "response": "def shell_channel(self):\n \"\"\"Get the REQ socket channel object to make requests of the kernel.\"\"\"\n if self._shell_channel is None:\n self._shell_channel = self.shell_channel_class(self.context,\n self.session,\n (self.ip, self.shell_port))\n return self._shell_channel"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sub_channel(self):\n if self._sub_channel is None:\n self._sub_channel = self.sub_channel_class(self.context,\n self.session,\n (self.ip, self.iopub_port))\n return self._sub_channel", "response": "Get the SUB socket channel object."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the REP socket channel object to handle stdin ( raw_input.", "response": "def stdin_channel(self):\n \"\"\"Get the REP socket channel object to handle stdin (raw_input).\"\"\"\n if self._stdin_channel is None:\n self._stdin_channel = self.stdin_channel_class(self.context,\n self.session,\n (self.ip, self.stdin_port))\n return self._stdin_channel"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the heartbeat socket channel object to check that the the kernel is alive.", "response": "def hb_channel(self):\n \"\"\"Get the heartbeat socket channel object to check that the\n kernel is alive.\"\"\"\n if self._hb_channel is None:\n self._hb_channel = self.hb_channel_class(self.context,\n self.session,\n (self.ip, self.hb_port))\n return self._hb_channel"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbinding an Engine s Kernel to be used as a full IPython kernel.", "response": "def bind_kernel(**kwargs):\n \"\"\"Bind an Engine's Kernel to be used as a full IPython kernel.\n \n This allows a running Engine to be used simultaneously as a full IPython kernel\n with the QtConsole or other frontends.\n \n This function returns immediately.\n \"\"\"\n from IPython.zmq.ipkernel import IPKernelApp\n from IPython.parallel.apps.ipengineapp import IPEngineApp\n \n # first check for IPKernelApp, in which case this should be a no-op\n # because there is already a bound kernel\n if IPKernelApp.initialized() and isinstance(IPKernelApp._instance, IPKernelApp):\n return\n \n if IPEngineApp.initialized():\n try:\n app = IPEngineApp.instance()\n except MultipleInstanceError:\n pass\n else:\n return app.bind_kernel(**kwargs)\n \n raise RuntimeError(\"bind_kernel be called from an IPEngineApp instance\")"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nemit a debugging message depending on the debugging level.", "response": "def debug(self, level, message):\n \"\"\"\n Emit a debugging message depending on the debugging level.\n\n :param level: The debugging level.\n :param message: The message to emit.\n \"\"\"\n\n if self._debug >= level:\n print(message, file=sys.stderr)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nretrieves the extension classes in proper priority order.", "response": "def _get_extension_classes(cls):\n \"\"\"\n Retrieve the extension classes in priority order.\n\n :returns: A list of extension classes, in proper priority\n order.\n \"\"\"\n\n if cls._extension_classes is None:\n exts = {}\n\n # Iterate over the entrypoints\n for ext in entry.points[NAMESPACE_EXTENSIONS]:\n exts.setdefault(ext.priority, [])\n exts[ext.priority].append(ext)\n\n # Save the list of extension classes\n cls._extension_classes = list(utils.iter_prio_dict(exts))\n\n return cls._extension_classes"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef prepare(cls, parser):\n\n debugger = ExtensionDebugger('prepare')\n\n for ext in cls._get_extension_classes():\n with debugger(ext):\n ext.prepare(parser)", "response": "Prepare all the extensions."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing the extensions that are being activated and returns an instance of the class that is returned by the activate method.", "response": "def activate(cls, ctxt, args):\n \"\"\"\n Initialize the extensions. This loops over each extension\n invoking its ``activate()`` method; those extensions that\n return an object are considered \"activated\" and will be called\n at later phases of extension processing.\n\n :param ctxt: An instance of ``timid.context.Context``.\n :param args: An instance of ``argparse.Namespace`` containing\n the result of processing command line arguments.\n\n :returns: An instance of ``ExtensionSet``.\n \"\"\"\n\n debugger = ExtensionDebugger('activate')\n\n exts = []\n for ext in cls._get_extension_classes():\n # Not using debugger as a context manager here, because we\n # want to know about the exception even if we're ignoring\n # it...but we need to notify which extension is being\n # processed!\n debugger(ext)\n\n try:\n # Check if the extension is being activated\n obj = ext.activate(ctxt, args)\n except Exception:\n # Hmmm, failed to activate; handle the error\n exc_info = sys.exc_info()\n if not debugger.__exit__(*exc_info):\n six.reraise(*exc_info)\n else:\n # OK, if the extension is activated, use it\n if obj is not None:\n exts.append(obj)\n debugger.debug(2, 'Activating extension \"%s.%s\"' %\n (ext.__module__, ext.__name__))\n\n # Initialize and return the ExtensionSet\n return cls(exts)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall after adding them to the list of test steps.", "response": "def read_steps(self, ctxt, steps):\n \"\"\"\n Called after reading steps, prior to adding them to the list of\n test steps. Extensions are able to alter the list (in place).\n\n :param ctxt: An instance of ``timid.context.Context``.\n :param steps: A list of ``timid.steps.Step`` instances.\n\n :returns: The ``steps`` parameter, for convenience.\n \"\"\"\n\n debugger = ExtensionDebugger('read_steps')\n\n for ext in self.exts:\n with debugger(ext):\n ext.read_steps(ctxt, steps)\n\n # Convenience return\n return steps"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalls prior to executing a step. Returns True if the step is to be skipped False otherwise.", "response": "def pre_step(self, ctxt, step, idx):\n \"\"\"\n Called prior to executing a step.\n\n :param ctxt: An instance of ``timid.context.Context``.\n :param step: An instance of ``timid.steps.Step`` describing\n the step to be executed.\n :param idx: The index of the step in the list of steps.\n\n :returns: A ``True`` value if the step is to be skipped,\n ``False`` otherwise.\n \"\"\"\n\n debugger = ExtensionDebugger('pre_step')\n\n for ext in self.exts:\n with debugger(ext):\n if ext.pre_step(ctxt, step, idx):\n # Step must be skipped\n debugger.debug(3, 'Skipping step %d' % idx)\n return True\n\n return False"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef post_step(self, ctxt, step, idx, result):\n\n debugger = ExtensionDebugger('post_step')\n\n for ext in self.exts:\n with debugger(ext):\n ext.post_step(ctxt, step, idx, result)\n\n # Convenience return\n return result", "response": "Called after executing a step."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef finalize(self, ctxt, result):\n\n debugger = ExtensionDebugger('finalize')\n\n for ext in self.exts:\n with debugger(ext):\n result = ext.finalize(ctxt, result)\n\n return result", "response": "Called at the end of processing."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwalk an unpacked egg s contents skipping the EGG - INFO directory", "response": "def walk_egg(egg_dir):\n \"\"\"Walk an unpacked egg's contents, skipping the metadata directory\"\"\"\n walker = os.walk(egg_dir)\n base,dirs,files = walker.next()\n if 'EGG-INFO' in dirs:\n dirs.remove('EGG-INFO')\n yield base,dirs,files\n for bdf in walker:\n yield bdf"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nscans a module for unsafe - for - zipfiles.", "response": "def scan_module(egg_dir, base, name, stubs):\n \"\"\"Check whether module possibly uses unsafe-for-zipfile stuff\"\"\"\n\n filename = os.path.join(base,name)\n if filename[:-1] in stubs:\n return True # Extension module\n pkg = base[len(egg_dir)+1:].replace(os.sep,'.')\n module = pkg+(pkg and '.' or '')+os.path.splitext(name)[0]\n if sys.version_info < (3, 3):\n skip = 8 # skip magic & date\n else:\n skip = 12 # skip magic & date & file size\n f = open(filename,'rb'); f.read(skip)\n code = marshal.load(f); f.close()\n safe = True\n symbols = dict.fromkeys(iter_symbols(code))\n for bad in ['__file__', '__path__']:\n if bad in symbols:\n log.warn(\"%s: module references %s\", module, bad)\n safe = False\n if 'inspect' in symbols:\n for bad in [\n 'getsource', 'getabsfile', 'getsourcefile', 'getfile'\n 'getsourcelines', 'findsource', 'getcomments', 'getframeinfo',\n 'getinnerframes', 'getouterframes', 'stack', 'trace'\n ]:\n if bad in symbols:\n log.warn(\"%s: module MAY be using inspect.%s\", module, bad)\n safe = False\n if '__name__' in symbols and '__main__' in symbols and '.' not in module:\n if sys.version[:3]==\"2.4\": # -m works w/zipfiles in 2.5\n log.warn(\"%s: top-level module may be 'python -m' script\", module)\n safe = False\n return safe"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_init_files(self):\n init_files = []\n for base,dirs,files in walk_egg(self.bdist_dir):\n if base==self.bdist_dir:\n # don't put an __init__ in the root\n continue\n for name in files:\n if name.endswith('.py'):\n if '__init__.py' not in files:\n pkg = base[len(self.bdist_dir)+1:].replace(os.sep,'.')\n if self.distribution.has_contents_for(pkg):\n log.warn(\"Creating missing __init__.py for %s\",pkg)\n filename = os.path.join(base,'__init__.py')\n if not self.dry_run:\n f = open(filename,'w'); f.write(NS_PKG_STUB)\n f.close()\n init_files.append(filename)\n break\n else:\n # not a package, don't traverse to subdirectories\n dirs[:] = []\n\n return init_files", "response": "Create missing package __init__ files"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef launch_new_instance():\n if sys.platform == 'win32':\n # make sure we don't get called from a multiprocessing subprocess\n # this can result in infinite Controllers being started on Windows\n # which doesn't have a proper fork, so multiprocessing is wonky\n \n # this only comes up when IPython has been installed using vanilla\n # setuptools, and *not* distribute.\n import multiprocessing\n p = multiprocessing.current_process()\n # the main process has name 'MainProcess'\n # subprocesses will have names like 'Process-1'\n if p.name != 'MainProcess':\n # we are a subprocess, don't start another Controller!\n return\n app = IPControllerApp.instance()\n app.initialize()\n app.start()", "response": "Create and run the IPython controller"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves a connection dict to json file", "response": "def save_connection_dict(self, fname, cdict):\n \"\"\"save a connection dict to json file.\"\"\"\n c = self.config\n url = cdict['url']\n location = cdict['location']\n if not location:\n try:\n proto,ip,port = split_url(url)\n except AssertionError:\n pass\n else:\n try:\n location = socket.gethostbyname_ex(socket.gethostname())[2][-1]\n except (socket.gaierror, IndexError):\n self.log.warn(\"Could not identify this machine's IP, assuming 127.0.0.1.\"\n \" You may need to specify '--location=' to help\"\n \" IPython decide when to connect via loopback.\")\n location = '127.0.0.1'\n cdict['location'] = location\n fname = os.path.join(self.profile_dir.security_dir, fname)\n self.log.info(\"writing connection info to %s\", fname)\n with open(fname, 'w') as f:\n f.write(json.dumps(cdict, indent=2))\n os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_config_from_json(self):\n c = self.config\n self.log.debug(\"loading config from JSON\")\n # load from engine config\n fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)\n self.log.info(\"loading connection info from %s\", fname)\n with open(fname) as f:\n cfg = json.loads(f.read())\n key = cfg['exec_key']\n # json gives unicode, Session.key wants bytes\n c.Session.key = key.encode('ascii')\n xport,addr = cfg['url'].split('://')\n c.HubFactory.engine_transport = xport\n ip,ports = addr.split(':')\n c.HubFactory.engine_ip = ip\n c.HubFactory.regport = int(ports)\n self.location = cfg['location']\n if not self.engine_ssh_server:\n self.engine_ssh_server = cfg['ssh']\n # load client config\n fname = os.path.join(self.profile_dir.security_dir, self.client_json_file)\n self.log.info(\"loading connection info from %s\", fname)\n with open(fname) as f:\n cfg = json.loads(f.read())\n assert key == cfg['exec_key'], \"exec_key mismatch between engine and client keys\"\n xport,addr = cfg['url'].split('://')\n c.HubFactory.client_transport = xport\n ip,ports = addr.split(':')\n c.HubFactory.client_ip = ip\n if not self.ssh_server:\n self.ssh_server = cfg['ssh']\n assert int(ports) == c.HubFactory.regport, \"regport mismatch\"", "response": "load config from existing json connector files."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading secondary config from JSON and setting defaults", "response": "def load_secondary_config(self):\n \"\"\"secondary config, loading from JSON and setting defaults\"\"\"\n if self.reuse_files:\n try:\n self.load_config_from_json()\n except (AssertionError,IOError) as e:\n self.log.error(\"Could not load config from JSON: %s\" % e)\n else:\n # successfully loaded config from JSON, and reuse=True\n # no need to wite back the same file\n self.write_connection_files = False\n \n # switch Session.key default to secure\n default_secure(self.config)\n self.log.debug(\"Config changed\")\n self.log.debug(repr(self.config))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef script_args(f):\n args = [\n magic_arguments.argument(\n '--out', type=str,\n help=\"\"\"The variable in which to store stdout from the script.\n If the script is backgrounded, this will be the stdout *pipe*,\n instead of the stderr text itself.\n \"\"\"\n ),\n magic_arguments.argument(\n '--err', type=str,\n help=\"\"\"The variable in which to store stderr from the script.\n If the script is backgrounded, this will be the stderr *pipe*,\n instead of the stderr text itself.\n \"\"\"\n ),\n magic_arguments.argument(\n '--bg', action=\"store_true\",\n help=\"\"\"Whether to run the script in the background.\n If given, the only way to see the output of the command is\n with --out/err.\n \"\"\"\n ),\n magic_arguments.argument(\n '--proc', type=str,\n help=\"\"\"The variable in which to store Popen instance.\n This is used only when --bg option is given.\n \"\"\"\n ),\n ]\n for arg in args:\n f = arg(f)\n return f", "response": "add script args to the function f"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef exec_args(f):\n args = [\n magic_arguments.argument('-b', '--block', action=\"store_const\",\n const=True, dest='block',\n help=\"use blocking (sync) execution\",\n ),\n magic_arguments.argument('-a', '--noblock', action=\"store_const\",\n const=False, dest='block',\n help=\"use non-blocking (async) execution\",\n ),\n magic_arguments.argument('-t', '--targets', type=str,\n help=\"specify the targets on which to execute\",\n ),\n magic_arguments.argument('--verbose', action=\"store_const\",\n const=True, dest=\"set_verbose\",\n help=\"print a message at each execution\",\n ),\n magic_arguments.argument('--no-verbose', action=\"store_const\",\n const=False, dest=\"set_verbose\",\n help=\"don't print any messages\",\n ),\n ]\n for a in args:\n f = a(f)\n return f", "response": "decorator for adding block and targets args for execution\n \n applied to %pxconfig and %%px\n "} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef output_args(f):\n args = [\n magic_arguments.argument('-r', action=\"store_const\", dest='groupby',\n const='order',\n help=\"collate outputs in order (same as group-outputs=order)\"\n ),\n magic_arguments.argument('-e', action=\"store_const\", dest='groupby',\n const='engine',\n help=\"group outputs by engine (same as group-outputs=engine)\"\n ),\n magic_arguments.argument('--group-outputs', dest='groupby', type=str,\n choices=['engine', 'order', 'type'], default='type',\n help=\"\"\"Group the outputs in a particular way.\n \n Choices are:\n \n type: group outputs of all engines by type (stdout, stderr, displaypub, etc.).\n \n engine: display all output for each engine together.\n\n order: like type, but individual displaypub output from each engine is collated.\n For example, if multiple plots are generated by each engine, the first\n figure of each engine will be displayed, then the second of each, etc.\n \"\"\"\n ),\n magic_arguments.argument('-o', '--out', dest='save_name', type=str,\n help=\"\"\"store the AsyncResult object for this computation\n in the global namespace under this name.\n \"\"\"\n ),\n ]\n for a in args:\n f = a(f)\n return f", "response": "decorator for output - formatting args"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pxconfig(self, line):\n args = magic_arguments.parse_argstring(self.pxconfig, line)\n if args.targets:\n self.view.targets = self._eval_target_str(args.targets)\n if args.block is not None:\n self.view.block = args.block\n if args.set_verbose is not None:\n self.verbose = args.set_verbose", "response": "configure default targets blocking for %px magics"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprinting the result of the last asynchronous command.", "response": "def result(self, line=''):\n \"\"\"Print the result of the last asynchronous %px command.\n \n This lets you recall the results of %px computations after\n asynchronous submission (block=False).\n\n Examples\n --------\n ::\n\n In [23]: %px os.getpid()\n Async parallel execution on engine(s): all\n\n In [24]: %pxresult\n Out[8:10]: 60920\n Out[9:10]: 60921\n Out[10:10]: 60922\n Out[11:10]: 60923\n \"\"\"\n args = magic_arguments.parse_argstring(self.result, line)\n \n if self.last_result is None:\n raise UsageError(NO_LAST_RESULT)\n \n self.last_result.get()\n self.last_result.display_outputs(groupby=args.groupby)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cell_px(self, line='', cell=None):\n \n args = magic_arguments.parse_argstring(self.cell_px, line)\n \n if args.targets:\n save_targets = self.view.targets\n self.view.targets = self._eval_target_str(args.targets)\n try:\n return self.parallel_execute(cell, block=args.block,\n groupby=args.groupby,\n save_name=args.save_name,\n )\n finally:\n if args.targets:\n self.view.targets = save_targets", "response": "Executes the cell in parallel."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _enable_autopx(self):\n # override run_cell\n self._original_run_cell = self.shell.run_cell\n self.shell.run_cell = self.pxrun_cell\n\n self._autopx = True\n print \"%autopx enabled\"", "response": "Enable autopx mode by saving the original run_cell and installing\n pxrun_cell."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndisable automatic mode for the current instance of the class.", "response": "def _disable_autopx(self):\n \"\"\"Disable %autopx by restoring the original InteractiveShell.run_cell.\n \"\"\"\n if self._autopx:\n self.shell.run_cell = self._original_run_cell\n self._autopx = False\n print \"%autopx disabled\""} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pxrun_cell(self, raw_cell, store_history=False, silent=False):\n\n if (not raw_cell) or raw_cell.isspace():\n return\n\n ipself = self.shell\n\n with ipself.builtin_trap:\n cell = ipself.prefilter_manager.prefilter_lines(raw_cell)\n\n # Store raw and processed history\n if store_history:\n ipself.history_manager.store_inputs(ipself.execution_count,\n cell, raw_cell)\n\n # ipself.logger.log(cell, raw_cell)\n\n cell_name = ipself.compile.cache(cell, ipself.execution_count)\n\n try:\n ast.parse(cell, filename=cell_name)\n except (OverflowError, SyntaxError, ValueError, TypeError,\n MemoryError):\n # Case 1\n ipself.showsyntaxerror()\n ipself.execution_count += 1\n return None\n except NameError:\n # ignore name errors, because we don't know the remote keys\n pass\n\n if store_history:\n # Write output to the database. Does nothing unless\n # history output logging is enabled.\n ipself.history_manager.store_output(ipself.execution_count)\n # Each cell is a *single* input, regardless of how many lines it has\n ipself.execution_count += 1\n if re.search(r'get_ipython\\(\\)\\.magic\\(u?[\"\\']%?autopx', cell):\n self._disable_autopx()\n return False\n else:\n try:\n result = self.view.execute(cell, silent=False, block=False)\n except:\n ipself.showtraceback()\n return True\n else:\n if self.view.block:\n try:\n result.get()\n except:\n self.shell.showtraceback()\n return True\n else:\n with ipself.builtin_trap:\n result.display_outputs()\n return False", "response": "Execute a single cell and return the result."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef run_heartbeat(message):\n then = arrow.get(message['time'])\n now = arrow.get()\n\n if (now - then) > timezone.timedelta(seconds=(TICK_FREQ+1)):\n pass # discard old ticks\n else:\n Task.run_tasks()", "response": "Internal consumer to process tasks"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npatches the protocol s makeConnection and connectionLost methods to make the protocol behave more like what Agent expects.", "response": "def patch_protocol_for_agent(protocol):\n \"\"\"\n Patch the protocol's makeConnection and connectionLost methods to make the\n protocol and its transport behave more like what `Agent` expects.\n\n While `Agent` is the driving force behind this, other clients and servers\n will no doubt have similar requirements.\n \"\"\"\n old_makeConnection = protocol.makeConnection\n old_connectionLost = protocol.connectionLost\n\n def new_makeConnection(transport):\n patch_transport_fake_push_producer(transport)\n patch_transport_abortConnection(transport, protocol)\n return old_makeConnection(transport)\n\n def new_connectionLost(reason):\n # Replace ConnectionDone with ConnectionAborted if we aborted.\n if protocol._fake_connection_aborted and reason.check(ConnectionDone):\n reason = Failure(ConnectionAborted())\n return old_connectionLost(reason)\n\n protocol.makeConnection = new_makeConnection\n protocol.connectionLost = new_connectionLost\n protocol._fake_connection_aborted = False"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npatch a method onto an object if it isn t already there.", "response": "def patch_if_missing(obj, name, method):\n \"\"\"\n Patch a method onto an object if it isn't already there.\n \"\"\"\n setattr(obj, name, getattr(obj, name, method))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef patch_transport_fake_push_producer(transport):\n patch_if_missing(transport, 'pauseProducing', lambda: None)\n patch_if_missing(transport, 'resumeProducing', lambda: None)\n patch_if_missing(transport, 'stopProducing', transport.loseConnection)", "response": "Patch the transport to have the same methods as IPushProducer."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef patch_transport_abortConnection(transport, protocol):\n def abortConnection():\n protocol._fake_connection_aborted = True\n transport.loseConnection()\n patch_if_missing(transport, 'abortConnection', abortConnection)", "response": "Patch abortConnection onto the transport."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef accept_connection(self):\n assert self.pending, \"Connection is not pending.\"\n self.server_protocol = self.server.server_factory.buildProtocol(None)\n self._accept_d.callback(\n FakeServerProtocolWrapper(self, self.server_protocol))\n return self.await_connected()", "response": "Accept a pending connection."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrejecting a pending connection.", "response": "def reject_connection(self, reason=None):\n \"\"\"\n Reject a pending connection.\n \"\"\"\n assert self.pending, \"Connection is not pending.\"\n if reason is None:\n reason = ConnectionRefusedError()\n self._accept_d.errback(reason)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_agent(self, reactor=None, contextFactory=None):\n return ProxyAgentWithContext(\n self.endpoint, reactor=reactor, contextFactory=contextFactory)", "response": "Returns an IAgent that makes requests to this fake server."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nhandling the validation of the object.", "response": "def form_valid(self, form):\n \"\"\"\n Calls pre and post save hooks.\n \"\"\"\n self.object = form.save(commit=False)\n\n # Invoke pre_save hook, and allow it to abort the saving\n # process and do a redirect.\n response = self.pre_save(self.object)\n if response:\n return response\n\n self.object.save()\n form.save_m2m()\n self.post_save(self.object)\n\n return HttpResponseRedirect(self.get_success_url())"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndelete the object and all its children.", "response": "def delete(self, request, *args, **kwargs):\n \"\"\"\n Calls pre and post delete hooks for DelteViews.\n \"\"\"\n\n self.object = self.get_object()\n success_url = self.get_success_url()\n self.pre_delete(self.object)\n self.object.delete()\n self.post_delete(self.object)\n\n return HttpResponseRedirect(success_url)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsupplying user object as initial data for the specified user_field.", "response": "def get_initial(self):\n \"\"\"\n Supply user object as initial data for the specified user_field(s).\n \"\"\"\n\n data = super(UserViewMixin, self).get_initial()\n for k in self.user_field:\n data[k] = self.request.user\n\n return data"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the user of the current user in the current record.", "response": "def pre_save(self, instance):\n super(UserViewMixin, self).pre_save(instance)\n\n \"\"\"\n Use SaveHookMixin pre_save to set the user.\n \"\"\"\n if self.request.user.is_authenticated():\n for field in self.user_field:\n setattr(instance, field, self.request.user)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef report(self, morfs, outfile=None):\n self.find_code_units(morfs)\n\n # Prepare the formatting strings\n max_name = max([len(cu.name) for cu in self.code_units] + [5])\n fmt_name = \"%%- %ds \" % max_name\n fmt_err = \"%s %s: %s\\n\"\n header = (fmt_name % \"Name\") + \" Stmts Miss\"\n fmt_coverage = fmt_name + \"%6d %6d\"\n if self.branches:\n header += \" Branch BrMiss\"\n fmt_coverage += \" %6d %6d\"\n width100 = Numbers.pc_str_width()\n header += \"%*s\" % (width100+4, \"Cover\")\n fmt_coverage += \"%%%ds%%%%\" % (width100+3,)\n if self.config.show_missing:\n header += \" Missing\"\n fmt_coverage += \" %s\"\n rule = \"-\" * len(header) + \"\\n\"\n header += \"\\n\"\n fmt_coverage += \"\\n\"\n\n if not outfile:\n outfile = sys.stdout\n\n # Write the header\n outfile.write(header)\n outfile.write(rule)\n\n total = Numbers()\n\n for cu in self.code_units:\n try:\n analysis = self.coverage._analyze(cu)\n nums = analysis.numbers\n args = (cu.name, nums.n_statements, nums.n_missing)\n if self.branches:\n args += (nums.n_branches, nums.n_missing_branches)\n args += (nums.pc_covered_str,)\n if self.config.show_missing:\n args += (analysis.missing_formatted(),)\n outfile.write(fmt_coverage % args)\n total += nums\n except KeyboardInterrupt: # pragma: not covered\n raise\n except:\n report_it = not self.config.ignore_errors\n if report_it:\n typ, msg = sys.exc_info()[:2]\n if typ is NotPython and not cu.should_be_python():\n report_it = False\n if report_it:\n outfile.write(fmt_err % (cu.name, typ.__name__, msg))\n\n if total.n_files > 1:\n outfile.write(rule)\n args = (\"TOTAL\", total.n_statements, total.n_missing)\n if self.branches:\n args += (total.n_branches, total.n_missing_branches)\n args += (total.pc_covered_str,)\n if self.config.show_missing:\n args += (\"\",)\n outfile.write(fmt_coverage % args)\n\n return total.pc_covered", "response": "Writes a report summarizing coverage statistics per module."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_compiled_ext():\n for ext, mode, typ in imp.get_suffixes():\n if typ == imp.PY_COMPILED:\n return ext", "response": "Get the extension of compiled files."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreplace stuff in the __dict__ of a class with new.", "response": "def update_class(old, new):\n \"\"\"Replace stuff in the __dict__ of a class, and upgrade\n method code objects\"\"\"\n for key in old.__dict__.keys():\n old_obj = getattr(old, key)\n\n try:\n new_obj = getattr(new, key)\n except AttributeError:\n # obsolete attribute: remove it\n try:\n delattr(old, key)\n except (AttributeError, TypeError):\n pass\n continue\n\n if update_generic(old_obj, new_obj): continue\n\n try:\n setattr(old, key, getattr(new, key))\n except (AttributeError, TypeError):\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks whether some modules need to be reloaded.", "response": "def check(self, check_all=False):\n \"\"\"Check whether some modules need to be reloaded.\"\"\"\n\n if not self.enabled and not check_all:\n return\n\n if check_all or self.check_all:\n modules = sys.modules.keys()\n else:\n modules = self.modules.keys()\n\n for modname in modules:\n m = sys.modules.get(modname, None)\n\n if modname in self.skip_modules:\n continue\n\n if not hasattr(m, '__file__'):\n continue\n\n if m.__name__ == '__main__':\n # we cannot reload(__main__)\n continue\n\n filename = m.__file__\n path, ext = os.path.splitext(filename)\n\n if ext.lower() == '.py':\n ext = PY_COMPILED_EXT\n pyc_filename = pyfile.cache_from_source(filename)\n py_filename = filename\n else:\n pyc_filename = filename\n try:\n py_filename = pyfile.source_from_cache(filename)\n except ValueError:\n continue\n\n try:\n pymtime = os.stat(py_filename).st_mtime\n if pymtime <= os.stat(pyc_filename).st_mtime:\n continue\n if self.failed.get(py_filename, None) == pymtime:\n continue\n except OSError:\n continue\n\n try:\n superreload(m, reload, self.old_objects)\n if py_filename in self.failed:\n del self.failed[py_filename]\n except:\n print >> sys.stderr, \"[autoreload of %s failed: %s]\" % (\n modname, traceback.format_exc(1))\n self.failed[py_filename] = pymtime"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nopen the default editor at the given filename and line number.", "response": "def editor(self, filename, linenum=None, wait=True):\n \"\"\"Open the default editor at the given filename and linenumber.\n\n This is IPython's default editor hook, you can use it as an example to\n write your own modified one. To set your own editor function as the\n new editor hook, call ip.set_hook('editor',yourfunc).\"\"\"\n\n # IPython configures a default editor at startup by reading $EDITOR from\n # the environment, and falling back on vi (unix) or notepad (win32).\n editor = self.editor\n\n # marker for at which line to open the file (for existing objects)\n if linenum is None or editor=='notepad':\n linemark = ''\n else:\n linemark = '+%d' % int(linenum)\n\n # Enclose in quotes if necessary and legal\n if ' ' in editor and os.path.isfile(editor) and editor[0] != '\"':\n editor = '\"%s\"' % editor\n\n # Call the actual editor\n proc = subprocess.Popen('%s %s %s' % (editor, linemark, filename),\n shell=True)\n if wait and proc.wait() != 0:\n raise TryNext()"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fix_error_editor(self,filename,linenum,column,msg):\n def vim_quickfix_file():\n t = tempfile.NamedTemporaryFile()\n t.write('%s:%d:%d:%s\\n' % (filename,linenum,column,msg))\n t.flush()\n return t\n if os.path.basename(self.editor) != 'vim':\n self.hooks.editor(filename,linenum)\n return\n t = vim_quickfix_file()\n try:\n if os.system('vim --cmd \"set errorformat=%f:%l:%c:%m\" -q ' + t.name):\n raise TryNext()\n finally:\n t.close()", "response": "Open the editor at the given filename linenumber column and show an error message."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef clipboard_get(self):\n from IPython.lib.clipboard import (\n osx_clipboard_get, tkinter_clipboard_get,\n win32_clipboard_get\n )\n if sys.platform == 'win32':\n chain = [win32_clipboard_get, tkinter_clipboard_get]\n elif sys.platform == 'darwin':\n chain = [osx_clipboard_get, tkinter_clipboard_get]\n else:\n chain = [tkinter_clipboard_get]\n dispatcher = CommandChainDispatcher()\n for func in chain:\n dispatcher.add(func)\n text = dispatcher()\n return text", "response": "Get text from the clipboard."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd a function to the cmd chain with given priority", "response": "def add(self, func, priority=0):\n \"\"\" Add a func to the cmd chain with given priority \"\"\"\n self.chain.append((priority, func))\n self.chain.sort(key=lambda x: x[0])"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ntry to create a Distribution a Wheel a Develop or a Sdist file.", "response": "def get_metadata(path_or_module, metadata_version=None):\n \"\"\" Try to create a Distribution 'path_or_module'.\n \n o 'path_or_module' may be a module object.\n\n o If a string, 'path_or_module' may point to an sdist file, a bdist\n file, an installed package, or a working checkout (if it contains\n PKG-INFO).\n \n o Return None if 'path_or_module' can't be parsed.\n \"\"\"\n if isinstance(path_or_module, ModuleType):\n try:\n return Installed(path_or_module, metadata_version)\n except (ValueError, IOError): #pragma NO COVER\n pass\n\n try:\n __import__(path_or_module)\n except ImportError:\n pass\n else:\n try:\n return Installed(path_or_module, metadata_version)\n except (ValueError, IOError): #pragma NO COVER\n pass\n\n if os.path.isfile(path_or_module):\n try:\n return SDist(path_or_module, metadata_version)\n except (ValueError, IOError):\n pass\n\n try:\n return BDist(path_or_module, metadata_version)\n except (ValueError, IOError): #pragma NO COVER\n pass\n\n try:\n return Wheel(path_or_module, metadata_version)\n except (ValueError, IOError): #pragma NO COVER\n pass\n\n if os.path.isdir(path_or_module):\n try:\n return Develop(path_or_module, metadata_version)\n except (ValueError, IOError): #pragma NO COVER\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconfigures which kinds of exceptions trigger plugin.", "response": "def configure(self, options, conf):\n \"\"\"Configure which kinds of exceptions trigger plugin.\n \"\"\"\n self.conf = conf\n self.enabled = options.debugErrors or options.debugFailures\n self.enabled_for_errors = options.debugErrors\n self.enabled_for_failures = options.debugFailures"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef obfuscate(email, linktext=None, autoescape=None):\n if autoescape:\n esc = conditional_escape\n else:\n esc = lambda x: x\n\n email = re.sub('@', '\\\\\\\\100', re.sub('\\.', '\\\\\\\\056', esc(email))).encode('rot13')\n\n if linktext:\n linktext = esc(linktext).encode('rot13')\n else:\n linktext = email\n\n rotten_link = \"\"\"\"\"\" % (email, linktext)\n return mark_safe(rotten_link)", "response": "Returns a string representing an email address with rot13 JavaScript obfuscation."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef roundplus(number):\n num = str(number)\n if not num.isdigit():\n return num\n\n num = str(number)\n digits = len(num)\n rounded = '100+'\n\n if digits < 3:\n rounded = num\n elif digits == 3:\n rounded = num[0] + '00+'\n elif digits == 4:\n rounded = num[0] + 'K+'\n elif digits == 5:\n rounded = num[:1] + 'K+'\n else:\n rounded = '100K+'\n\n return rounded", "response": "This function rounds the number given an integer and returns the number as a string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nimport and return bar given the string foo. bar.", "response": "def import_item(name):\n \"\"\"Import and return bar given the string foo.bar.\"\"\"\n package = '.'.join(name.split('.')[0:-1])\n obj = name.split('.')[-1]\n \n # Note: the original code for this was the following. We've left it\n # visible for now in case the new implementation shows any problems down\n # the road, to make it easier on anyone looking for a problem. This code\n # should be removed once we're comfortable we didn't break anything.\n \n ## execString = 'from %s import %s' % (package, obj)\n ## try:\n ## exec execString\n ## except SyntaxError:\n ## raise ImportError(\"Invalid class specification: %s\" % name)\n ## exec 'temp = %s' % obj\n ## return temp\n\n if package:\n module = __import__(package,fromlist=[obj])\n try:\n pak = module.__dict__[obj]\n except KeyError:\n raise ImportError('No module named %s' % obj)\n return pak\n else:\n return __import__(obj)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef try_passwordless_ssh(server, keyfile, paramiko=None):\n if paramiko is None:\n paramiko = sys.platform == 'win32'\n if not paramiko:\n f = _try_passwordless_openssh\n else:\n f = _try_passwordless_paramiko\n return f(server, keyfile)", "response": "Attempt to make an ssh connection to a server with a password."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _try_passwordless_openssh(server, keyfile):\n if pexpect is None:\n raise ImportError(\"pexpect unavailable, use paramiko\")\n cmd = 'ssh -f '+ server\n if keyfile:\n cmd += ' -i ' + keyfile\n cmd += ' exit'\n p = pexpect.spawn(cmd)\n while True:\n try:\n p.expect('[Pp]assword:', timeout=.1)\n except pexpect.TIMEOUT:\n continue\n except pexpect.EOF:\n return True\n else:\n return False", "response": "Try passwordless login with shell ssh command."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _try_passwordless_paramiko(server, keyfile):\n if paramiko is None:\n msg = \"Paramiko unavaliable, \"\n if sys.platform == 'win32':\n msg += \"Paramiko is required for ssh tunneled connections on Windows.\"\n else:\n msg += \"use OpenSSH.\"\n raise ImportError(msg)\n username, server, port = _split_server(server)\n client = paramiko.SSHClient()\n client.load_system_host_keys()\n client.set_missing_host_key_policy(paramiko.WarningPolicy())\n try:\n client.connect(server, port, username=username, key_filename=keyfile,\n look_for_keys=True)\n except paramiko.AuthenticationException:\n return False\n else:\n client.close()\n return True", "response": "Try passwordless login with paramiko."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef tunnel_connection(socket, addr, server, keyfile=None, password=None, paramiko=None, timeout=60):\n new_url, tunnel = open_tunnel(addr, server, keyfile=keyfile, password=password, paramiko=paramiko, timeout=timeout)\n socket.connect(new_url)\n return tunnel", "response": "Connect a socket to an address via an ssh tunnel."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nopen a tunneled connection from a 0MQ url.", "response": "def open_tunnel(addr, server, keyfile=None, password=None, paramiko=None, timeout=60):\n \"\"\"Open a tunneled connection from a 0MQ url.\n\n For use inside tunnel_connection.\n\n Returns\n -------\n\n (url, tunnel): The 0MQ url that has been forwarded, and the tunnel object\n \"\"\"\n\n lport = select_random_ports(1)[0]\n transport, addr = addr.split('://')\n ip,rport = addr.split(':')\n rport = int(rport)\n if paramiko is None:\n paramiko = sys.platform == 'win32'\n if paramiko:\n tunnelf = paramiko_tunnel\n else:\n tunnelf = openssh_tunnel\n\n tunnel = tunnelf(lport, rport, server, remoteip=ip, keyfile=keyfile, password=password, timeout=timeout)\n return 'tcp://127.0.0.1:%i'%lport, tunnel"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate an ssh tunnel for connecting to a remote machine.", "response": "def openssh_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60):\n \"\"\"Create an ssh tunnel using command-line ssh that connects port lport\n on this machine to localhost:rport on server. The tunnel\n will automatically close when not in use, remaining open\n for a minimum of timeout seconds for an initial connection.\n\n This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`,\n as seen from `server`.\n\n keyfile and password may be specified, but ssh config is checked for defaults.\n\n Parameters\n ----------\n\n lport : int\n local port for connecting to the tunnel from this machine.\n rport : int\n port on the remote machine to connect to.\n server : str\n The ssh server to connect to. The full ssh server string will be parsed.\n user@server:port\n remoteip : str [Default: 127.0.0.1]\n The remote ip, specifying the destination of the tunnel.\n Default is localhost, which means that the tunnel would redirect\n localhost:lport on this machine to localhost:rport on the *server*.\n\n keyfile : str; path to public key file\n This specifies a key to be used in ssh login, default None.\n Regular default ssh keys will be used without specifying this argument.\n password : str;\n Your ssh password to the ssh server. Note that if this is left None,\n you will be prompted for it if passwordless key based login is unavailable.\n timeout : int [default: 60]\n The time (in seconds) after which no activity will result in the tunnel\n closing. This prevents orphaned tunnels from running forever.\n \"\"\"\n if pexpect is None:\n raise ImportError(\"pexpect unavailable, use paramiko_tunnel\")\n ssh=\"ssh \"\n if keyfile:\n ssh += \"-i \" + keyfile\n \n if ':' in server:\n server, port = server.split(':')\n ssh += \" -p %s\" % port\n \n cmd = \"%s -f -L 127.0.0.1:%i:%s:%i %s sleep %i\" % (\n ssh, lport, remoteip, rport, server, timeout)\n tunnel = pexpect.spawn(cmd)\n failed = False\n while True:\n try:\n tunnel.expect('[Pp]assword:', timeout=.1)\n except pexpect.TIMEOUT:\n continue\n except pexpect.EOF:\n if tunnel.exitstatus:\n print (tunnel.exitstatus)\n print (tunnel.before)\n print (tunnel.after)\n raise RuntimeError(\"tunnel '%s' failed to start\"%(cmd))\n else:\n return tunnel.pid\n else:\n if failed:\n print(\"Password rejected, try again\")\n password=None\n if password is None:\n password = getpass(\"%s's password: \"%(server))\n tunnel.sendline(password)\n failed = True"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nlaunches a tunner with paramiko", "response": "def paramiko_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60):\n \"\"\"launch a tunner with paramiko in a subprocess. This should only be used\n when shell ssh is unavailable (e.g. Windows).\n\n This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`,\n as seen from `server`.\n\n If you are familiar with ssh tunnels, this creates the tunnel:\n\n ssh server -L localhost:lport:remoteip:rport\n\n keyfile and password may be specified, but ssh config is checked for defaults.\n\n\n Parameters\n ----------\n\n lport : int\n local port for connecting to the tunnel from this machine.\n rport : int\n port on the remote machine to connect to.\n server : str\n The ssh server to connect to. The full ssh server string will be parsed.\n user@server:port\n remoteip : str [Default: 127.0.0.1]\n The remote ip, specifying the destination of the tunnel.\n Default is localhost, which means that the tunnel would redirect\n localhost:lport on this machine to localhost:rport on the *server*.\n\n keyfile : str; path to public key file\n This specifies a key to be used in ssh login, default None.\n Regular default ssh keys will be used without specifying this argument.\n password : str;\n Your ssh password to the ssh server. Note that if this is left None,\n you will be prompted for it if passwordless key based login is unavailable.\n timeout : int [default: 60]\n The time (in seconds) after which no activity will result in the tunnel\n closing. This prevents orphaned tunnels from running forever.\n\n \"\"\"\n if paramiko is None:\n raise ImportError(\"Paramiko not available\")\n\n if password is None:\n if not _try_passwordless_paramiko(server, keyfile):\n password = getpass(\"%s's password: \"%(server))\n\n p = Process(target=_paramiko_tunnel,\n args=(lport, rport, server, remoteip),\n kwargs=dict(keyfile=keyfile, password=password))\n p.daemon=False\n p.start()\n atexit.register(_shutdown_process, p)\n return p"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _paramiko_tunnel(lport, rport, server, remoteip, keyfile=None, password=None):\n username, server, port = _split_server(server)\n client = paramiko.SSHClient()\n client.load_system_host_keys()\n client.set_missing_host_key_policy(paramiko.WarningPolicy())\n\n try:\n client.connect(server, port, username=username, key_filename=keyfile,\n look_for_keys=True, password=password)\n# except paramiko.AuthenticationException:\n# if password is None:\n# password = getpass(\"%s@%s's password: \"%(username, server))\n# client.connect(server, port, username=username, password=password)\n# else:\n# raise\n except Exception as e:\n print ('*** Failed to connect to %s:%d: %r' % (server, port, e))\n sys.exit(1)\n\n # print ('Now forwarding port %d to %s:%d ...' % (lport, server, rport))\n\n try:\n forward_tunnel(lport, remoteip, rport, client.get_transport())\n except KeyboardInterrupt:\n print ('SIGINT: Port forwarding stopped cleanly')\n sys.exit(0)\n except Exception as e:\n print (\"Port forwarding stopped uncleanly: %s\"%e)\n sys.exit(255)", "response": "Function for actually starting a paramiko tunnel."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalling the method f first to sync state prior to calling the method.", "response": "def spin_first(f, self, *args, **kwargs):\n \"\"\"Call spin() to sync state prior to calling the method.\"\"\"\n self.spin()\n return f(self, *args, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _update_engines(self, engines):\n for k,v in engines.iteritems():\n eid = int(k)\n self._engines[eid] = v\n self._ids.append(eid)\n self._ids = sorted(self._ids)\n if sorted(self._engines.keys()) != range(len(self._engines)) and \\\n self._task_scheme == 'pure' and self._task_socket:\n self._stop_scheduling_tasks()", "response": "Update our engines dict and _ids from a dict of the form uuid."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _stop_scheduling_tasks(self):\n self._task_socket.close()\n self._task_socket = None\n msg = \"An engine has been unregistered, and we are using pure \" +\\\n \"ZMQ task scheduling. Task farming will be disabled.\"\n if self.outstanding:\n msg += \" If you were running tasks when this happened, \" +\\\n \"some `outstanding` msg_ids may never resolve.\"\n warnings.warn(msg, RuntimeWarning)", "response": "Stop scheduling tasks because an engine has been unregistered from a pure ZMQ scheduler."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nturns valid target IDs or all into two lists of int_ids uuids.", "response": "def _build_targets(self, targets):\n \"\"\"Turn valid target IDs or 'all' into two lists:\n (int_ids, uuids).\n \"\"\"\n if not self._ids:\n # flush notification socket if no engines yet, just in case\n if not self.ids:\n raise error.NoEnginesRegistered(\"Can't build targets without any engines\")\n\n if targets is None:\n targets = self._ids\n elif isinstance(targets, basestring):\n if targets.lower() == 'all':\n targets = self._ids\n else:\n raise TypeError(\"%r not valid str target, must be 'all'\"%(targets))\n elif isinstance(targets, int):\n if targets < 0:\n targets = self.ids[targets]\n if targets not in self._ids:\n raise IndexError(\"No such engine: %i\"%targets)\n targets = [targets]\n\n if isinstance(targets, slice):\n indices = range(len(self._ids))[targets]\n ids = self.ids\n targets = [ ids[i] for i in indices ]\n\n if not isinstance(targets, (tuple, list, xrange)):\n raise TypeError(\"targets by int/slice/collection of ints only, not %s\"%(type(targets)))\n\n return [cast_bytes(self._engines[t]) for t in targets], list(targets)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _connect(self, sshserver, ssh_kwargs, timeout):\n\n # Maybe allow reconnecting?\n if self._connected:\n return\n self._connected=True\n\n def connect_socket(s, url):\n url = util.disambiguate_url(url, self._config['location'])\n if self._ssh:\n return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)\n else:\n return s.connect(url)\n\n self.session.send(self._query_socket, 'connection_request')\n # use Poller because zmq.select has wrong units in pyzmq 2.1.7\n poller = zmq.Poller()\n poller.register(self._query_socket, zmq.POLLIN)\n # poll expects milliseconds, timeout is seconds\n evts = poller.poll(timeout*1000)\n if not evts:\n raise error.TimeoutError(\"Hub connection request timed out\")\n idents,msg = self.session.recv(self._query_socket,mode=0)\n if self.debug:\n pprint(msg)\n msg = Message(msg)\n content = msg.content\n self._config['registration'] = dict(content)\n if content.status == 'ok':\n ident = self.session.bsession\n if content.mux:\n self._mux_socket = self._context.socket(zmq.DEALER)\n self._mux_socket.setsockopt(zmq.IDENTITY, ident)\n connect_socket(self._mux_socket, content.mux)\n if content.task:\n self._task_scheme, task_addr = content.task\n self._task_socket = self._context.socket(zmq.DEALER)\n self._task_socket.setsockopt(zmq.IDENTITY, ident)\n connect_socket(self._task_socket, task_addr)\n if content.notification:\n self._notification_socket = self._context.socket(zmq.SUB)\n connect_socket(self._notification_socket, content.notification)\n self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')\n # if content.query:\n # self._query_socket = self._context.socket(zmq.DEALER)\n # self._query_socket.setsockopt(zmq.IDENTITY, self.session.bsession)\n # connect_socket(self._query_socket, content.query)\n if content.control:\n self._control_socket = self._context.socket(zmq.DEALER)\n self._control_socket.setsockopt(zmq.IDENTITY, ident)\n connect_socket(self._control_socket, content.control)\n if content.iopub:\n self._iopub_socket = self._context.socket(zmq.SUB)\n self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')\n self._iopub_socket.setsockopt(zmq.IDENTITY, ident)\n connect_socket(self._iopub_socket, content.iopub)\n self._update_engines(dict(content.engines))\n else:\n self._connected = False\n raise Exception(\"Failed to connect!\")", "response": "setup all our socket connections to the cluster. This is called from the _connect method."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nunwraps exception and remap engine_id to int.", "response": "def _unwrap_exception(self, content):\n \"\"\"unwrap exception, and remap engine_id to int.\"\"\"\n e = error.unwrap_exception(content)\n # print e.traceback\n if e.engine_info:\n e_uuid = e.engine_info['engine_uuid']\n eid = self._engines[e_uuid]\n e.engine_info['engine_id'] = eid\n return e"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nregistering a new engine and update our connection info.", "response": "def _register_engine(self, msg):\n \"\"\"Register a new engine, and update our connection info.\"\"\"\n content = msg['content']\n eid = content['id']\n d = {eid : content['queue']}\n self._update_engines(d)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _unregister_engine(self, msg):\n content = msg['content']\n eid = int(content['id'])\n if eid in self._ids:\n self._ids.remove(eid)\n uuid = self._engines.pop(eid)\n\n self._handle_stranded_msgs(eid, uuid)\n\n if self._task_socket and self._task_scheme == 'pure':\n self._stop_scheduling_tasks()", "response": "Unregister an engine that has died."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nhandles stranded messages from the engine.", "response": "def _handle_stranded_msgs(self, eid, uuid):\n \"\"\"Handle messages known to be on an engine when the engine unregisters.\n\n It is possible that this will fire prematurely - that is, an engine will\n go down after completing a result, and the client will be notified\n of the unregistration and later receive the successful result.\n \"\"\"\n\n outstanding = self._outstanding_dict[uuid]\n\n for msg_id in list(outstanding):\n if msg_id in self.results:\n # we already\n continue\n try:\n raise error.EngineError(\"Engine %r died while running task %r\"%(eid, msg_id))\n except:\n content = error.wrap_exception()\n # build a fake message:\n parent = {}\n header = {}\n parent['msg_id'] = msg_id\n header['engine'] = uuid\n header['date'] = datetime.now()\n msg = dict(parent_header=parent, header=header, content=content)\n self._handle_apply_reply(msg)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _handle_execute_reply(self, msg):\n\n parent = msg['parent_header']\n msg_id = parent['msg_id']\n if msg_id not in self.outstanding:\n if msg_id in self.history:\n print (\"got stale result: %s\"%msg_id)\n else:\n print (\"got unknown result: %s\"%msg_id)\n else:\n self.outstanding.remove(msg_id)\n\n content = msg['content']\n header = msg['header']\n\n # construct metadata:\n md = self.metadata[msg_id]\n md.update(self._extract_metadata(header, parent, content))\n # is this redundant?\n self.metadata[msg_id] = md\n \n e_outstanding = self._outstanding_dict[md['engine_uuid']]\n if msg_id in e_outstanding:\n e_outstanding.remove(msg_id)\n\n # construct result:\n if content['status'] == 'ok':\n self.results[msg_id] = ExecuteReply(msg_id, content, md)\n elif content['status'] == 'aborted':\n self.results[msg_id] = error.TaskAborted(msg_id)\n elif content['status'] == 'resubmitted':\n # TODO: handle resubmission\n pass\n else:\n self.results[msg_id] = self._unwrap_exception(content)", "response": "Handle an execute reply message."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nflushes notifications of engine registrations waiting in ZMQ queue.", "response": "def _flush_notifications(self):\n \"\"\"Flush notifications of engine registrations waiting\n in ZMQ queue.\"\"\"\n idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)\n while msg is not None:\n if self.debug:\n pprint(msg)\n msg_type = msg['header']['msg_type']\n handler = self._notification_handlers.get(msg_type, None)\n if handler is None:\n raise Exception(\"Unhandled message type: %s\"%msg.msg_type)\n else:\n handler(msg)\n idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _flush_results(self, sock):\n idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)\n while msg is not None:\n if self.debug:\n pprint(msg)\n msg_type = msg['header']['msg_type']\n handler = self._queue_handlers.get(msg_type, None)\n if handler is None:\n raise Exception(\"Unhandled message type: %s\"%msg.msg_type)\n else:\n handler(msg)\n idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)", "response": "Flush task or queue results waiting in ZMQ queue."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _flush_control(self, sock):\n if self._ignored_control_replies <= 0:\n return\n idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)\n while msg is not None:\n self._ignored_control_replies -= 1\n if self.debug:\n pprint(msg)\n idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)", "response": "Flushes all messages from the control channel waiting\n Currently : ignore them."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nflushes ignored control replies", "response": "def _flush_ignored_control(self):\n \"\"\"flush ignored control replies\"\"\"\n while self._ignored_control_replies > 0:\n self.session.recv(self._control_socket)\n self._ignored_control_replies -= 1"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _flush_iopub(self, sock):\n idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)\n while msg is not None:\n if self.debug:\n pprint(msg)\n parent = msg['parent_header']\n # ignore IOPub messages with no parent.\n # Caused by print statements or warnings from before the first execution.\n if not parent:\n continue\n msg_id = parent['msg_id']\n content = msg['content']\n header = msg['header']\n msg_type = msg['header']['msg_type']\n\n # init metadata:\n md = self.metadata[msg_id]\n\n if msg_type == 'stream':\n name = content['name']\n s = md[name] or ''\n md[name] = s + content['data']\n elif msg_type == 'pyerr':\n md.update({'pyerr' : self._unwrap_exception(content)})\n elif msg_type == 'pyin':\n md.update({'pyin' : content['code']})\n elif msg_type == 'display_data':\n md['outputs'].append(content)\n elif msg_type == 'pyout':\n md['pyout'] = content\n elif msg_type == 'status':\n # idle message comes after all outputs\n if content['execution_state'] == 'idle':\n md['outputs_ready'] = True\n else:\n # unhandled msg_type (status, etc.)\n pass\n\n # reduntant?\n self.metadata[msg_id] = md\n\n idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)", "response": "Flushes all messages from the iopub channel waiting\n in the ZMQ queue."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a DirectView and register it with IPython magics .", "response": "def activate(self, targets='all', suffix=''):\n \"\"\"Create a DirectView and register it with IPython magics\n \n Defines the magics `%px, %autopx, %pxresult, %%px`\n \n Parameters\n ----------\n \n targets: int, list of ints, or 'all'\n The engines on which the view's magics will run\n suffix: str [default: '']\n The suffix, if any, for the magics. This allows you to have\n multiple views associated with parallel magics at the same time.\n \n e.g. ``rc.activate(targets=0, suffix='0')`` will give you\n the magics ``%px0``, ``%pxresult0``, etc. for running magics just\n on engine 0.\n \"\"\"\n view = self.direct_view(targets)\n view.block = True\n view.activate(suffix)\n return view"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntargets func for use in spin_thread", "response": "def _spin_every(self, interval=1):\n \"\"\"target func for use in spin_thread\"\"\"\n while True:\n if self._stop_spinning.is_set():\n return\n time.sleep(interval)\n self.spin()"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef spin_thread(self, interval=1):\n if self._spin_thread is not None:\n self.stop_spin_thread()\n self._stop_spinning.clear()\n self._spin_thread = Thread(target=self._spin_every, args=(interval,))\n self._spin_thread.daemon = True\n self._spin_thread.start()", "response": "call Client. spin in a background thread on some regular interval"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef stop_spin_thread(self):\n if self._spin_thread is not None:\n self._stop_spinning.set()\n self._spin_thread.join()\n self._spin_thread = None", "response": "stop background spin_thread if any"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nflushing any pending messages and any results that are waiting in the ZMQ queue.", "response": "def spin(self):\n \"\"\"Flush any registration notifications and execution results\n waiting in the ZMQ queue.\n \"\"\"\n if self._notification_socket:\n self._flush_notifications()\n if self._iopub_socket:\n self._flush_iopub(self._iopub_socket)\n if self._mux_socket:\n self._flush_results(self._mux_socket)\n if self._task_socket:\n self._flush_results(self._task_socket)\n if self._control_socket:\n self._flush_control(self._control_socket)\n if self._query_socket:\n self._flush_ignored_hub_replies()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwait on one or more messages from the queue for up to timeout seconds.", "response": "def wait(self, jobs=None, timeout=-1):\n \"\"\"waits on one or more `jobs`, for up to `timeout` seconds.\n\n Parameters\n ----------\n\n jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects\n ints are indices to self.history\n strs are msg_ids\n default: wait on all outstanding messages\n timeout : float\n a time in seconds, after which to give up.\n default is -1, which means no timeout\n\n Returns\n -------\n\n True : when all msg_ids are done\n False : timeout reached, some msg_ids still outstanding\n \"\"\"\n tic = time.time()\n if jobs is None:\n theids = self.outstanding\n else:\n if isinstance(jobs, (int, basestring, AsyncResult)):\n jobs = [jobs]\n theids = set()\n for job in jobs:\n if isinstance(job, int):\n # index access\n job = self.history[job]\n elif isinstance(job, AsyncResult):\n map(theids.add, job.msg_ids)\n continue\n theids.add(job)\n if not theids.intersection(self.outstanding):\n return True\n self.spin()\n while theids.intersection(self.outstanding):\n if timeout >= 0 and ( time.time()-tic ) > timeout:\n break\n time.sleep(1e-3)\n self.spin()\n return len(theids.intersection(self.outstanding)) == 0"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nclear the namespace in target(s ).", "response": "def clear(self, targets=None, block=None):\n \"\"\"Clear the namespace in target(s).\"\"\"\n block = self.block if block is None else block\n targets = self._build_targets(targets)[0]\n for t in targets:\n self.session.send(self._control_socket, 'clear_request', content={}, ident=t)\n error = False\n if block:\n self._flush_ignored_control()\n for i in range(len(targets)):\n idents,msg = self.session.recv(self._control_socket,0)\n if self.debug:\n pprint(msg)\n if msg['content']['status'] != 'ok':\n error = self._unwrap_exception(msg['content'])\n else:\n self._ignored_control_replies += len(targets)\n if error:\n raise error"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef abort(self, jobs=None, targets=None, block=None):\n block = self.block if block is None else block\n jobs = jobs if jobs is not None else list(self.outstanding)\n targets = self._build_targets(targets)[0]\n \n msg_ids = []\n if isinstance(jobs, (basestring,AsyncResult)):\n jobs = [jobs]\n bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)\n if bad_ids:\n raise TypeError(\"Invalid msg_id type %r, expected str or AsyncResult\"%bad_ids[0])\n for j in jobs:\n if isinstance(j, AsyncResult):\n msg_ids.extend(j.msg_ids)\n else:\n msg_ids.append(j)\n content = dict(msg_ids=msg_ids)\n for t in targets:\n self.session.send(self._control_socket, 'abort_request',\n content=content, ident=t)\n error = False\n if block:\n self._flush_ignored_control()\n for i in range(len(targets)):\n idents,msg = self.session.recv(self._control_socket,0)\n if self.debug:\n pprint(msg)\n if msg['content']['status'] != 'ok':\n error = self._unwrap_exception(msg['content'])\n else:\n self._ignored_control_replies += len(targets)\n if error:\n raise error", "response": "Aborts specific jobs from the execution queues of target(s."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef shutdown(self, targets='all', restart=False, hub=False, block=None):\n \n if restart:\n raise NotImplementedError(\"Engine restart is not yet implemented\")\n \n block = self.block if block is None else block\n if hub:\n targets = 'all'\n targets = self._build_targets(targets)[0]\n for t in targets:\n self.session.send(self._control_socket, 'shutdown_request',\n content={'restart':restart},ident=t)\n error = False\n if block or hub:\n self._flush_ignored_control()\n for i in range(len(targets)):\n idents,msg = self.session.recv(self._control_socket, 0)\n if self.debug:\n pprint(msg)\n if msg['content']['status'] != 'ok':\n error = self._unwrap_exception(msg['content'])\n else:\n self._ignored_control_replies += len(targets)\n\n if hub:\n time.sleep(0.25)\n self.session.send(self._query_socket, 'shutdown_request')\n idents,msg = self.session.recv(self._query_socket, 0)\n if self.debug:\n pprint(msg)\n if msg['content']['status'] != 'ok':\n error = self._unwrap_exception(msg['content'])\n\n if error:\n raise error", "response": "Terminates one or more engines."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, track=False,\n ident=None):\n \"\"\"construct and send an apply message via a socket.\n\n This is the principal method with which all engine execution is performed by views.\n \"\"\"\n\n if self._closed:\n raise RuntimeError(\"Client cannot be used after its sockets have been closed\")\n \n # defaults:\n args = args if args is not None else []\n kwargs = kwargs if kwargs is not None else {}\n subheader = subheader if subheader is not None else {}\n\n # validate arguments\n if not callable(f) and not isinstance(f, Reference):\n raise TypeError(\"f must be callable, not %s\"%type(f))\n if not isinstance(args, (tuple, list)):\n raise TypeError(\"args must be tuple or list, not %s\"%type(args))\n if not isinstance(kwargs, dict):\n raise TypeError(\"kwargs must be dict, not %s\"%type(kwargs))\n if not isinstance(subheader, dict):\n raise TypeError(\"subheader must be dict, not %s\"%type(subheader))\n\n bufs = util.pack_apply_message(f,args,kwargs)\n\n msg = self.session.send(socket, \"apply_request\", buffers=bufs, ident=ident,\n subheader=subheader, track=track)\n\n msg_id = msg['header']['msg_id']\n self.outstanding.add(msg_id)\n if ident:\n # possibly routed to a specific engine\n if isinstance(ident, list):\n ident = ident[-1]\n if ident in self._engines.values():\n # save for later, in case of engine death\n self._outstanding_dict[ident].add(msg_id)\n self.history.append(msg_id)\n self.metadata[msg_id]['submitted'] = datetime.now()\n\n return msg", "response": "construct and send an apply request via a socket."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef send_execute_request(self, socket, code, silent=True, subheader=None, ident=None):\n\n if self._closed:\n raise RuntimeError(\"Client cannot be used after its sockets have been closed\")\n \n # defaults:\n subheader = subheader if subheader is not None else {}\n\n # validate arguments\n if not isinstance(code, basestring):\n raise TypeError(\"code must be text, not %s\" % type(code))\n if not isinstance(subheader, dict):\n raise TypeError(\"subheader must be dict, not %s\" % type(subheader))\n \n content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={})\n\n\n msg = self.session.send(socket, \"execute_request\", content=content, ident=ident,\n subheader=subheader)\n\n msg_id = msg['header']['msg_id']\n self.outstanding.add(msg_id)\n if ident:\n # possibly routed to a specific engine\n if isinstance(ident, list):\n ident = ident[-1]\n if ident in self._engines.values():\n # save for later, in case of engine death\n self._outstanding_dict[ident].add(msg_id)\n self.history.append(msg_id)\n self.metadata[msg_id]['submitted'] = datetime.now()\n\n return msg", "response": "construct and send an execute request via a socket."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconstructing a DirectView object.", "response": "def load_balanced_view(self, targets=None):\n \"\"\"construct a DirectView object.\n\n If no arguments are specified, create a LoadBalancedView\n using all engines.\n\n Parameters\n ----------\n\n targets: list,slice,int,etc. [default: use all engines]\n The subset of engines across which to load-balance\n \"\"\"\n if targets == 'all':\n targets = None\n if targets is not None:\n targets = self._build_targets(targets)[1]\n return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef direct_view(self, targets='all'):\n single = isinstance(targets, int)\n # allow 'all' to be lazily evaluated at each execution\n if targets != 'all':\n targets = self._build_targets(targets)[1]\n if single:\n targets = targets[0]\n return DirectView(client=self, socket=self._mux_socket, targets=targets)", "response": "construct a DirectView object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_result(self, indices_or_msg_ids=None, block=None):\n block = self.block if block is None else block\n if indices_or_msg_ids is None:\n indices_or_msg_ids = -1\n\n if not isinstance(indices_or_msg_ids, (list,tuple)):\n indices_or_msg_ids = [indices_or_msg_ids]\n\n theids = []\n for id in indices_or_msg_ids:\n if isinstance(id, int):\n id = self.history[id]\n if not isinstance(id, basestring):\n raise TypeError(\"indices must be str or int, not %r\"%id)\n theids.append(id)\n\n local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids)\n remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids)\n\n if remote_ids:\n ar = AsyncHubResult(self, msg_ids=theids)\n else:\n ar = AsyncResult(self, msg_ids=theids)\n\n if block:\n ar.wait()\n\n return ar", "response": "Retrieve a result by message id or history index."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nresubmits one or more in - flight tasks.", "response": "def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None):\n \"\"\"Resubmit one or more tasks.\n\n in-flight tasks may not be resubmitted.\n\n Parameters\n ----------\n\n indices_or_msg_ids : integer history index, str msg_id, or list of either\n The indices or msg_ids of indices to be retrieved\n\n block : bool\n Whether to wait for the result to be done\n\n Returns\n -------\n\n AsyncHubResult\n A subclass of AsyncResult that retrieves results from the Hub\n\n \"\"\"\n block = self.block if block is None else block\n if indices_or_msg_ids is None:\n indices_or_msg_ids = -1\n\n if not isinstance(indices_or_msg_ids, (list,tuple)):\n indices_or_msg_ids = [indices_or_msg_ids]\n\n theids = []\n for id in indices_or_msg_ids:\n if isinstance(id, int):\n id = self.history[id]\n if not isinstance(id, basestring):\n raise TypeError(\"indices must be str or int, not %r\"%id)\n theids.append(id)\n\n content = dict(msg_ids = theids)\n\n self.session.send(self._query_socket, 'resubmit_request', content)\n\n zmq.select([self._query_socket], [], [])\n idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)\n if self.debug:\n pprint(msg)\n content = msg['content']\n if content['status'] != 'ok':\n raise self._unwrap_exception(content)\n mapping = content['resubmitted']\n new_ids = [ mapping[msg_id] for msg_id in theids ]\n\n ar = AsyncHubResult(self, msg_ids=new_ids)\n\n if block:\n ar.wait()\n\n return ar"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef result_status(self, msg_ids, status_only=True):\n if not isinstance(msg_ids, (list,tuple)):\n msg_ids = [msg_ids]\n\n theids = []\n for msg_id in msg_ids:\n if isinstance(msg_id, int):\n msg_id = self.history[msg_id]\n if not isinstance(msg_id, basestring):\n raise TypeError(\"msg_ids must be str, not %r\"%msg_id)\n theids.append(msg_id)\n\n completed = []\n local_results = {}\n\n # comment this block out to temporarily disable local shortcut:\n for msg_id in theids:\n if msg_id in self.results:\n completed.append(msg_id)\n local_results[msg_id] = self.results[msg_id]\n theids.remove(msg_id)\n\n if theids: # some not locally cached\n content = dict(msg_ids=theids, status_only=status_only)\n msg = self.session.send(self._query_socket, \"result_request\", content=content)\n zmq.select([self._query_socket], [], [])\n idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)\n if self.debug:\n pprint(msg)\n content = msg['content']\n if content['status'] != 'ok':\n raise self._unwrap_exception(content)\n buffers = msg['buffers']\n else:\n content = dict(completed=[],pending=[])\n\n content['completed'].extend(completed)\n\n if status_only:\n return content\n\n failures = []\n # load cached results into result:\n content.update(local_results)\n\n # update cache with results:\n for msg_id in sorted(theids):\n if msg_id in content['completed']:\n rec = content[msg_id]\n parent = rec['header']\n header = rec['result_header']\n rcontent = rec['result_content']\n iodict = rec['io']\n if isinstance(rcontent, str):\n rcontent = self.session.unpack(rcontent)\n\n md = self.metadata[msg_id]\n md.update(self._extract_metadata(header, parent, rcontent))\n if rec.get('received'):\n md['received'] = rec['received']\n md.update(iodict)\n \n if rcontent['status'] == 'ok':\n if header['msg_type'] == 'apply_reply':\n res,buffers = util.unserialize_object(buffers)\n elif header['msg_type'] == 'execute_reply':\n res = ExecuteReply(msg_id, rcontent, md)\n else:\n raise KeyError(\"unhandled msg type: %r\" % header[msg_type])\n else:\n res = self._unwrap_exception(rcontent)\n failures.append(res)\n\n self.results[msg_id] = res\n content[msg_id] = res\n\n if len(theids) == 1 and failures:\n raise failures[0]\n\n error.collect_exceptions(failures, \"result_status\")\n return content", "response": "Check on the status of the results of the apply request with the given list of message ids."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfetch the status of the engines queues.", "response": "def queue_status(self, targets='all', verbose=False):\n \"\"\"Fetch the status of engine queues.\n\n Parameters\n ----------\n\n targets : int/str/list of ints/strs\n the engines whose states are to be queried.\n default : all\n verbose : bool\n Whether to return lengths only, or lists of ids for each element\n \"\"\"\n if targets == 'all':\n # allow 'all' to be evaluated on the engine\n engine_ids = None\n else:\n engine_ids = self._build_targets(targets)[1]\n content = dict(targets=engine_ids, verbose=verbose)\n self.session.send(self._query_socket, \"queue_request\", content=content)\n idents,msg = self.session.recv(self._query_socket, 0)\n if self.debug:\n pprint(msg)\n content = msg['content']\n status = content.pop('status')\n if status != 'ok':\n raise self._unwrap_exception(content)\n content = rekey(content)\n if isinstance(targets, int):\n return content[targets]\n else:\n return content"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntells the Hub to forget results.", "response": "def purge_results(self, jobs=[], targets=[]):\n \"\"\"Tell the Hub to forget results.\n\n Individual results can be purged by msg_id, or the entire\n history of specific targets can be purged.\n\n Use `purge_results('all')` to scrub everything from the Hub's db.\n\n Parameters\n ----------\n\n jobs : str or list of str or AsyncResult objects\n the msg_ids whose results should be forgotten.\n targets : int/str/list of ints/strs\n The targets, by int_id, whose entire history is to be purged.\n\n default : None\n \"\"\"\n if not targets and not jobs:\n raise ValueError(\"Must specify at least one of `targets` and `jobs`\")\n if targets:\n targets = self._build_targets(targets)[1]\n\n # construct msg_ids from jobs\n if jobs == 'all':\n msg_ids = jobs\n else:\n msg_ids = []\n if isinstance(jobs, (basestring,AsyncResult)):\n jobs = [jobs]\n bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)\n if bad_ids:\n raise TypeError(\"Invalid msg_id type %r, expected str or AsyncResult\"%bad_ids[0])\n for j in jobs:\n if isinstance(j, AsyncResult):\n msg_ids.extend(j.msg_ids)\n else:\n msg_ids.append(j)\n\n content = dict(engine_ids=targets, msg_ids=msg_ids)\n self.session.send(self._query_socket, \"purge_request\", content=content)\n idents, msg = self.session.recv(self._query_socket, 0)\n if self.debug:\n pprint(msg)\n content = msg['content']\n if content['status'] != 'ok':\n raise self._unwrap_exception(content)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget the Hub s history.", "response": "def hub_history(self):\n \"\"\"Get the Hub's history\n\n Just like the Client, the Hub has a history, which is a list of msg_ids.\n This will contain the history of all clients, and, depending on configuration,\n may contain history across multiple cluster sessions.\n\n Any msg_id returned here is a valid argument to `get_result`.\n\n Returns\n -------\n\n msg_ids : list of strs\n list of all msg_ids, ordered by task submission time.\n \"\"\"\n\n self.session.send(self._query_socket, \"history_request\", content={})\n idents, msg = self.session.recv(self._query_socket, 0)\n\n if self.debug:\n pprint(msg)\n content = msg['content']\n if content['status'] != 'ok':\n raise self._unwrap_exception(content)\n else:\n return content['history']"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nquery the Hub s TaskRecord database for the specified query.", "response": "def db_query(self, query, keys=None):\n \"\"\"Query the Hub's TaskRecord database\n\n This will return a list of task record dicts that match `query`\n\n Parameters\n ----------\n\n query : mongodb query dict\n The search dict. See mongodb query docs for details.\n keys : list of strs [optional]\n The subset of keys to be returned. The default is to fetch everything but buffers.\n 'msg_id' will *always* be included.\n \"\"\"\n if isinstance(keys, basestring):\n keys = [keys]\n content = dict(query=query, keys=keys)\n self.session.send(self._query_socket, \"db_request\", content=content)\n idents, msg = self.session.recv(self._query_socket, 0)\n if self.debug:\n pprint(msg)\n content = msg['content']\n if content['status'] != 'ok':\n raise self._unwrap_exception(content)\n\n records = content['records']\n\n buffer_lens = content['buffer_lens']\n result_buffer_lens = content['result_buffer_lens']\n buffers = msg['buffers']\n has_bufs = buffer_lens is not None\n has_rbufs = result_buffer_lens is not None\n for i,rec in enumerate(records):\n # relink buffers\n if has_bufs:\n blen = buffer_lens[i]\n rec['buffers'], buffers = buffers[:blen],buffers[blen:]\n if has_rbufs:\n blen = result_buffer_lens[i]\n rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]\n\n return records"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a set of opcodes by the names in names.", "response": "def _opcode_set(*names):\n \"\"\"Return a set of opcodes by the names in `names`.\"\"\"\n s = set()\n for name in names:\n try:\n s.add(_opcode(name))\n except KeyError:\n pass\n return s"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a ByteParser on demand.", "response": "def _get_byte_parser(self):\n \"\"\"Create a ByteParser on demand.\"\"\"\n if not self._byte_parser:\n self._byte_parser = \\\n ByteParser(text=self.text, filename=self.filename)\n return self._byte_parser"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef lines_matching(self, *regexes):\n regex_c = re.compile(join_regex(regexes))\n matches = set()\n for i, ltext in enumerate(self.lines):\n if regex_c.search(ltext):\n matches.add(i+1)\n return matches", "response": "Find the lines matching one of a list of regexes."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _raw_parse(self):\n # Find lines which match an exclusion pattern.\n if self.exclude:\n self.excluded = self.lines_matching(self.exclude)\n\n # Tokenize, to find excluded suites, to find docstrings, and to find\n # multi-line statements.\n indent = 0\n exclude_indent = 0\n excluding = False\n prev_toktype = token.INDENT\n first_line = None\n empty = True\n\n tokgen = generate_tokens(self.text)\n for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen:\n if self.show_tokens: # pragma: not covered\n print(\"%10s %5s %-20r %r\" % (\n tokenize.tok_name.get(toktype, toktype),\n nice_pair((slineno, elineno)), ttext, ltext\n ))\n if toktype == token.INDENT:\n indent += 1\n elif toktype == token.DEDENT:\n indent -= 1\n elif toktype == token.NAME and ttext == 'class':\n # Class definitions look like branches in the byte code, so\n # we need to exclude them. The simplest way is to note the\n # lines with the 'class' keyword.\n self.classdefs.add(slineno)\n elif toktype == token.OP and ttext == ':':\n if not excluding and elineno in self.excluded:\n # Start excluding a suite. We trigger off of the colon\n # token so that the #pragma comment will be recognized on\n # the same line as the colon.\n exclude_indent = indent\n excluding = True\n elif toktype == token.STRING and prev_toktype == token.INDENT:\n # Strings that are first on an indented line are docstrings.\n # (a trick from trace.py in the stdlib.) This works for\n # 99.9999% of cases. For the rest (!) see:\n # http://stackoverflow.com/questions/1769332/x/1769794#1769794\n self.docstrings.update(range(slineno, elineno+1))\n elif toktype == token.NEWLINE:\n if first_line is not None and elineno != first_line:\n # We're at the end of a line, and we've ended on a\n # different line than the first line of the statement,\n # so record a multi-line range.\n rng = (first_line, elineno)\n for l in range(first_line, elineno+1):\n self.multiline[l] = rng\n first_line = None\n\n if ttext.strip() and toktype != tokenize.COMMENT:\n # A non-whitespace token.\n empty = False\n if first_line is None:\n # The token is not whitespace, and is the first in a\n # statement.\n first_line = slineno\n # Check whether to end an excluded suite.\n if excluding and indent <= exclude_indent:\n excluding = False\n if excluding:\n self.excluded.add(elineno)\n\n prev_toktype = toktype\n\n # Find the starts of the executable statements.\n if not empty:\n self.statement_starts.update(self.byte_parser._find_statements())", "response": "Parse the source to find the interesting facts about its lines."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the first line number of the statement including line.", "response": "def first_line(self, line):\n \"\"\"Return the first line number of the statement including `line`.\"\"\"\n rng = self.multiline.get(line)\n if rng:\n first_line = rng[0]\n else:\n first_line = line\n return first_line"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef first_lines(self, lines, *ignores):\n ignore = set()\n for ign in ignores:\n ignore.update(ign)\n lset = set()\n for l in lines:\n if l in ignore:\n continue\n new_l = self.first_line(l)\n if new_l not in ignore:\n lset.add(new_l)\n return lset", "response": "Map the line numbers in lines to the correct first line of the the\n statement."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef parse_source(self):\n try:\n self._raw_parse()\n except (tokenize.TokenError, IndentationError):\n _, tokerr, _ = sys.exc_info()\n msg, lineno = tokerr.args\n raise NotPython(\n \"Couldn't parse '%s' as Python source: '%s' at %s\" %\n (self.filename, msg, lineno)\n )\n\n excluded_lines = self.first_lines(self.excluded)\n lines = self.first_lines(\n self.statement_starts,\n excluded_lines,\n self.docstrings\n )\n\n return lines, excluded_lines", "response": "Parse source text to find executable lines excluded lines etc."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget information about the arcs available in the code.", "response": "def arcs(self):\n \"\"\"Get information about the arcs available in the code.\n\n Returns a sorted list of line number pairs. Line numbers have been\n normalized to the first line of multiline statements.\n\n \"\"\"\n all_arcs = []\n for l1, l2 in self.byte_parser._all_arcs():\n fl1 = self.first_line(l1)\n fl2 = self.first_line(l2)\n if fl1 != fl2:\n all_arcs.append((fl1, fl2))\n return sorted(all_arcs)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef exit_counts(self):\n excluded_lines = self.first_lines(self.excluded)\n exit_counts = {}\n for l1, l2 in self.arcs():\n if l1 < 0:\n # Don't ever report -1 as a line number\n continue\n if l1 in excluded_lines:\n # Don't report excluded lines as line numbers.\n continue\n if l2 in excluded_lines:\n # Arcs to excluded lines shouldn't count.\n continue\n if l1 not in exit_counts:\n exit_counts[l1] = 0\n exit_counts[l1] += 1\n\n # Class definitions have one extra exit, so remove one for each:\n for l in self.classdefs:\n # Ensure key is there: classdefs can include excluded lines.\n if l in exit_counts:\n exit_counts[l] -= 1\n\n return exit_counts", "response": "Get a mapping from line numbers to count of exits from that line."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\niterating over all the code objects nested within this one.", "response": "def child_parsers(self):\n \"\"\"Iterate over all the code objects nested within this one.\n\n The iteration includes `self` as its first value.\n\n \"\"\"\n children = CodeObjects(self.code)\n return [ByteParser(code=c, text=self.text) for c in children]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _bytes_lines(self):\n # Adapted from dis.py in the standard library.\n byte_increments = bytes_to_ints(self.code.co_lnotab[0::2])\n line_increments = bytes_to_ints(self.code.co_lnotab[1::2])\n\n last_line_num = None\n line_num = self.code.co_firstlineno\n byte_num = 0\n for byte_incr, line_incr in zip(byte_increments, line_increments):\n if byte_incr:\n if line_num != last_line_num:\n yield (byte_num, line_num)\n last_line_num = line_num\n byte_num += byte_incr\n line_num += line_incr\n if line_num != last_line_num:\n yield (byte_num, line_num)", "response": "Map byte offsets to line numbers in code."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfinding the statements in self. code.", "response": "def _find_statements(self):\n \"\"\"Find the statements in `self.code`.\n\n Produce a sequence of line numbers that start statements. Recurses\n into all code objects reachable from `self.code`.\n\n \"\"\"\n for bp in self.child_parsers():\n # Get all of the lineno information from this code.\n for _, l in bp._bytes_lines():\n yield l"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _block_stack_repr(self, block_stack):\n blocks = \", \".join(\n [\"(%s, %r)\" % (dis.opname[b[0]], b[1]) for b in block_stack]\n )\n return \"[\" + blocks + \"]\"", "response": "Get a string version of block_stack for debugging."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _split_into_chunks(self):\n # The list of chunks so far, and the one we're working on.\n chunks = []\n chunk = None\n\n # A dict mapping byte offsets of line starts to the line numbers.\n bytes_lines_map = dict(self._bytes_lines())\n\n # The block stack: loops and try blocks get pushed here for the\n # implicit jumps that can occur.\n # Each entry is a tuple: (block type, destination)\n block_stack = []\n\n # Some op codes are followed by branches that should be ignored. This\n # is a count of how many ignores are left.\n ignore_branch = 0\n\n # We have to handle the last two bytecodes specially.\n ult = penult = None\n\n # Get a set of all of the jump-to points.\n jump_to = set()\n bytecodes = list(ByteCodes(self.code.co_code))\n for bc in bytecodes:\n if bc.jump_to >= 0:\n jump_to.add(bc.jump_to)\n\n chunk_lineno = 0\n\n # Walk the byte codes building chunks.\n for bc in bytecodes:\n # Maybe have to start a new chunk\n start_new_chunk = False\n first_chunk = False\n if bc.offset in bytes_lines_map:\n # Start a new chunk for each source line number.\n start_new_chunk = True\n chunk_lineno = bytes_lines_map[bc.offset]\n first_chunk = True\n elif bc.offset in jump_to:\n # To make chunks have a single entrance, we have to make a new\n # chunk when we get to a place some bytecode jumps to.\n start_new_chunk = True\n elif bc.op in OPS_CHUNK_BEGIN:\n # Jumps deserve their own unnumbered chunk. This fixes\n # problems with jumps to jumps getting confused.\n start_new_chunk = True\n\n if not chunk or start_new_chunk:\n if chunk:\n chunk.exits.add(bc.offset)\n chunk = Chunk(bc.offset, chunk_lineno, first_chunk)\n chunks.append(chunk)\n\n # Look at the opcode\n if bc.jump_to >= 0 and bc.op not in OPS_NO_JUMP:\n if ignore_branch:\n # Someone earlier wanted us to ignore this branch.\n ignore_branch -= 1\n else:\n # The opcode has a jump, it's an exit for this chunk.\n chunk.exits.add(bc.jump_to)\n\n if bc.op in OPS_CODE_END:\n # The opcode can exit the code object.\n chunk.exits.add(-self.code.co_firstlineno)\n if bc.op in OPS_PUSH_BLOCK:\n # The opcode adds a block to the block_stack.\n block_stack.append((bc.op, bc.jump_to))\n if bc.op in OPS_POP_BLOCK:\n # The opcode pops a block from the block stack.\n block_stack.pop()\n if bc.op in OPS_CHUNK_END:\n # This opcode forces the end of the chunk.\n if bc.op == OP_BREAK_LOOP:\n # A break is implicit: jump where the top of the\n # block_stack points.\n chunk.exits.add(block_stack[-1][1])\n chunk = None\n if bc.op == OP_END_FINALLY:\n # For the finally clause we need to find the closest exception\n # block, and use its jump target as an exit.\n for block in reversed(block_stack):\n if block[0] in OPS_EXCEPT_BLOCKS:\n chunk.exits.add(block[1])\n break\n if bc.op == OP_COMPARE_OP and bc.arg == COMPARE_EXCEPTION:\n # This is an except clause. We want to overlook the next\n # branch, so that except's don't count as branches.\n ignore_branch += 1\n\n penult = ult\n ult = bc\n\n if chunks:\n # The last two bytecodes could be a dummy \"return None\" that\n # shouldn't be counted as real code. Every Python code object seems\n # to end with a return, and a \"return None\" is inserted if there\n # isn't an explicit return in the source.\n if ult and penult:\n if penult.op == OP_LOAD_CONST and ult.op == OP_RETURN_VALUE:\n if self.code.co_consts[penult.arg] is None:\n # This is \"return None\", but is it dummy? A real line\n # would be a last chunk all by itself.\n if chunks[-1].byte != penult.offset:\n ex = -self.code.co_firstlineno\n # Split the last chunk\n last_chunk = chunks[-1]\n last_chunk.exits.remove(ex)\n last_chunk.exits.add(penult.offset)\n chunk = Chunk(\n penult.offset, last_chunk.line, False\n )\n chunk.exits.add(ex)\n chunks.append(chunk)\n\n # Give all the chunks a length.\n chunks[-1].length = bc.next_offset - chunks[-1].byte # pylint: disable=W0631,C0301\n for i in range(len(chunks)-1):\n chunks[i].length = chunks[i+1].byte - chunks[i].byte\n\n #self.validate_chunks(chunks)\n return chunks", "response": "Split the code object into a list of Chunk objects."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nvalidates the rule that a single chunk has a single entrance.", "response": "def validate_chunks(self, chunks):\n \"\"\"Validate the rule that chunks have a single entrance.\"\"\"\n # starts is the entrances to the chunks\n starts = set([ch.byte for ch in chunks])\n for ch in chunks:\n assert all([(ex in starts or ex < 0) for ex in ch.exits])"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _arcs(self):\n chunks = self._split_into_chunks()\n\n # A map from byte offsets to chunks jumped into.\n byte_chunks = dict([(c.byte, c) for c in chunks])\n\n # There's always an entrance at the first chunk.\n yield (-1, byte_chunks[0].line)\n\n # Traverse from the first chunk in each line, and yield arcs where\n # the trace function will be invoked.\n for chunk in chunks:\n if not chunk.first:\n continue\n\n chunks_considered = set()\n chunks_to_consider = [chunk]\n while chunks_to_consider:\n # Get the chunk we're considering, and make sure we don't\n # consider it again\n this_chunk = chunks_to_consider.pop()\n chunks_considered.add(this_chunk)\n\n # For each exit, add the line number if the trace function\n # would be triggered, or add the chunk to those being\n # considered if not.\n for ex in this_chunk.exits:\n if ex < 0:\n yield (chunk.line, ex)\n else:\n next_chunk = byte_chunks[ex]\n if next_chunk in chunks_considered:\n continue\n\n # The trace function is invoked if visiting the first\n # bytecode in a line, or if the transition is a\n # backward jump.\n backward_jump = next_chunk.byte < this_chunk.byte\n if next_chunk.first or backward_jump:\n if next_chunk.line != chunk.line:\n yield (chunk.line, next_chunk.line)\n else:\n chunks_to_consider.append(next_chunk)", "response": "Find the executable arcs in the code object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of Chunk objects for this code and its children.", "response": "def _all_chunks(self):\n \"\"\"Returns a list of `Chunk` objects for this code and its children.\n\n See `_split_into_chunks` for details.\n\n \"\"\"\n chunks = []\n for bp in self.child_parsers():\n chunks.extend(bp._split_into_chunks())\n\n return chunks"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _all_arcs(self):\n arcs = set()\n for bp in self.child_parsers():\n arcs.update(bp._arcs())\n\n return arcs", "response": "Get the set of all arcs in this code object and its children."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds options to command line.", "response": "def options(self, parser, env):\n \"\"\"\n Add options to command line.\n \"\"\"\n super(Coverage, self).options(parser, env)\n parser.add_option(\"--cover-package\", action=\"append\",\n default=env.get('NOSE_COVER_PACKAGE'),\n metavar=\"PACKAGE\",\n dest=\"cover_packages\",\n help=\"Restrict coverage output to selected packages \"\n \"[NOSE_COVER_PACKAGE]\")\n parser.add_option(\"--cover-erase\", action=\"store_true\",\n default=env.get('NOSE_COVER_ERASE'),\n dest=\"cover_erase\",\n help=\"Erase previously collected coverage \"\n \"statistics before run\")\n parser.add_option(\"--cover-tests\", action=\"store_true\",\n dest=\"cover_tests\",\n default=env.get('NOSE_COVER_TESTS'),\n help=\"Include test modules in coverage report \"\n \"[NOSE_COVER_TESTS]\")\n parser.add_option(\"--cover-min-percentage\", action=\"store\",\n dest=\"cover_min_percentage\",\n default=env.get('NOSE_COVER_MIN_PERCENTAGE'),\n help=\"Minimum percentage of coverage for tests\"\n \"to pass [NOSE_COVER_MIN_PERCENTAGE]\")\n parser.add_option(\"--cover-inclusive\", action=\"store_true\",\n dest=\"cover_inclusive\",\n default=env.get('NOSE_COVER_INCLUSIVE'),\n help=\"Include all python files under working \"\n \"directory in coverage report. Useful for \"\n \"discovering holes in test coverage if not all \"\n \"files are imported by the test suite. \"\n \"[NOSE_COVER_INCLUSIVE]\")\n parser.add_option(\"--cover-html\", action=\"store_true\",\n default=env.get('NOSE_COVER_HTML'),\n dest='cover_html',\n help=\"Produce HTML coverage information\")\n parser.add_option('--cover-html-dir', action='store',\n default=env.get('NOSE_COVER_HTML_DIR', 'cover'),\n dest='cover_html_dir',\n metavar='DIR',\n help='Produce HTML coverage information in dir')\n parser.add_option(\"--cover-branches\", action=\"store_true\",\n default=env.get('NOSE_COVER_BRANCHES'),\n dest=\"cover_branches\",\n help=\"Include branch coverage in coverage report \"\n \"[NOSE_COVER_BRANCHES]\")\n parser.add_option(\"--cover-xml\", action=\"store_true\",\n default=env.get('NOSE_COVER_XML'),\n dest=\"cover_xml\",\n help=\"Produce XML coverage information\")\n parser.add_option(\"--cover-xml-file\", action=\"store\",\n default=env.get('NOSE_COVER_XML_FILE', 'coverage.xml'),\n dest=\"cover_xml_file\",\n metavar=\"FILE\",\n help=\"Produce XML coverage information in file\")"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef begin(self):\n log.debug(\"Coverage begin\")\n self.skipModules = sys.modules.keys()[:]\n if self.coverErase:\n log.debug(\"Clearing previously collected coverage statistics\")\n self.coverInstance.combine()\n self.coverInstance.erase()\n self.coverInstance.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]')\n self.coverInstance.load()\n self.coverInstance.start()", "response": "Begin recording coverage information."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef report(self, stream):\n log.debug(\"Coverage report\")\n self.coverInstance.stop()\n self.coverInstance.combine()\n self.coverInstance.save()\n modules = [module\n for name, module in sys.modules.items()\n if self.wantModuleCoverage(name, module)]\n log.debug(\"Coverage report will cover modules: %s\", modules)\n self.coverInstance.report(modules, file=stream)\n if self.coverHtmlDir:\n log.debug(\"Generating HTML coverage report\")\n self.coverInstance.html_report(modules, self.coverHtmlDir)\n if self.coverXmlFile:\n log.debug(\"Generating XML coverage report\")\n self.coverInstance.xml_report(modules, self.coverXmlFile)\n\n # make sure we have minimum required coverage\n if self.coverMinPercentage:\n f = StringIO.StringIO()\n self.coverInstance.report(modules, file=f)\n m = re.search(r'-------\\s\\w+\\s+\\d+\\s+\\d+\\s+(\\d+)%\\s+\\d*\\s{0,1}$', f.getvalue())\n if m:\n percentage = int(m.groups()[0])\n if percentage < self.coverMinPercentage:\n log.error('TOTAL Coverage did not reach minimum '\n 'required: %d%%' % self.coverMinPercentage)\n sys.exit(1)\n else:\n log.error(\"No total percentage was found in coverage output, \"\n \"something went wrong.\")", "response": "Output code coverage report."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef wantFile(self, file, package=None):\n if self.coverInclusive:\n if file.endswith(\".py\"):\n if package and self.coverPackages:\n for want in self.coverPackages:\n if package.startswith(want):\n return True\n else:\n return True\n return None", "response": "Returns True if file is in wanted packages."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef interpret_distro_name(location, basename, metadata,\n py_version=None, precedence=SOURCE_DIST, platform=None\n):\n \"\"\"Generate alternative interpretations of a source distro name\n\n Note: if `location` is a filesystem filename, you should call\n ``pkg_resources.normalize_path()`` on it before passing it to this\n routine!\n \"\"\"\n # Generate alternative interpretations of a source distro name\n # Because some packages are ambiguous as to name/versions split\n # e.g. \"adns-python-1.1.0\", \"egenix-mx-commercial\", etc.\n # So, we generate each possible interepretation (e.g. \"adns, python-1.1.0\"\n # \"adns-python, 1.1.0\", and \"adns-python-1.1.0, no version\"). In practice,\n # the spurious interpretations should be ignored, because in the event\n # there's also an \"adns\" package, the spurious \"python-1.1.0\" version will\n # compare lower than any numeric version number, and is therefore unlikely\n # to match a request for it. It's still a potential problem, though, and\n # in the long run PyPI and the distutils should go for \"safe\" names and\n # versions in distribution archive names (sdist and bdist).\n\n parts = basename.split('-')\n if not py_version:\n for i,p in enumerate(parts[2:]):\n if len(p)==5 and p.startswith('py2.'):\n return # It's a bdist_dumb, not an sdist -- bail out\n\n for p in range(1,len(parts)+1):\n yield Distribution(\n location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),\n py_version=py_version, precedence = precedence,\n platform = platform\n )", "response": "Generate an alternative interpretation of a source distro name."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _encode_auth(auth):\n auth_s = urllib2.unquote(auth)\n # convert to bytes\n auth_bytes = auth_s.encode()\n # use the legacy interface for Python 2.3 support\n encoded_bytes = base64.encodestring(auth_bytes)\n # convert back to a string\n encoded = encoded_bytes.decode()\n # strip the trailing carriage return\n return encoded.rstrip()", "response": "Encode auth string into base64 encoded string."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nopening a urllib2 request handling HTTP authentication", "response": "def open_with_auth(url):\n \"\"\"Open a urllib2 request, handling HTTP authentication\"\"\"\n\n scheme, netloc, path, params, query, frag = urlparse.urlparse(url)\n\n # Double scheme does not raise on Mac OS X as revealed by a\n # failing test. We would expect \"nonnumeric port\". Refs #20.\n if netloc.endswith(':'):\n raise httplib.InvalidURL(\"nonnumeric port: ''\")\n\n if scheme in ('http', 'https'):\n auth, host = urllib2.splituser(netloc)\n else:\n auth = None\n\n if auth:\n auth = \"Basic \" + _encode_auth(auth)\n new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))\n request = urllib2.Request(new_url)\n request.add_header(\"Authorization\", auth)\n else:\n request = urllib2.Request(url)\n\n request.add_header('User-Agent', user_agent)\n fp = urllib2.urlopen(request)\n\n if auth:\n # Put authentication info back into request URL if same host,\n # so that links found on the page will work\n s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)\n if s2==scheme and h2==host:\n fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))\n\n return fp"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfetches a specific version of a specific distribution from the given requirement.", "response": "def fetch_distribution(self,\n requirement, tmpdir, force_scan=False, source=False, develop_ok=False,\n local_index=None\n ):\n \"\"\"Obtain a distribution suitable for fulfilling `requirement`\n\n `requirement` must be a ``pkg_resources.Requirement`` instance.\n If necessary, or if the `force_scan` flag is set, the requirement is\n searched for in the (online) package index as well as the locally\n installed packages. If a distribution matching `requirement` is found,\n the returned distribution's ``location`` is the value you would have\n gotten from calling the ``download()`` method with the matching\n distribution's URL or filename. If no matching distribution is found,\n ``None`` is returned.\n\n If the `source` flag is set, only source distributions and source\n checkout links will be considered. Unless the `develop_ok` flag is\n set, development and system eggs (i.e., those using the ``.egg-info``\n format) will be ignored.\n \"\"\"\n\n # process a Requirement\n self.info(\"Searching for %s\", requirement)\n skipped = {}\n dist = None\n\n def find(req, env=None):\n if env is None:\n env = self\n # Find a matching distribution; may be called more than once\n\n for dist in env[req.key]:\n\n if dist.precedence==DEVELOP_DIST and not develop_ok:\n if dist not in skipped:\n self.warn(\"Skipping development or system egg: %s\",dist)\n skipped[dist] = 1\n continue\n\n if dist in req and (dist.precedence<=SOURCE_DIST or not source):\n self.info(\"Best match: %s\", dist)\n return dist.clone(\n location=self.download(dist.location, tmpdir)\n )\n\n if force_scan:\n self.prescan()\n self.find_packages(requirement)\n dist = find(requirement)\n\n if local_index is not None:\n dist = dist or find(requirement, local_index)\n\n if dist is None and self.to_scan is not None:\n self.prescan()\n dist = find(requirement)\n\n if dist is None and not force_scan:\n self.find_packages(requirement)\n dist = find(requirement)\n\n if dist is None:\n self.warn(\n \"No local packages or download links found for %s%s\",\n (source and \"a source distribution of \" or \"\"),\n requirement,\n )\n return dist"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_parent(obj):\n '''\n get parent from obj.\n '''\n names = obj.__qualname__.split('.')[:-1]\n if '' in names: # locals function\n raise ValueError('cannot get parent from locals object.')\n module = sys.modules[obj.__module__]\n parent = module\n while names:\n parent = getattr(parent, names.pop(0))\n return parent", "response": "get parent from obj.\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef root_topic(self):\n if isinstance(getattr(self.engine, 'id', None), int):\n return \"engine.%i\"%self.engine.id\n else:\n return \"engine\"", "response": "this is a property in case the handler is created\n before the engine gets registered with an id"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize a FakeModule instance s __dict__.", "response": "def init_fakemod_dict(fm,adict=None):\n \"\"\"Initialize a FakeModule instance __dict__.\n\n Kept as a standalone function and not a method so the FakeModule API can\n remain basically empty.\n\n This should be considered for private IPython use, used in managing\n namespaces for %run.\n\n Parameters\n ----------\n\n fm : FakeModule instance\n\n adict : dict, optional\n \"\"\"\n\n dct = {}\n # It seems pydoc (and perhaps others) needs any module instance to\n # implement a __nonzero__ method, so we add it if missing:\n dct.setdefault('__nonzero__',lambda : True)\n dct.setdefault('__file__',__file__)\n\n if adict is not None:\n dct.update(adict)\n\n # Hard assignment of the object's __dict__. This is nasty but deliberate.\n fm.__dict__.clear()\n fm.__dict__.update(dct)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrender a template with the given content and context.", "response": "def render_template(content, context):\n \"\"\" renders context aware template \"\"\"\n rendered = Template(content).render(Context(context))\n return rendered"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef configure(self, options, conf):\n self.conf = conf\n if not options.capture:\n self.enabled = False", "response": "Configure plugin. Plugin is enabled by default."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef formatError(self, test, err):\n test.capturedOutput = output = self.buffer\n self._buf = None\n if not output:\n # Don't return None as that will prevent other\n # formatters from formatting and remove earlier formatters\n # formats, instead return the err we got\n return err\n ec, ev, tb = err\n return (ec, self.addCaptureToErr(ev, output), tb)", "response": "Add captured output to error report."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nturns a list to list of list", "response": "def splitBy(data, num):\n \"\"\" Turn a list to list of list \"\"\"\n return [data[i:i + num] for i in range(0, len(data), num)]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef convert_to_this_nbformat(nb, orig_version=2, orig_minor=0):\n if orig_version == 1:\n nb = v2.convert_to_this_nbformat(nb)\n orig_version = 2\n if orig_version == 2:\n # Mark the original nbformat so consumers know it has been converted.\n nb.nbformat = nbformat\n nb.nbformat_minor = nbformat_minor\n \n nb.orig_nbformat = 2\n return nb\n elif orig_version == 3:\n if orig_minor != nbformat_minor:\n nb.orig_nbformat_minor = orig_minor\n nb.nbformat_minor = nbformat_minor\n return nb\n else:\n raise ValueError('Cannot convert a notebook from v%s to v3' % orig_version)", "response": "Convert a notebook to the v2 format."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef hex_to_rgb(color):\n if color.startswith('#'):\n color = color[1:]\n if len(color) == 3:\n color = ''.join([c*2 for c in color])\n if len(color) != 6:\n return False\n try:\n r = int(color[:2],16)\n g = int(color[2:4],16)\n b = int(color[4:],16)\n except ValueError:\n return False\n else:\n return r,g,b", "response": "Convert a hex color to rgb integer tuple."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_colors(stylename):\n style = get_style_by_name(stylename)\n fgcolor = style.style_for_token(Token.Text)['color'] or ''\n if len(fgcolor) in (3,6):\n # could be 'abcdef' or 'ace' hex, which needs '#' prefix\n try:\n int(fgcolor, 16)\n except TypeError:\n pass\n else:\n fgcolor = \"#\"+fgcolor\n\n return dict(\n bgcolor = style.background_color,\n select = style.highlight_color,\n fgcolor = fgcolor\n )", "response": "Construct the keys to be used building the base stylesheet\n from a templatee."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sheet_from_template(name, colors='lightbg'):\n colors = colors.lower()\n if colors=='lightbg':\n return default_light_style_template%get_colors(name)\n elif colors=='linux':\n return default_dark_style_template%get_colors(name)\n elif colors=='nocolor':\n return default_bw_style_sheet\n else:\n raise KeyError(\"No such color scheme: %s\"%colors)", "response": "Use one of the base templates and set bg / fg select colors."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_font(family, fallback=None):\n font = QtGui.QFont(family)\n # Check whether we got what we wanted using QFontInfo, since exactMatch()\n # is overly strict and returns false in too many cases.\n font_info = QtGui.QFontInfo(font)\n if fallback is not None and font_info.family() != family:\n font = QtGui.QFont(fallback)\n return font", "response": "Returns a font object for the given family."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nhandles a complete reply.", "response": "def _handle_complete_reply(self, rep):\n \"\"\" Reimplemented to support IPython's improved completion machinery.\n \"\"\"\n self.log.debug(\"complete: %s\", rep.get('content', ''))\n cursor = self._get_cursor()\n info = self._request_info.get('complete')\n if info and info.id == rep['parent_header']['msg_id'] and \\\n info.pos == cursor.position():\n matches = rep['content']['matches']\n text = rep['content']['matched_text']\n offset = len(text)\n\n # Clean up matches with period and path separators if the matched\n # text has not been transformed. This is done by truncating all\n # but the last component and then suitably decreasing the offset\n # between the current cursor position and the start of completion.\n if len(matches) > 1 and matches[0][:offset] == text:\n parts = re.split(r'[./\\\\]', text)\n sep_count = len(parts) - 1\n if sep_count:\n chop_length = sum(map(len, parts[:sep_count])) + sep_count\n matches = [ match[chop_length:] for match in matches ]\n offset -= chop_length\n\n # Move the cursor to the start of the match and complete.\n cursor.movePosition(QtGui.QTextCursor.Left, n=offset)\n self._complete_with_items(cursor, matches)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _handle_history_reply(self, msg):\n content = msg['content']\n if 'history' not in content:\n self.log.error(\"History request failed: %r\"%content)\n if content.get('status', '') == 'aborted' and \\\n not self._retrying_history_request:\n # a *different* action caused this request to be aborted, so\n # we should try again.\n self.log.error(\"Retrying aborted history request\")\n # prevent multiple retries of aborted requests:\n self._retrying_history_request = True\n # wait out the kernel's queue flush, which is currently timed at 0.1s\n time.sleep(0.25)\n self.kernel_manager.shell_channel.history(hist_access_type='tail',n=1000)\n else:\n self._retrying_history_request = False\n return\n # reset retry flag\n self._retrying_history_request = False\n history_items = content['history']\n self.log.debug(\"Received history reply with %i entries\", len(history_items))\n items = []\n last_cell = u\"\"\n for _, _, cell in history_items:\n cell = cell.rstrip()\n if cell != last_cell:\n items.append(cell)\n last_cell = cell\n self._set_history(items)", "response": "Handles the history reply from the kernel."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _handle_display_data(self, msg):\n self.log.debug(\"display: %s\", msg.get('content', ''))\n # For now, we don't display data from other frontends, but we\n # eventually will as this allows all frontends to monitor the display\n # data. But we need to figure out how to handle this in the GUI.\n if not self._hidden and self._is_from_this_session(msg):\n source = msg['content']['source']\n data = msg['content']['data']\n metadata = msg['content']['metadata']\n # In the regular IPythonWidget, we simply print the plain text\n # representation.\n if data.has_key('text/html'):\n html = data['text/html']\n self._append_html(html, True)\n elif data.has_key('text/plain'):\n text = data['text/plain']\n self._append_plain_text(text, True)\n # This newline seems to be needed for text and html output.\n self._append_plain_text(u'\\n', True)", "response": "Handle the display_data message."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _started_channels(self):\n super(IPythonWidget, self)._started_channels()\n self._load_guiref_magic()\n self.kernel_manager.shell_channel.history(hist_access_type='tail',\n n=1000)", "response": "Reimplemented to make a history request and load %guiref."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexecutes a file in the current directory.", "response": "def execute_file(self, path, hidden=False):\n \"\"\" Reimplemented to use the 'run' magic.\n \"\"\"\n # Use forward slashes on Windows to avoid escaping each separator.\n if sys.platform == 'win32':\n path = os.path.normpath(path).replace('\\\\', '/')\n\n # Perhaps we should not be using %run directly, but while we\n # are, it is necessary to quote or escape filenames containing spaces \n # or quotes. \n \n # In earlier code here, to minimize escaping, we sometimes quoted the \n # filename with single quotes. But to do this, this code must be\n # platform-aware, because run uses shlex rather than python string\n # parsing, so that:\n # * In Win: single quotes can be used in the filename without quoting,\n # and we cannot use single quotes to quote the filename.\n # * In *nix: we can escape double quotes in a double quoted filename,\n # but can't escape single quotes in a single quoted filename.\n \n # So to keep this code non-platform-specific and simple, we now only\n # use double quotes to quote filenames, and escape when needed:\n if ' ' in path or \"'\" in path or '\"' in path:\n path = '\"%s\"' % path.replace('\"', '\\\\\"')\n self.execute('%%run %s' % path, hidden=hidden)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsends a complete request to the kernel.", "response": "def _complete(self):\n \"\"\" Reimplemented to support IPython's improved completion machinery.\n \"\"\"\n # We let the kernel split the input line, so we *always* send an empty\n # text field. Readline-based frontends do get a real text field which\n # they can use.\n text = ''\n\n # Send the completion request to the kernel\n msg_id = self.kernel_manager.shell_channel.complete(\n text, # text\n self._get_input_buffer_cursor_line(), # line\n self._get_input_buffer_cursor_column(), # cursor_pos\n self.input_buffer) # block\n pos = self._get_cursor().position()\n info = self._CompletionRequest(msg_id, pos)\n self._request_info['complete'] = info"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _process_execute_error(self, msg):\n content = msg['content']\n traceback = '\\n'.join(content['traceback']) + '\\n'\n if False:\n # FIXME: For now, tracebacks come as plain text, so we can't use\n # the html renderer yet. Once we refactor ultratb to produce\n # properly styled tracebacks, this branch should be the default\n traceback = traceback.replace(' ', ' ')\n traceback = traceback.replace('\\n', '
')\n\n ename = content['ename']\n ename_styled = '%s' % ename\n traceback = traceback.replace(ename, ename_styled)\n\n self._append_html(traceback)\n else:\n # This is the fallback for now, using plain text with ansi escapes\n self._append_plain_text(traceback)", "response": "Reimplemented for IPython - style traceback formatting."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _process_execute_payload(self, item):\n handler = self._payload_handlers.get(item['source'])\n if handler is None:\n # We have no handler for this type of payload, simply ignore it\n return False\n else:\n handler(item)\n return True", "response": "Reimplemented to dispatch payloads to handler methods."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _show_interpreter_prompt(self, number=None):\n # If a number was not specified, make a prompt number request.\n if number is None:\n msg_id = self.kernel_manager.shell_channel.execute('', silent=True)\n info = self._ExecutionRequest(msg_id, 'prompt')\n self._request_info['execute'][msg_id] = info\n return\n\n # Show a new prompt and save information about it so that it can be\n # updated later if the prompt number turns out to be wrong.\n self._prompt_sep = self.input_sep\n self._show_prompt(self._make_in_prompt(number), html=True)\n block = self._control.document().lastBlock()\n length = len(self._prompt)\n self._previous_prompt_obj = self._PromptBlock(block, length, number)\n\n # Update continuation prompt to reflect (possibly) new prompt length.\n self._set_continuation_prompt(\n self._make_continuation_prompt(self._prompt), html=True)", "response": "Reimplemented for IPython - style prompts."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the widget style to the class defaults.", "response": "def set_default_style(self, colors='lightbg'):\n \"\"\" Sets the widget style to the class defaults.\n\n Parameters:\n -----------\n colors : str, optional (default lightbg)\n Whether to use the default IPython light background or dark\n background or B&W style.\n \"\"\"\n colors = colors.lower()\n if colors=='lightbg':\n self.style_sheet = styles.default_light_style_sheet\n self.syntax_style = styles.default_light_syntax_style\n elif colors=='linux':\n self.style_sheet = styles.default_dark_style_sheet\n self.syntax_style = styles.default_dark_syntax_style\n elif colors=='nocolor':\n self.style_sheet = styles.default_bw_style_sheet\n self.syntax_style = styles.default_bw_syntax_style\n else:\n raise KeyError(\"No such color scheme: %s\"%colors)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nopens a Python script for editing the local system file.", "response": "def _edit(self, filename, line=None):\n \"\"\" Opens a Python script for editing.\n\n Parameters:\n -----------\n filename : str\n A path to a local system file.\n\n line : int, optional\n A line of interest in the file.\n \"\"\"\n if self.custom_edit:\n self.custom_edit_requested.emit(filename, line)\n elif not self.editor:\n self._append_plain_text('No default editor available.\\n'\n 'Specify a GUI text editor in the `IPythonWidget.editor` '\n 'configurable to enable the %edit magic')\n else:\n try:\n filename = '\"%s\"' % filename\n if line and self.editor_line:\n command = self.editor_line.format(filename=filename,\n line=line)\n else:\n try:\n command = self.editor.format()\n except KeyError:\n command = self.editor.format(filename=filename)\n else:\n command += ' ' + filename\n except KeyError:\n self._append_plain_text('Invalid editor command.\\n')\n else:\n try:\n Popen(command, shell=True)\n except OSError:\n msg = 'Opening editor with command \"%s\" failed.\\n'\n self._append_plain_text(msg % command)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngives a prompt number returns an HTML In prompt.", "response": "def _make_in_prompt(self, number):\n \"\"\" Given a prompt number, returns an HTML In prompt.\n \"\"\"\n try:\n body = self.in_prompt % number\n except TypeError:\n # allow in_prompt to leave out number, e.g. '>>> '\n body = self.in_prompt\n return '%s' % body"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngiving a plain text version of an In prompt returns an HTML continuation prompt.", "response": "def _make_continuation_prompt(self, prompt):\n \"\"\" Given a plain text version of an In prompt, returns an HTML\n continuation prompt.\n \"\"\"\n end_chars = '...: '\n space_count = len(prompt.lstrip('\\n')) - len(end_chars)\n body = ' ' * space_count + end_chars\n return '%s' % body"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _style_sheet_changed(self):\n self.setStyleSheet(self.style_sheet)\n if self._control is not None:\n self._control.document().setDefaultStyleSheet(self.style_sheet)\n bg_color = self._control.palette().window().color()\n self._ansi_processor.set_background_color(bg_color)\n \n if self._page_control is not None:\n self._page_control.document().setDefaultStyleSheet(self.style_sheet)", "response": "Set the style sheets of the underlying widgets."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting the style for the syntax highlighter.", "response": "def _syntax_style_changed(self):\n \"\"\" Set the style for the syntax highlighter.\n \"\"\"\n if self._highlighter is None:\n # ignore premature calls\n return\n if self.syntax_style:\n self._highlighter.set_style(self.syntax_style)\n else:\n self._highlighter.set_style_sheet(self.style_sheet)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\nasync def request(self, command: str, **kwargs) -> dict:\n kwargs.update(dict(apikey=self.api_key, command=command, response='json'))\n\n if 'list' in command.lower(): # list APIs can be paginated, therefore include max_page_size and page parameter\n kwargs.update(dict(pagesize=self.max_page_size, page=1))\n\n final_data = dict()\n while True:\n async with self.client_session.get(self.end_point, params=self._sign(kwargs)) as response:\n data = await self._handle_response(response=response,\n await_final_result='queryasyncjobresult' not in command.lower())\n try:\n count = data.pop('count')\n except KeyError:\n if not data:\n # Empty dictionary is returned in case a query does not contain any results.\n # Can happen also if a listAPI is called with a page that does not exits. Therefore, final_data\n # has to be returned in order to return all results from preceding pages.\n return final_data\n else:\n # Only list API calls have a 'count' key inside the response,\n # return data as it is in other cases!\n return data\n else:\n # update final_data using paginated results, dictionaries of the response contain the count key\n # and one key pointing to the actual data values\n for key, value in data.items():\n final_data.setdefault(key, []).extend(value)\n final_data['count'] = final_data.setdefault('count', 0) + count\n kwargs['page'] += 1\n if count < self.max_page_size: # no more pages exists\n return final_data", "response": "A method to perform a request to a CloudStack API."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\nasync def _handle_response(self, response: aiohttp.client_reqrep.ClientResponse, await_final_result: bool) -> dict:\n try:\n data = await response.json()\n except aiohttp.client_exceptions.ContentTypeError:\n text = await response.text()\n logging.debug('Content returned by server not of type \"application/json\"\\n Content: {}'.format(text))\n raise CloudStackClientException(message=\"Could not decode content. Server did not return json content!\")\n else:\n data = self._transform_data(data)\n if response.status != 200:\n raise CloudStackClientException(message=\"Async CloudStack call failed!\",\n error_code=data.get(\"errorcode\", response.status),\n error_text=data.get(\"errortext\"),\n response=data)\n\n while await_final_result and ('jobid' in data):\n await asyncio.sleep(self.async_poll_latency)\n data = await self.queryAsyncJobResult(jobid=data['jobid'])\n if data['jobstatus']: # jobstatus is 0 for pending async CloudStack calls\n if not data['jobresultcode']: # exit code is zero\n try:\n return data['jobresult']\n except KeyError:\n pass\n logging.debug(\"Async CloudStack call returned {}\".format(str(data)))\n raise CloudStackClientException(message=\"Async CloudStack call failed!\",\n error_code=data.get(\"errorcode\"),\n error_text=data.get(\"errortext\"),\n response=data)\n\n return data", "response": "Handles the response from the async API call."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _sign(self, url_parameters: dict) -> dict:\n if url_parameters:\n url_parameters.pop('signature', None) # remove potential existing signature from url parameters\n request_string = urlencode(sorted(url_parameters.items()), safe='.-*_', quote_via=quote).lower()\n digest = hmac.new(self.api_secret.encode('utf-8'), request_string.encode('utf-8'), hashlib.sha1).digest()\n url_parameters['signature'] = base64.b64encode(digest).decode('utf-8').strip()\n return url_parameters", "response": "Signs the url_parameters with the API key and returns the url_parameters with the signature"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _transform_data(data: dict) -> dict:\n for key in data.keys():\n return_value = data[key]\n if isinstance(return_value, dict):\n return return_value\n return data", "response": "This function transforms the data structure to a nested dictionary structure."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef virtual_memory():\n mem = _psutil_bsd.get_virtual_mem()\n total, free, active, inactive, wired, cached, buffers, shared = mem\n avail = inactive + cached + free\n used = active + wired + cached\n percent = usage_percent((total - avail), total, _round=1)\n return nt_virtmem_info(total, avail, percent, used, free,\n active, inactive, buffers, cached, shared, wired)", "response": "System virtual memory as a namedutple."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_system_cpu_times():\n user, nice, system, idle, irq = _psutil_bsd.get_system_cpu_times()\n return _cputimes_ntuple(user, nice, system, idle, irq)", "response": "Return system per - CPU times as a named tuple"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_system_per_cpu_times():\n ret = []\n for cpu_t in _psutil_bsd.get_system_per_cpu_times():\n user, nice, system, idle, irq = cpu_t\n item = _cputimes_ntuple(user, nice, system, idle, irq)\n ret.append(item)\n return ret", "response": "Return a list of named tuples representing the system CPU times in the system CPU pool."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn real effective and saved user ids.", "response": "def get_process_uids(self):\n \"\"\"Return real, effective and saved user ids.\"\"\"\n real, effective, saved = _psutil_bsd.get_process_uids(self.pid)\n return nt_uids(real, effective, saved)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns real effective and saved group ids.", "response": "def get_process_gids(self):\n \"\"\"Return real, effective and saved group ids.\"\"\"\n real, effective, saved = _psutil_bsd.get_process_gids(self.pid)\n return nt_gids(real, effective, saved)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_cpu_times(self):\n user, system = _psutil_bsd.get_process_cpu_times(self.pid)\n return nt_cputimes(user, system)", "response": "return a tuple containing process user and kernel time."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_memory_info(self):\n rss, vms = _psutil_bsd.get_process_memory_info(self.pid)[:2]\n return nt_meminfo(rss, vms)", "response": "Return a tuple with the process RSS and VMS size."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_process_threads(self):\n rawlist = _psutil_bsd.get_process_threads(self.pid)\n retlist = []\n for thread_id, utime, stime in rawlist:\n ntuple = nt_thread(thread_id, utime, stime)\n retlist.append(ntuple)\n return retlist", "response": "Return the number of threads belonging to the process."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn files opened by process as a list of namedtuples.", "response": "def get_open_files(self):\n \"\"\"Return files opened by process as a list of namedtuples.\"\"\"\n # XXX - C implementation available on FreeBSD >= 8 only\n # else fallback on lsof parser\n if hasattr(_psutil_bsd, \"get_process_open_files\"):\n rawlist = _psutil_bsd.get_process_open_files(self.pid)\n return [nt_openfile(path, fd) for path, fd in rawlist]\n else:\n lsof = _psposix.LsofParser(self.pid, self._process_name)\n return lsof.get_process_open_files()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_connection_file(app=None):\n if app is None:\n from IPython.zmq.ipkernel import IPKernelApp\n if not IPKernelApp.initialized():\n raise RuntimeError(\"app not specified, and not in a running Kernel\")\n\n app = IPKernelApp.instance()\n return filefind(app.connection_file, ['.', app.profile_dir.security_dir])", "response": "Returns the path to the connection file of an app\n \n "} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfind a connection file and return its absolute path.", "response": "def find_connection_file(filename, profile=None):\n \"\"\"find a connection file, and return its absolute path.\n \n The current working directory and the profile's security\n directory will be searched for the file if it is not given by\n absolute path.\n \n If profile is unspecified, then the current running application's\n profile will be used, or 'default', if not run from IPython.\n \n If the argument does not match an existing file, it will be interpreted as a\n fileglob, and the matching file in the profile's security dir with\n the latest access time will be used.\n \n Parameters\n ----------\n filename : str\n The connection file or fileglob to search for.\n profile : str [optional]\n The name of the profile to use when searching for the connection file,\n if different from the current IPython session or 'default'.\n \n Returns\n -------\n str : The absolute path of the connection file.\n \"\"\"\n from IPython.core.application import BaseIPythonApplication as IPApp\n try:\n # quick check for absolute path, before going through logic\n return filefind(filename)\n except IOError:\n pass\n \n if profile is None:\n # profile unspecified, check if running from an IPython app\n if IPApp.initialized():\n app = IPApp.instance()\n profile_dir = app.profile_dir\n else:\n # not running in IPython, use default profile\n profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default')\n else:\n # find profiledir by profile name:\n profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)\n security_dir = profile_dir.security_dir\n \n try:\n # first, try explicit name\n return filefind(filename, ['.', security_dir])\n except IOError:\n pass\n \n # not found by full name\n \n if '*' in filename:\n # given as a glob already\n pat = filename\n else:\n # accept any substring match\n pat = '*%s*' % filename\n matches = glob.glob( os.path.join(security_dir, pat) )\n if not matches:\n raise IOError(\"Could not find %r in %r\" % (filename, security_dir))\n elif len(matches) == 1:\n return matches[0]\n else:\n # get most recent match, by access time:\n return sorted(matches, key=lambda f: os.stat(f).st_atime)[-1]"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_connection_info(connection_file=None, unpack=False, profile=None):\n if connection_file is None:\n # get connection file from current kernel\n cf = get_connection_file()\n else:\n # connection file specified, allow shortnames:\n cf = find_connection_file(connection_file, profile=profile)\n \n with open(cf) as f:\n info = f.read()\n \n if unpack:\n info = json.loads(info)\n # ensure key is bytes:\n info['key'] = str_to_bytes(info.get('key', ''))\n return info", "response": "Returns the connection information for the current kernel."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef connect_qtconsole(connection_file=None, argv=None, profile=None):\n argv = [] if argv is None else argv\n \n if connection_file is None:\n # get connection file from current kernel\n cf = get_connection_file()\n else:\n cf = find_connection_file(connection_file, profile=profile)\n \n cmd = ';'.join([\n \"from IPython.frontend.qt.console import qtconsoleapp\",\n \"qtconsoleapp.main()\"\n ])\n \n return Popen([sys.executable, '-c', cmd, '--existing', cf] + argv, stdout=PIPE, stderr=PIPE)", "response": "Connect a qtconsole to the current kernel."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntunnels connections to a kernel via ssh", "response": "def tunnel_to_kernel(connection_info, sshserver, sshkey=None):\n \"\"\"tunnel connections to a kernel via ssh\n \n This will open four SSH tunnels from localhost on this machine to the\n ports associated with the kernel. They can be either direct\n localhost-localhost tunnels, or if an intermediate server is necessary,\n the kernel must be listening on a public IP.\n \n Parameters\n ----------\n connection_info : dict or str (path)\n Either a connection dict, or the path to a JSON connection file\n sshserver : str\n The ssh sever to use to tunnel to the kernel. Can be a full\n `user@server:port` string. ssh config aliases are respected.\n sshkey : str [optional]\n Path to file containing ssh key to use for authentication.\n Only necessary if your ssh config does not already associate\n a keyfile with the host.\n \n Returns\n -------\n \n (shell, iopub, stdin, hb) : ints\n The four ports on localhost that have been forwarded to the kernel.\n \"\"\"\n if isinstance(connection_info, basestring):\n # it's a path, unpack it\n with open(connection_info) as f:\n connection_info = json.loads(f.read())\n \n cf = connection_info\n \n lports = tunnel.select_random_ports(4)\n rports = cf['shell_port'], cf['iopub_port'], cf['stdin_port'], cf['hb_port']\n \n remote_ip = cf['ip']\n \n if tunnel.try_passwordless_ssh(sshserver, sshkey):\n password=False\n else:\n password = getpass(\"SSH Password for %s: \"%sshserver)\n \n for lp,rp in zip(lports, rports):\n tunnel.ssh_tunnel(lp, rp, sshserver, remote_ip, sshkey, password)\n \n return tuple(lports)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef swallow_argv(argv, aliases=None, flags=None):\n \n if aliases is None:\n aliases = set()\n if flags is None:\n flags = set()\n \n stripped = list(argv) # copy\n \n swallow_next = False\n was_flag = False\n for a in argv:\n if swallow_next:\n swallow_next = False\n # last arg was an alias, remove the next one\n # *unless* the last alias has a no-arg flag version, in which\n # case, don't swallow the next arg if it's also a flag:\n if not (was_flag and a.startswith('-')):\n stripped.remove(a)\n continue\n if a.startswith('-'):\n split = a.lstrip('-').split('=')\n alias = split[0]\n if alias in aliases:\n stripped.remove(a)\n if len(split) == 1:\n # alias passed with arg via space\n swallow_next = True\n # could have been a flag that matches an alias, e.g. `existing`\n # in which case, we might not swallow the next arg\n was_flag = alias in flags\n elif alias in flags and len(split) == 1:\n # strip flag, but don't swallow next, as flags don't take args\n stripped.remove(a)\n \n # return shortened list\n return stripped", "response": "This function swallows the arguments passed through to a subprocess."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the short form of commit hash given directory pkg_path", "response": "def pkg_commit_hash(pkg_path):\n \"\"\"Get short form of commit hash given directory `pkg_path`\n\n We get the commit hash from (in order of preference):\n\n * IPython.utils._sysinfo.commit\n * git output, if we are in a git repository\n\n If these fail, we return a not-found placeholder tuple\n\n Parameters\n ----------\n pkg_path : str\n directory containing package\n only used for getting commit from active repo\n\n Returns\n -------\n hash_from : str\n Where we got the hash from - description\n hash_str : str\n short form of hash\n \"\"\"\n # Try and get commit from written commit text file\n if _sysinfo.commit:\n return \"installation\", _sysinfo.commit\n\n # maybe we are in a repository\n proc = subprocess.Popen('git rev-parse --short HEAD',\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n cwd=pkg_path, shell=True)\n repo_commit, _ = proc.communicate()\n if repo_commit:\n return 'repository', repo_commit.strip()\n return '(none found)', ''"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pkg_info(pkg_path):\n src, hsh = pkg_commit_hash(pkg_path)\n return dict(\n ipython_version=release.version,\n ipython_path=pkg_path,\n commit_source=src,\n commit_hash=hsh,\n sys_version=sys.version,\n sys_executable=sys.executable,\n sys_platform=sys.platform,\n platform=platform.platform(),\n os_name=os.name,\n default_encoding=encoding.DEFAULT_ENCODING,\n )", "response": "Return dict describing the context of this package"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sys_info():\n p = os.path\n path = p.dirname(p.abspath(p.join(__file__, '..')))\n return pprint.pformat(pkg_info(path))", "response": "Return useful information about IPython and system."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _num_cpus_darwin():\n p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)\n return p.stdout.read()", "response": "Return the number of active CPUs on a Darwin system."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef num_cpus():\n\n # Many thanks to the Parallel Python project (http://www.parallelpython.com)\n # for the names of the keys we needed to look up for this function. This\n # code was inspired by their equivalent function.\n\n ncpufuncs = {'Linux':_num_cpus_unix,\n 'Darwin':_num_cpus_darwin,\n 'Windows':_num_cpus_windows,\n # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'\n # See http://bugs.python.org/issue1082 for details.\n 'Microsoft':_num_cpus_windows,\n }\n\n ncpufunc = ncpufuncs.get(platform.system(),\n # default to unix version (Solaris, AIX, etc)\n _num_cpus_unix)\n\n try:\n ncpus = max(1,int(ncpufunc()))\n except:\n ncpus = 1\n return ncpus", "response": "Return the effective number of CPUs in the system as an integer."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef handle(self, integers, **options):\n print integers, options\n return super(Command, self).handle(integers, **options)", "response": "Handle the n - tuple command."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadvance to the next result set. Returns 1 if there are no more result sets. Returns None if there are no more result sets.", "response": "def nextset(self):\n \"\"\"Advance to the next result set.\n\n Returns None if there are no more result sets.\n \"\"\"\n if self._executed:\n self.fetchall()\n del self.messages[:]\n \n db = self._get_db()\n nr = db.next_result()\n if nr == -1:\n return None\n self._do_get_result()\n self._post_get_result()\n self._warning_check()\n return 1"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef execute(self, query, args=None):\n\n \"\"\"Execute a query.\n \n query -- string, query to execute on server\n args -- optional sequence or mapping, parameters to use with query.\n\n Note: If args is a sequence, then %s must be used as the\n parameter placeholder in the query. If a mapping is used,\n %(key)s must be used as the placeholder.\n\n Returns long integer rows affected, if any\n\n \"\"\"\n del self.messages[:]\n db = self._get_db()\n if isinstance(query, unicode):\n query = query.encode(db.unicode_literal.charset)\n if args is not None:\n query = query % db.literal(args)\n try:\n r = None\n r = self._query(query)\n except TypeError, m:\n if m.args[0] in (\"not enough arguments for format string\",\n \"not all arguments converted\"):\n self.messages.append((ProgrammingError, m.args[0]))\n self.errorhandler(self, ProgrammingError, m.args[0])\n else:\n self.messages.append((TypeError, m))\n self.errorhandler(self, TypeError, m)\n except (SystemExit, KeyboardInterrupt):\n raise\n except:\n exc, value, tb = sys.exc_info()\n del tb\n self.messages.append((exc, value))\n self.errorhandler(self, exc, value)\n self._executed = query\n if not self._defer_warnings: self._warning_check()\n return r", "response": "Executes a query on the server and returns the number of affected rows."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nexecutes a multi - row query.", "response": "def executemany(self, query, args):\n\n \"\"\"Execute a multi-row query.\n \n query -- string, query to execute on server\n\n args\n\n Sequence of sequences or mappings, parameters to use with\n query.\n \n Returns long integer rows affected, if any.\n \n This method improves performance on multiple-row INSERT and\n REPLACE. Otherwise it is equivalent to looping over args with\n execute().\n\n \"\"\"\n del self.messages[:]\n db = self._get_db()\n if not args: return\n if isinstance(query, unicode):\n query = query.encode(db.unicode_literal.charset)\n m = insert_values.search(query)\n if not m:\n r = 0\n for a in args:\n r = r + self.execute(query, a)\n return r\n p = m.start(1)\n e = m.end(1)\n qv = m.group(1)\n try:\n q = [ qv % db.literal(a) for a in args ]\n except TypeError, msg:\n if msg.args[0] in (\"not enough arguments for format string\",\n \"not all arguments converted\"):\n self.errorhandler(self, ProgrammingError, msg.args[0])\n else:\n self.errorhandler(self, TypeError, msg)\n except (SystemExit, KeyboardInterrupt):\n raise\n except:\n exc, value, tb = sys.exc_info()\n del tb\n self.errorhandler(self, exc, value)\n r = self._query('\\n'.join([query[:p], ',\\n'.join(q), query[e:]]))\n if not self._defer_warnings: self._warning_check()\n return r"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef callproc(self, procname, args=()):\n\n \"\"\"Execute stored procedure procname with args\n \n procname -- string, name of procedure to execute on server\n\n args -- Sequence of parameters to use with procedure\n\n Returns the original args.\n\n Compatibility warning: PEP-249 specifies that any modified\n parameters must be returned. This is currently impossible\n as they are only available by storing them in a server\n variable and then retrieved by a query. Since stored\n procedures return zero or more result sets, there is no\n reliable way to get at OUT or INOUT parameters via callproc.\n The server variables are named @_procname_n, where procname\n is the parameter above and n is the position of the parameter\n (from zero). Once all result sets generated by the procedure\n have been fetched, you can issue a SELECT @_procname_0, ...\n query using .execute() to get any OUT or INOUT values.\n\n Compatibility warning: The act of calling a stored procedure\n itself creates an empty result set. This appears after any\n result sets generated by the procedure. This is non-standard\n behavior with respect to the DB-API. Be sure to use nextset()\n to advance through all result sets; otherwise you may get\n disconnected.\n \"\"\"\n\n db = self._get_db()\n for index, arg in enumerate(args):\n q = \"SET @_%s_%d=%s\" % (procname, index,\n db.literal(arg))\n if isinstance(q, unicode):\n q = q.encode(db.unicode_literal.charset)\n self._query(q)\n self.nextset()\n \n q = \"CALL %s(%s)\" % (procname,\n ','.join(['@_%s_%d' % (procname, i)\n for i in range(len(args))]))\n if type(q) is UnicodeType:\n q = q.encode(db.unicode_literal.charset)\n self._query(q)\n self._executed = q\n if not self._defer_warnings: self._warning_check()\n return args", "response": "Execute a stored procedure procname with args returning the original args."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfetch a single row from the cursor. Returns None if there is no more rows.", "response": "def fetchone(self):\n \"\"\"Fetches a single row from the cursor.\"\"\"\n self._check_executed()\n r = self._fetch_row(1)\n if not r:\n self._warning_check()\n return None\n self.rownumber = self.rownumber + 1\n return r[0]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fetchmany(self, size=None):\n self._check_executed()\n r = self._fetch_row(size or self.arraysize)\n self.rownumber = self.rownumber + len(r)\n if not r:\n self._warning_check()\n return r", "response": "Fetch up to size rows from the cursor."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfetches all available rows from the cursor.", "response": "def fetchall(self):\n \"\"\"Fetchs all available rows from the cursor.\"\"\"\n self._check_executed()\n r = self._fetch_row(0)\n self.rownumber = self.rownumber + len(r)\n self._warning_check()\n return r"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfetches several rows as a list of dictionaries.", "response": "def fetchmanyDict(self, size=None):\n \"\"\"Fetch several rows as a list of dictionaries. Deprecated:\n Use fetchmany() instead. Will be removed in 1.3.\"\"\"\n from warnings import warn\n warn(\"fetchmanyDict() is non-standard and will be removed in 1.3\",\n DeprecationWarning, 2)\n return self.fetchmany(size)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconnecting a list of peers to a tree", "response": "def connect(com, peers, tree, pub_url, root_id):\n \"\"\"this function will be called on the engines\"\"\"\n com.connect(peers, tree, pub_url, root_id)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef parse_json(s, **kwargs):\n d = json.loads(s, **kwargs)\n nbf = d.get('nbformat', 1)\n nbm = d.get('nbformat_minor', 0)\n return nbf, nbm, d", "response": "Parse a string into a nbformat nbm dict tuple."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef parse_py(s, **kwargs):\n nbf = current_nbformat\n nbm = current_nbformat_minor\n \n pattern = r'# (?P\\d+[\\.\\d+]*)'\n m = re.search(pattern,s)\n if m is not None:\n digits = m.group('nbformat').split('.')\n nbf = int(digits[0])\n if len(digits) > 1:\n nbm = int(digits[1])\n\n return nbf, nbm, s", "response": "Parse a string into a ( nbformat nbm string ) tuple."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef reads_json(s, **kwargs):\n nbf, minor, d = parse_json(s, **kwargs)\n if nbf == 1:\n nb = v1.to_notebook_json(d, **kwargs)\n nb = v3.convert_to_this_nbformat(nb, orig_version=1)\n elif nbf == 2:\n nb = v2.to_notebook_json(d, **kwargs)\n nb = v3.convert_to_this_nbformat(nb, orig_version=2)\n elif nbf == 3:\n nb = v3.to_notebook_json(d, **kwargs)\n nb = v3.convert_to_this_nbformat(nb, orig_version=3, orig_minor=minor)\n else:\n raise NBFormatError('Unsupported JSON nbformat version: %i' % nbf)\n return nb", "response": "Read a JSON notebook from a string and return the NotebookNode object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading a. py notebook from a string and return the NotebookNode object.", "response": "def reads_py(s, **kwargs):\n \"\"\"Read a .py notebook from a string and return the NotebookNode object.\"\"\"\n nbf, nbm, s = parse_py(s, **kwargs)\n if nbf == 2:\n nb = v2.to_notebook_py(s, **kwargs)\n elif nbf == 3:\n nb = v3.to_notebook_py(s, **kwargs)\n else:\n raise NBFormatError('Unsupported PY nbformat version: %i' % nbf)\n return nb"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading a notebook from a string and return the NotebookNode object.", "response": "def reads(s, format, **kwargs):\n \"\"\"Read a notebook from a string and return the NotebookNode object.\n\n This function properly handles notebooks of any version. The notebook\n returned will always be in the current version's format.\n\n Parameters\n ----------\n s : unicode\n The raw unicode string to read the notebook from.\n format : (u'json', u'ipynb', u'py')\n The format that the string is in.\n\n Returns\n -------\n nb : NotebookNode\n The notebook that was read.\n \"\"\"\n format = unicode(format)\n if format == u'json' or format == u'ipynb':\n return reads_json(s, **kwargs)\n elif format == u'py':\n return reads_py(s, **kwargs)\n else:\n raise NBFormatError('Unsupported format: %s' % format)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites a notebook to a string in a given format in the current nbformat version.", "response": "def writes(nb, format, **kwargs):\n \"\"\"Write a notebook to a string in a given format in the current nbformat version.\n\n This function always writes the notebook in the current nbformat version.\n\n Parameters\n ----------\n nb : NotebookNode\n The notebook to write.\n format : (u'json', u'ipynb', u'py')\n The format to write the notebook in.\n\n Returns\n -------\n s : unicode\n The notebook string.\n \"\"\"\n format = unicode(format)\n if format == u'json' or format == u'ipynb':\n return writes_json(nb, **kwargs)\n elif format == u'py':\n return writes_py(nb, **kwargs)\n else:\n raise NBFormatError('Unsupported format: %s' % format)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a notebook to a file in a given format in the current nbformat version.", "response": "def write(nb, fp, format, **kwargs):\n \"\"\"Write a notebook to a file in a given format in the current nbformat version.\n\n This function always writes the notebook in the current nbformat version.\n\n Parameters\n ----------\n nb : NotebookNode\n The notebook to write.\n fp : file\n Any file-like object with a write method.\n format : (u'json', u'ipynb', u'py')\n The format to write the notebook in.\n\n Returns\n -------\n s : unicode\n The notebook string.\n \"\"\"\n return fp.write(writes(nb, format, **kwargs))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _convert_to_metadata():\n import glob\n for fname in glob.glob('*.ipynb'):\n print('Converting file:',fname)\n with open(fname,'r') as f:\n nb = read(f,u'json')\n md = new_metadata()\n if u'name' in nb:\n md.name = nb.name\n del nb[u'name'] \n nb.metadata = md\n with open(fname,'w') as f:\n write(nb, f, u'json')", "response": "Convert a notebook having notebook metadata to a notebook having notebook metadata."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload value from dict", "response": "def load_from_dict(self, src: dict, key):\n '''\n try load value from dict.\n if key is not exists, mark as state unset.\n '''\n if key in src:\n self.value = src[key]\n else:\n self.reset()"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns the pyglet event loop by processing pending events only. This keeps processing pending events until stdin is ready. After processing all pending events, a call to time.sleep is inserted. This is needed, otherwise, CPU usage is at 100%. This sleep time should be tuned though for best performance.", "response": "def inputhook_pyglet():\n \"\"\"Run the pyglet event loop by processing pending events only.\n\n This keeps processing pending events until stdin is ready. After\n processing all pending events, a call to time.sleep is inserted. This is\n needed, otherwise, CPU usage is at 100%. This sleep time should be tuned\n though for best performance.\n \"\"\"\n # We need to protect against a user pressing Control-C when IPython is\n # idle and this is running. We trap KeyboardInterrupt and pass.\n try:\n t = clock()\n while not stdin_ready():\n pyglet.clock.tick()\n for window in pyglet.app.windows:\n window.switch_to()\n window.dispatch_events()\n window.dispatch_event('on_draw')\n flip(window)\n\n # We need to sleep at this point to keep the idle CPU load\n # low. However, if sleep to long, GUI response is poor. As\n # a compromise, we watch how often GUI events are being processed\n # and switch between a short and long sleep time. Here are some\n # stats useful in helping to tune this.\n # time CPU load\n # 0.001 13%\n # 0.005 3%\n # 0.01 1.5%\n # 0.05 0.5%\n used_time = clock() - t\n if used_time > 5*60.0:\n # print 'Sleep for 5 s' # dbg\n time.sleep(5.0)\n elif used_time > 10.0:\n # print 'Sleep for 1 s' # dbg\n time.sleep(1.0)\n elif used_time > 0.1:\n # Few GUI events coming in, so we can sleep longer\n # print 'Sleep for 0.05 s' # dbg\n time.sleep(0.05)\n else:\n # Many GUI events coming in, so sleep only very little\n time.sleep(0.001)\n except KeyboardInterrupt:\n pass\n return 0"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndoing the name match my requirements?", "response": "def matches(self, name):\n \"\"\"Does the name match my requirements?\n\n To match, a name must match config.testMatch OR config.include\n and it must not match config.exclude\n \"\"\"\n return ((self.match.search(name)\n or (self.include and\n filter(None,\n [inc.search(name) for inc in self.include])))\n and ((not self.exclude)\n or not filter(None,\n [exc.search(name) for exc in self.exclude])\n ))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nbeing the class a wanted test class?", "response": "def wantClass(self, cls):\n \"\"\"Is the class a wanted test class?\n\n A class must be a unittest.TestCase subclass, or match test name\n requirements. Classes that start with _ are always excluded.\n \"\"\"\n declared = getattr(cls, '__test__', None)\n if declared is not None:\n wanted = declared\n else:\n wanted = (not cls.__name__.startswith('_')\n and (issubclass(cls, unittest.TestCase)\n or self.matches(cls.__name__)))\n \n plug_wants = self.plugins.wantClass(cls) \n if plug_wants is not None:\n log.debug(\"Plugin setting selection of %s to %s\", cls, plug_wants)\n wanted = plug_wants\n log.debug(\"wantClass %s? %s\", cls, wanted)\n return wanted"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nis the directory a wanted test directory?", "response": "def wantDirectory(self, dirname):\n \"\"\"Is the directory a wanted test directory?\n\n All package directories match, so long as they do not match exclude. \n All other directories must match test requirements.\n \"\"\"\n tail = op_basename(dirname)\n if ispackage(dirname):\n wanted = (not self.exclude\n or not filter(None,\n [exc.search(tail) for exc in self.exclude]\n ))\n else:\n wanted = (self.matches(tail)\n or (self.config.srcDirs\n and tail in self.config.srcDirs))\n plug_wants = self.plugins.wantDirectory(dirname)\n if plug_wants is not None:\n log.debug(\"Plugin setting selection of %s to %s\",\n dirname, plug_wants)\n wanted = plug_wants\n log.debug(\"wantDirectory %s? %s\", dirname, wanted)\n return wanted"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nam the file a wanted test file?", "response": "def wantFile(self, file):\n \"\"\"Is the file a wanted test file?\n\n The file must be a python source file and match testMatch or\n include, and not match exclude. Files that match ignore are *never*\n wanted, regardless of plugin, testMatch, include or exclude settings.\n \"\"\"\n # never, ever load files that match anything in ignore\n # (.* _* and *setup*.py by default)\n base = op_basename(file)\n ignore_matches = [ ignore_this for ignore_this in self.ignoreFiles\n if ignore_this.search(base) ]\n if ignore_matches:\n log.debug('%s matches ignoreFiles pattern; skipped',\n base) \n return False\n if not self.config.includeExe and os.access(file, os.X_OK):\n log.info('%s is executable; skipped', file)\n return False\n dummy, ext = op_splitext(base)\n pysrc = ext == '.py'\n\n wanted = pysrc and self.matches(base) \n plug_wants = self.plugins.wantFile(file)\n if plug_wants is not None:\n log.debug(\"plugin setting want %s to %s\", file, plug_wants)\n wanted = plug_wants\n log.debug(\"wantFile %s? %s\", file, wanted)\n return wanted"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef wantFunction(self, function):\n try:\n if hasattr(function, 'compat_func_name'):\n funcname = function.compat_func_name\n else:\n funcname = function.__name__\n except AttributeError:\n # not a function\n return False\n declared = getattr(function, '__test__', None)\n if declared is not None:\n wanted = declared\n else:\n wanted = not funcname.startswith('_') and self.matches(funcname)\n plug_wants = self.plugins.wantFunction(function)\n if plug_wants is not None:\n wanted = plug_wants\n log.debug(\"wantFunction %s? %s\", function, wanted)\n return wanted", "response": "Is the function a test function?"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef wantMethod(self, method):\n try:\n method_name = method.__name__\n except AttributeError:\n # not a method\n return False\n if method_name.startswith('_'):\n # never collect 'private' methods\n return False\n declared = getattr(method, '__test__', None)\n if declared is not None:\n wanted = declared\n else:\n wanted = self.matches(method_name)\n plug_wants = self.plugins.wantMethod(method)\n if plug_wants is not None:\n wanted = plug_wants\n log.debug(\"wantMethod %s? %s\", method, wanted)\n return wanted", "response": "Is the method a test method?"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbeing the module a test module?", "response": "def wantModule(self, module):\n \"\"\"Is the module a test module?\n\n The tail of the module name must match test requirements. One exception:\n we always want __main__.\n \"\"\"\n declared = getattr(module, '__test__', None)\n if declared is not None:\n wanted = declared\n else:\n wanted = self.matches(module.__name__.split('.')[-1]) \\\n or module.__name__ == '__main__'\n plug_wants = self.plugins.wantModule(module)\n if plug_wants is not None:\n wanted = plug_wants\n log.debug(\"wantModule %s? %s\", module, wanted)\n return wanted"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef decorate_fn_with_doc(new_fn, old_fn, additional_text=\"\"):\n def wrapper(*args, **kw):\n return new_fn(*args, **kw)\n if old_fn.__doc__:\n wrapper.__doc__ = old_fn.__doc__ + additional_text\n return wrapper", "response": "Make new_fn have old_fn s doc string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the contents of a named file as a list of lines.", "response": "def _file_lines(fname):\n \"\"\"Return the contents of a named file as a list of lines.\n\n This function never raises an IOError exception: if the file can't be\n read, it simply returns an empty list.\"\"\"\n\n try:\n outfile = open(fname)\n except IOError:\n return []\n else:\n out = outfile.readlines()\n outfile.close()\n return out"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef list_command_pydb(self, arg):\n filename, first, last = OldPdb.parse_list_cmd(self, arg)\n if filename is not None:\n self.print_list_lines(filename, first, last)", "response": "List command to use if we have a newer pydb installed"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef print_list_lines(self, filename, first, last):\n try:\n Colors = self.color_scheme_table.active_colors\n ColorsNormal = Colors.Normal\n tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)\n tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)\n src = []\n for lineno in range(first, last+1):\n line = linecache.getline(filename, lineno)\n if not line:\n break\n\n if lineno == self.curframe.f_lineno:\n line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)\n else:\n line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)\n\n src.append(line)\n self.lineno = lineno\n\n print >>io.stdout, ''.join(src)\n\n except KeyboardInterrupt:\n pass", "response": "The printing part of a list command."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef checkline(self, filename, lineno):\n #######################################################################\n # XXX Hack! Use python-2.5 compatible code for this call, because with\n # all of our changes, we've drifted from the pdb api in 2.6. For now,\n # changing:\n #\n #line = linecache.getline(filename, lineno, self.curframe.f_globals)\n # to:\n #\n line = linecache.getline(filename, lineno)\n #\n # does the trick. But in reality, we need to fix this by reconciling\n # our updates with the new Pdb APIs in Python 2.6.\n #\n # End hack. The rest of this method is copied verbatim from 2.6 pdb.py\n #######################################################################\n\n if not line:\n print >>self.stdout, 'End of file'\n return 0\n line = line.strip()\n # Don't allow setting breakpoint at a blank line\n if (not line or (line[0] == '#') or\n (line[:3] == '\"\"\"') or line[:3] == \"'''\"):\n print >>self.stdout, '*** Blank or comment'\n return 0\n return lineno", "response": "Check whether line is executable."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates a multiplying factor used to convert two currencies", "response": "def conversion_factor(from_symbol, to_symbol, date):\n \"\"\"\n Generates a multiplying factor used to convert two currencies\n \"\"\"\n\n from_currency = Currency.objects.get(symbol=from_symbol)\n try:\n from_currency_price = CurrencyPrice.objects.get(currency=from_currency, date=date).mid_price\n except CurrencyPrice.DoesNotExist:\n print \"Cannot fetch prices for %s on %s\" % (str(from_currency), str(date))\n return None\n\n to_currency = Currency.objects.get(symbol=to_symbol)\n try:\n to_currency_price = CurrencyPrice.objects.get(currency=to_currency, date=date).mid_price\n except CurrencyPrice.DoesNotExist:\n print \"Cannot fetch prices for %s on %s\" % (str(to_currency), str(date))\n return None\n\n return to_currency_price / from_currency_price"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert an amount of money from one currency to another on a specified date.", "response": "def convert_currency(from_symbol, to_symbol, value, date):\n \"\"\"\n Converts an amount of money from one currency to another on a specified date.\n \"\"\"\n if from_symbol == to_symbol:\n return value\n\n factor = conversion_factor(from_symbol, to_symbol, date)\n\n if type(value) == float:\n output = value * float(factor)\n elif type(value) == Decimal:\n output = Decimal(format(value * factor, '.%sf' % str(PRICE_PRECISION)))\n elif type(value) in [np.float16, np.float32, np.float64, np.float128, np.float]:\n output = float(value) * float(factor)\n else:\n output = None\n\n return output"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef compute_return(self, start_date, end_date, rate=\"MID\"):\n if rate not in [\"MID\", \"ASK\", \"BID\"]:\n raise ValueError(\"Unknown rate type (%s)- must be 'MID', 'ASK' or 'BID'\" % str(rate))\n\n if end_date <= start_date:\n raise ValueError(\"End date must be on or after start date\")\n\n df = self.generate_dataframe(start_date=start_date, end_date=end_date)\n start_price = df.ix[start_date][rate]\n end_price = df.ix[end_date][rate]\n\n currency_return = (end_price / start_price) - 1.0\n\n return currency_return", "response": "Compute the return of the currency between two dates\n "} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate a dataframe consisting of the currency prices of the specified symbols and date_index.", "response": "def generate_dataframe(self, symbols=None, date_index=None, price_type=\"mid\"):\n \"\"\"\n Generate a dataframe consisting of the currency prices (specified by symbols)\n from the start to end date\n \"\"\"\n\n # Set defaults if necessary\n if symbols is None:\n symbols = list(Currency.objects.all().values_list('symbol', flat=True))\n try:\n start_date = date_index[0]\n end_date = date_index[-1]\n except:\n start_date = DATEFRAME_START_DATE\n end_date = date.today()\n date_index = date_range(start_date, end_date)\n\n currency_price_data = CurrencyPrice.objects.filter(currency__symbol__in=symbols,\n date__gte=date_index[0],\n date__lte=date_index[-1]).values_list('date', 'currency__symbol', 'ask_price', 'bid_price')\n try:\n forex_data_array = np.core.records.fromrecords(currency_price_data, names=['date', 'symbol', 'ask_price', 'bid_price'])\n except IndexError:\n forex_data_array = np.core.records.fromrecords([(date(1900, 1, 1), \"\", 0, 0)], names=['date', 'symbol', 'ask_price', 'bid_price'])\n df = DataFrame.from_records(forex_data_array, index='date')\n df['date'] = df.index\n\n if price_type == \"mid\":\n df['price'] = (df['ask_price'] + df['bid_price']) / 2\n elif price_type == \"ask\":\n df['price'] = df['ask_price']\n elif price_type == \"bid\":\n df['price'] = df['bid_price']\n else:\n raise ValueError(\"Incorrect price_type (%s) must be on of 'ask', 'bid' or 'mid'\" % str(price_type))\n\n df = df.pivot(index='date', columns='symbol', values='price')\n df = df.reindex(date_index)\n df = df.fillna(method=\"ffill\")\n unlisted_symbols = list(set(symbols) - set(df.columns))\n for unlisted_symbol in unlisted_symbols:\n df[unlisted_symbol] = np.nan\n df = df[symbols]\n\n return df"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef save(self, *args, **kwargs):\n if self.ask_price < 0:\n raise ValidationError(\"Ask price must be greater than zero\")\n if self.bid_price < 0:\n raise ValidationError(\"Bid price must be greater than zero\")\n if self.ask_price < self.bid_price:\n raise ValidationError(\"Ask price must be at least Bid price\")\n\n super(CurrencyPrice, self).save(*args, **kwargs)", "response": "Save the currency price."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the given stream s encoding or a default.", "response": "def get_stream_enc(stream, default=None):\n \"\"\"Return the given stream's encoding or a default.\n\n There are cases where sys.std* might not actually be a stream, so\n check for the encoding attribute prior to returning it, and return\n a default if it doesn't exist or evaluates as False. `default'\n is None if not provided.\n \"\"\"\n if not hasattr(stream, 'encoding') or not stream.encoding:\n return default\n else:\n return stream.encoding"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the default encoding for bytes as text.", "response": "def getdefaultencoding():\n \"\"\"Return IPython's guess for the default encoding for bytes as text.\n\n Asks for stdin.encoding first, to match the calling Terminal, but that\n is often None for subprocesses. Fall back on locale.getpreferredencoding()\n which should be a sensible platform default (that respects LANG environment),\n and finally to sys.getdefaultencoding() which is the most conservative option,\n and usually ASCII.\n \"\"\"\n enc = get_stream_enc(sys.stdin)\n if not enc or enc=='ascii':\n try:\n # There are reports of getpreferredencoding raising errors\n # in some cases, which may well be fixed, but let's be conservative here.\n enc = locale.getpreferredencoding()\n except Exception:\n pass\n return enc or sys.getdefaultencoding()"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the username and password from config.", "response": "def get_userpass_value(cli_value, config, key, prompt_strategy):\n \"\"\"Gets the username / password from config.\n\n Uses the following rules:\n\n 1. If it is specified on the cli (`cli_value`), use that.\n 2. If `config[key]` is specified, use that.\n 3. Otherwise prompt using `prompt_strategy`.\n\n :param cli_value: The value supplied from the command line or `None`.\n :type cli_value: unicode or `None`\n :param config: Config dictionary\n :type config: dict\n :param key: Key to find the config value.\n :type key: unicode\n :prompt_strategy: Argumentless function to return fallback value.\n :type prompt_strategy: function\n :returns: The value for the username / password\n :rtype: unicode\n \"\"\"\n if cli_value is not None:\n return cli_value\n elif config.get(key):\n return config[key]\n else:\n return prompt_strategy()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_connection_file(self):\n try:\n fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])\n except IOError:\n self.log.debug(\"Connection file not found: %s\", self.connection_file)\n # This means I own it, so I will clean it up:\n atexit.register(self.cleanup_connection_file)\n return\n self.log.debug(u\"Loading connection file %s\", fname)\n with open(fname) as f:\n s = f.read()\n cfg = json.loads(s)\n if self.ip == LOCALHOST and 'ip' in cfg:\n # not overridden by config or cl_args\n self.ip = cfg['ip']\n for channel in ('hb', 'shell', 'iopub', 'stdin'):\n name = channel + '_port'\n if getattr(self, name) == 0 and name in cfg:\n # not overridden by config or cl_args\n setattr(self, name, cfg[name])\n if 'key' in cfg:\n self.config.Session.key = str_to_bytes(cfg['key'])", "response": "load ip port and hmac config from JSON connection file"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite connection info to JSON file", "response": "def write_connection_file(self):\n \"\"\"write connection info to JSON file\"\"\"\n if os.path.basename(self.connection_file) == self.connection_file:\n cf = os.path.join(self.profile_dir.security_dir, self.connection_file)\n else:\n cf = self.connection_file\n write_connection_file(cf, ip=self.ip, key=self.session.key,\n shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,\n iopub_port=self.iopub_port)\n \n self._full_connection_file = cf"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstarting the heart beating", "response": "def init_heartbeat(self):\n \"\"\"start the heart beating\"\"\"\n # heartbeat doesn't share context, because it mustn't be blocked\n # by the GIL, which is accessed by libzmq when freeing zero-copy messages\n hb_ctx = zmq.Context()\n self.heartbeat = Heartbeat(hb_ctx, (self.ip, self.hb_port))\n self.hb_port = self.heartbeat.port\n self.log.debug(\"Heartbeat REP Channel on port: %i\"%self.hb_port)\n self.heartbeat.start()\n\n # Helper to make it easier to connect to an existing kernel.\n # set log-level to critical, to make sure it is output\n self.log.critical(\"To connect another client to this kernel, use:\")"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef log_connection_info(self):\n basename = os.path.basename(self.connection_file)\n if basename == self.connection_file or \\\n os.path.dirname(self.connection_file) == self.profile_dir.security_dir:\n # use shortname\n tail = basename\n if self.profile != 'default':\n tail += \" --profile %s\" % self.profile\n else:\n tail = self.connection_file\n self.log.critical(\"--existing %s\", tail)\n\n\n self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,\n stdin=self.stdin_port, hb=self.hb_port)", "response": "display connection info and store ports"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates our session object", "response": "def init_session(self):\n \"\"\"create our session object\"\"\"\n default_secure(self.config)\n self.session = Session(config=self.config, username=u'kernel')"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef init_blackhole(self):\n if self.no_stdout or self.no_stderr:\n blackhole = open(os.devnull, 'w')\n if self.no_stdout:\n sys.stdout = sys.__stdout__ = blackhole\n if self.no_stderr:\n sys.stderr = sys.__stderr__ = blackhole", "response": "redirects stdout and stderr to devnull if necessary"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef init_io(self):\n if self.outstream_class:\n outstream_factory = import_item(str(self.outstream_class))\n sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')\n sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')\n if self.displayhook_class:\n displayhook_factory = import_item(str(self.displayhook_class))\n sys.displayhook = displayhook_factory(self.session, self.iopub_socket)", "response": "Redirect input streams and set a display hook."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating the Kernel object itself", "response": "def init_kernel(self):\n \"\"\"Create the Kernel object itself\"\"\"\n kernel_factory = import_item(str(self.kernel_class))\n self.kernel = kernel_factory(config=self.config, session=self.session,\n shell_socket=self.shell_socket,\n iopub_socket=self.iopub_socket,\n stdin_socket=self.stdin_socket,\n log=self.log\n )\n self.kernel.record_ports(self.ports)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef init_connector(self):\n self.using_ssh = bool(self.sshkey or self.sshserver)\n\n if self.sshkey and not self.sshserver:\n # We are using ssh directly to the controller, tunneling localhost to localhost\n self.sshserver = self.url.split('://')[1].split(':')[0]\n\n if self.using_ssh:\n if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko):\n password=False\n else:\n password = getpass(\"SSH Password for %s: \"%self.sshserver)\n else:\n password = False\n\n def connect(s, url):\n url = disambiguate_url(url, self.location)\n if self.using_ssh:\n self.log.debug(\"Tunneling connection to %s via %s\"%(url, self.sshserver))\n return tunnel.tunnel_connection(s, url, self.sshserver,\n keyfile=self.sshkey, paramiko=self.paramiko,\n password=password,\n )\n else:\n return s.connect(url)\n\n def maybe_tunnel(url):\n \"\"\"like connect, but don't complete the connection (for use by heartbeat)\"\"\"\n url = disambiguate_url(url, self.location)\n if self.using_ssh:\n self.log.debug(\"Tunneling connection to %s via %s\"%(url, self.sshserver))\n url,tunnelobj = tunnel.open_tunnel(url, self.sshserver,\n keyfile=self.sshkey, paramiko=self.paramiko,\n password=password,\n )\n return url\n return connect, maybe_tunnel", "response": "construct connection function which handles tunnels"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsend the registration request", "response": "def register(self):\n \"\"\"send the registration_request\"\"\"\n\n self.log.info(\"Registering with controller at %s\"%self.url)\n ctx = self.context\n connect,maybe_tunnel = self.init_connector()\n reg = ctx.socket(zmq.DEALER)\n reg.setsockopt(zmq.IDENTITY, self.bident)\n connect(reg, self.url)\n self.registrar = zmqstream.ZMQStream(reg, self.loop)\n\n\n content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident)\n self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel))\n # print (self.session.key)\n self.session.send(self.registrar, \"registration_request\",content=content)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert html content to plain text", "response": "def html_to_text(content):\n \"\"\" Converts html content to plain text \"\"\"\n text = None\n h2t = html2text.HTML2Text()\n h2t.ignore_links = False\n text = h2t.handle(content)\n return text"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef md_to_text(content):\n text = None\n html = markdown.markdown(content)\n if html:\n text = html_to_text(content)\n return text", "response": "Converts markdown content to text"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts uri parts to valid uri.", "response": "def parts_to_uri(base_uri, uri_parts):\n \"\"\"\n Converts uri parts to valid uri.\n Example: /memebers, ['profile', 'view'] => /memembers/profile/view\n \"\"\"\n uri = \"/\".join(map(lambda x: str(x).rstrip('/'), [base_uri] + uri_parts))\n return uri"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef domain_to_fqdn(domain, proto=None):\n from .generic import get_site_proto\n proto = proto or get_site_proto()\n fdqn = '{proto}://{domain}'.format(proto=proto, domain=domain)\n return fdqn", "response": "returns a fully qualified app domain name"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndefines the command line options for the plugin.", "response": "def options(self, parser, env=os.environ):\n \"\"\"Define the command line options for the plugin.\"\"\"\n super(NoseExclude, self).options(parser, env)\n env_dirs = []\n if 'NOSE_EXCLUDE_DIRS' in env:\n exclude_dirs = env.get('NOSE_EXCLUDE_DIRS','')\n env_dirs.extend(exclude_dirs.split(';'))\n parser.add_option(\n \"--exclude-dir\", action=\"append\",\n dest=\"exclude_dirs\",\n default=env_dirs,\n help=\"Directory to exclude from test discovery. \\\n Path can be relative to current working directory \\\n or an absolute path. May be specified multiple \\\n times. [NOSE_EXCLUDE_DIRS]\")\n\n parser.add_option(\n \"--exclude-dir-file\", type=\"string\",\n dest=\"exclude_dir_file\",\n default=env.get('NOSE_EXCLUDE_DIRS_FILE', False),\n help=\"A file containing a list of directories to exclude \\\n from test discovery. Paths can be relative to current \\\n working directory or an absolute path. \\\n [NOSE_EXCLUDE_DIRS_FILE]\")"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconfigure plugin based on command line options", "response": "def configure(self, options, conf):\n \"\"\"Configure plugin based on command line options\"\"\"\n super(NoseExclude, self).configure(options, conf)\n\n self.exclude_dirs = {}\n\n # preload directories from file\n if options.exclude_dir_file:\n if not options.exclude_dirs:\n options.exclude_dirs = []\n\n new_dirs = self._load_from_file(options.exclude_dir_file)\n options.exclude_dirs.extend(new_dirs)\n\n if not options.exclude_dirs:\n self.enabled = False\n return\n\n self.enabled = True\n root = os.getcwd()\n log.debug('cwd: %s' % root)\n\n # Normalize excluded directory names for lookup\n for exclude_param in options.exclude_dirs:\n # when using setup.cfg, you can specify only one 'exclude-dir'\n # separated by some character (new line is good enough)\n for d in exclude_param.split('\\n'):\n d = d.strip()\n abs_d = self._force_to_abspath(d)\n if abs_d:\n self.exclude_dirs[abs_d] = True\n\n exclude_str = \"excluding dirs: %s\" % \",\".join(self.exclude_dirs.keys())\n log.debug(exclude_str)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if a directory is eligible for test discovery", "response": "def wantDirectory(self, dirname):\n \"\"\"Check if directory is eligible for test discovery\"\"\"\n if dirname in self.exclude_dirs:\n log.debug(\"excluded: %s\" % dirname)\n return False\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn true if ext links to a dynamic lib in the same package", "response": "def links_to_dynamic(self, ext):\n \"\"\"Return true if 'ext' links to a dynamic lib in the same package\"\"\"\n # XXX this should check to ensure the lib is actually being built\n # XXX as dynamic, and not just using a locally-found version or a\n # XXX static-compiled version\n libnames = dict.fromkeys([lib._full_name for lib in self.shlibs])\n pkg = '.'.join(ext._full_name.split('.')[:-1]+[''])\n for libname in ext.libraries:\n if pkg+libname in libnames: return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef call_each(funcs: list, *args, **kwargs):\n '''\n call each func from func list.\n\n return the last func value or None if func list is empty.\n '''\n ret = None\n for func in funcs:\n ret = func(*args, **kwargs)\n return ret", "response": "call each func from func list.\n return the last func value or None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalling each func from reversed func list. return the last value or None", "response": "def call_each_reversed(funcs: list, *args, **kwargs):\n '''\n call each func from reversed func list.\n\n return the last func value or None if func list is empty.\n '''\n ret = None\n for func in reversed(funcs):\n ret = func(*args, **kwargs)\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef append_func(self, func, *args, **kwargs):\n '''\n append func with given arguments and keywords.\n '''\n wraped_func = partial(func, *args, **kwargs)\n self.append(wraped_func)", "response": "append func with given arguments and keywords.\n append func with given arguments and keywords.\n "} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nformats the usage message", "response": "def format_usage(self, usage):\n \"\"\"\n ensure there is only one newline between usage and the first heading\n if there is no description\n \"\"\"\n msg = 'Usage: %s' % usage\n if self.parser.description:\n msg += '\\n'\n return msg"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef write_pid_file(self, overwrite=False):\n pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')\n if os.path.isfile(pid_file):\n pid = self.get_pid_from_file()\n if not overwrite:\n raise PIDFileError(\n 'The pid file [%s] already exists. \\nThis could mean that this '\n 'server is already running with [pid=%s].' % (pid_file, pid)\n )\n with open(pid_file, 'w') as f:\n self.log.info(\"Creating pid file: %s\" % pid_file)\n f.write(repr(os.getpid())+'\\n')", "response": "Create a. pid file in the pid_dir with my pid."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nremoves the pid file.", "response": "def remove_pid_file(self):\n \"\"\"Remove the pid file.\n\n This should be called at shutdown by registering a callback with\n :func:`reactor.addSystemEventTrigger`. This needs to return\n ``None``.\n \"\"\"\n pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')\n if os.path.isfile(pid_file):\n try:\n self.log.info(\"Removing pid file: %s\" % pid_file)\n os.remove(pid_file)\n except:\n self.log.warn(\"Error removing the pid file: %s\" % pid_file)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_pid_from_file(self):\n pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')\n if os.path.isfile(pid_file):\n with open(pid_file, 'r') as f:\n s = f.read().strip()\n try:\n pid = int(s)\n except:\n raise PIDFileError(\"invalid pid file: %s (contents: %r)\"%(pid_file, s))\n return pid\n else:\n raise PIDFileError('pid file not found: %s' % pid_file)", "response": "Get the pid from the pid file."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconstruct an argument parser using the function decorations.", "response": "def construct_parser(magic_func):\n \"\"\" Construct an argument parser using the function decorations.\n \"\"\"\n kwds = getattr(magic_func, 'argcmd_kwds', {})\n if 'description' not in kwds:\n kwds['description'] = getattr(magic_func, '__doc__', None)\n arg_name = real_name(magic_func)\n parser = MagicArgumentParser(arg_name, **kwds)\n # Reverse the list of decorators in order to apply them in the\n # order in which they appear in the source.\n group = None\n for deco in magic_func.decorators[::-1]:\n result = deco.add_to_parser(parser, group)\n if result is not None:\n group = result\n\n # Replace the starting 'usage: ' with IPython's %.\n help_text = parser.format_help()\n if help_text.startswith('usage: '):\n help_text = help_text.replace('usage: ', '%', 1)\n else:\n help_text = '%' + help_text\n\n # Replace the magic function's docstring with the full help text.\n magic_func.__doc__ = help_text\n\n return parser"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfind the real name of the magic.", "response": "def real_name(magic_func):\n \"\"\" Find the real name of the magic.\n \"\"\"\n magic_name = magic_func.__name__\n if magic_name.startswith('magic_'):\n magic_name = magic_name[len('magic_'):]\n return getattr(magic_func, 'argcmd_name', magic_name)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_to_parser(self, parser, group):\n if group is not None:\n parser = group\n parser.add_argument(*self.args, **self.kwds)\n return None", "response": "Add this object s information to the parser."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_to_parser(self, parser, group):\n return parser.add_argument_group(*self.args, **self.kwds)", "response": "Add this object s information to the parser."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nhighlights a block of text. Reimplemented to highlight selectively.", "response": "def highlightBlock(self, string):\n \"\"\" Highlight a block of text. Reimplemented to highlight selectively.\n \"\"\"\n if not self.highlighting_on:\n return\n\n # The input to this function is a unicode string that may contain\n # paragraph break characters, non-breaking spaces, etc. Here we acquire\n # the string as plain text so we can compare it.\n current_block = self.currentBlock()\n string = self._frontend._get_block_plain_text(current_block)\n\n # Decide whether to check for the regular or continuation prompt.\n if current_block.contains(self._frontend._prompt_pos):\n prompt = self._frontend._prompt\n else:\n prompt = self._frontend._continuation_prompt\n\n # Only highlight if we can identify a prompt, but make sure not to\n # highlight the prompt.\n if string.startswith(prompt):\n self._current_offset = len(prompt)\n string = string[len(prompt):]\n super(FrontendHighlighter, self).highlightBlock(string)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef rehighlightBlock(self, block):\n old = self.highlighting_on\n self.highlighting_on = True\n super(FrontendHighlighter, self).rehighlightBlock(block)\n self.highlighting_on = old", "response": "Reimplemented to temporarily enable highlighting if disabled."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef setFormat(self, start, count, format):\n start += self._current_offset\n super(FrontendHighlighter, self).setFormat(start, count, format)", "response": "Set the format of the current node."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef copy(self):\n if self._page_control is not None and self._page_control.hasFocus():\n self._page_control.copy()\n elif self._control.hasFocus():\n text = self._control.textCursor().selection().toPlainText()\n if text:\n lines = map(self._transform_prompt, text.splitlines())\n text = '\\n'.join(lines)\n QtGui.QApplication.clipboard().setText(text)\n else:\n self.log.debug(\"frontend widget : unknown copy target\")", "response": "Copies the currently selected text to the clipboard removing prompts."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _is_complete(self, source, interactive):\n complete = self._input_splitter.push(source)\n if interactive:\n complete = not self._input_splitter.push_accepts_more()\n return complete", "response": "Returns whether the source can be completely processed and a new\n prompt created."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nexecuting the source message.", "response": "def _execute(self, source, hidden):\n \"\"\" Execute 'source'. If 'hidden', do not show any output.\n\n See parent class :meth:`execute` docstring for full details.\n \"\"\"\n msg_id = self.kernel_manager.shell_channel.execute(source, hidden)\n self._request_info['execute'][msg_id] = self._ExecutionRequest(msg_id, 'user')\n self._hidden = hidden\n if not hidden:\n self.executing.emit(source)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _prompt_finished_hook(self):\n # Flush all state from the input splitter so the next round of\n # reading input starts with a clean buffer.\n self._input_splitter.reset()\n\n if not self._reading:\n self._highlighter.highlighting_on = False", "response": "Called immediately after a prompt is finished."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _tab_pressed(self):\n # Perform tab completion if:\n # 1) The cursor is in the input buffer.\n # 2) There is a non-whitespace character before the cursor.\n text = self._get_input_buffer_cursor_line()\n if text is None:\n return False\n complete = bool(text[:self._get_input_buffer_cursor_column()].strip())\n if complete:\n self._complete()\n return not complete", "response": "Called when the tab key is pressed. Returns whether to continue\n processing the event."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _event_filter_console_keypress(self, event):\n key = event.key()\n if self._control_key_down(event.modifiers(), include_command=False):\n\n if key == QtCore.Qt.Key_C and self._executing:\n self.request_interrupt_kernel()\n return True\n\n elif key == QtCore.Qt.Key_Period:\n self.request_restart_kernel()\n return True\n\n elif not event.modifiers() & QtCore.Qt.AltModifier:\n\n # Smart backspace: remove four characters in one backspace if:\n # 1) everything left of the cursor is whitespace\n # 2) the four characters immediately left of the cursor are spaces\n if key == QtCore.Qt.Key_Backspace:\n col = self._get_input_buffer_cursor_column()\n cursor = self._control.textCursor()\n if col > 3 and not cursor.hasSelection():\n text = self._get_input_buffer_cursor_line()[:col]\n if text.endswith(' ') and not text.strip():\n cursor.movePosition(QtGui.QTextCursor.Left,\n QtGui.QTextCursor.KeepAnchor, 4)\n cursor.removeSelectedText()\n return True\n\n return super(FrontendWidget, self)._event_filter_console_keypress(event)", "response": "Reimplemented for execution interruption and smart backspace."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nhandling replies for tab completion.", "response": "def _handle_complete_reply(self, rep):\n \"\"\" Handle replies for tab completion.\n \"\"\"\n self.log.debug(\"complete: %s\", rep.get('content', ''))\n cursor = self._get_cursor()\n info = self._request_info.get('complete')\n if info and info.id == rep['parent_header']['msg_id'] and \\\n info.pos == cursor.position():\n text = '.'.join(self._get_context())\n cursor.movePosition(QtGui.QTextCursor.Left, n=len(text))\n self._complete_with_items(cursor, rep['content']['matches'])"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _silent_exec_callback(self, expr, callback):\n\n # generate uuid, which would be used as an indication of whether or\n # not the unique request originated from here (can use msg id ?)\n local_uuid = str(uuid.uuid1())\n msg_id = self.kernel_manager.shell_channel.execute('',\n silent=True, user_expressions={ local_uuid:expr })\n self._callback_dict[local_uuid] = callback\n self._request_info['execute'][msg_id] = self._ExecutionRequest(msg_id, 'silent_exec_callback')", "response": "Silently execute expr in the kernel and call callback with reply\n "} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nexecuting callback corresponding to msg", "response": "def _handle_exec_callback(self, msg):\n \"\"\"Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback``\n\n Parameters\n ----------\n msg : raw message send by the kernel containing an `user_expressions`\n and having a 'silent_exec_callback' kind.\n\n Notes\n -----\n This function will look for a `callback` associated with the\n corresponding message id. Association has been made by\n `_silent_exec_callback`. `callback` is then called with the `repr()`\n of the value of corresponding `user_expressions` as argument.\n `callback` is then removed from the known list so that any message\n coming again with the same id won't trigger it.\n\n \"\"\"\n\n user_exp = msg['content'].get('user_expressions')\n if not user_exp:\n return\n for expression in user_exp:\n if expression in self._callback_dict:\n self._callback_dict.pop(expression)(user_exp[expression])"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nhandling the reply from the execute command.", "response": "def _handle_execute_reply(self, msg):\n \"\"\" Handles replies for code execution.\n \"\"\"\n self.log.debug(\"execute: %s\", msg.get('content', ''))\n msg_id = msg['parent_header']['msg_id']\n info = self._request_info['execute'].get(msg_id)\n # unset reading flag, because if execute finished, raw_input can't\n # still be pending.\n self._reading = False\n if info and info.kind == 'user' and not self._hidden:\n # Make sure that all output from the SUB channel has been processed\n # before writing a new prompt.\n self.kernel_manager.sub_channel.flush()\n\n # Reset the ANSI style information to prevent bad text in stdout\n # from messing up our colors. We're not a true terminal so we're\n # allowed to do this.\n if self.ansi_codes:\n self._ansi_processor.reset_sgr()\n\n content = msg['content']\n status = content['status']\n if status == 'ok':\n self._process_execute_ok(msg)\n elif status == 'error':\n self._process_execute_error(msg)\n elif status == 'aborted':\n self._process_execute_abort(msg)\n\n self._show_interpreter_prompt_for_reply(msg)\n self.executed.emit(msg)\n self._request_info['execute'].pop(msg_id)\n elif info and info.kind == 'silent_exec_callback' and not self._hidden:\n self._handle_exec_callback(msg)\n self._request_info['execute'].pop(msg_id)\n else:\n super(FrontendWidget, self)._handle_execute_reply(msg)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _handle_input_request(self, msg):\n self.log.debug(\"input: %s\", msg.get('content', ''))\n if self._hidden:\n raise RuntimeError('Request for raw input during hidden execution.')\n\n # Make sure that all output from the SUB channel has been processed\n # before entering readline mode.\n self.kernel_manager.sub_channel.flush()\n\n def callback(line):\n self.kernel_manager.stdin_channel.input(line)\n if self._reading:\n self.log.debug(\"Got second input request, assuming first was interrupted.\")\n self._reading = False\n self._readline(msg['content']['prompt'], callback=callback)", "response": "Handle incoming raw_input request."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nhandle the kernel s death by asking if the user wants to restart the kernel.", "response": "def _handle_kernel_died(self, since_last_heartbeat):\n \"\"\" Handle the kernel's death by asking if the user wants to restart.\n \"\"\"\n self.log.debug(\"kernel died: %s\", since_last_heartbeat)\n if self.custom_restart:\n self.custom_restart_kernel_died.emit(since_last_heartbeat)\n else:\n message = 'The kernel heartbeat has been inactive for %.2f ' \\\n 'seconds. Do you want to restart the kernel? You may ' \\\n 'first want to check the network connection.' % \\\n since_last_heartbeat\n self.restart_kernel(message, now=True)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nhandling the reply for object info.", "response": "def _handle_object_info_reply(self, rep):\n \"\"\" Handle replies for call tips.\n \"\"\"\n self.log.debug(\"oinfo: %s\", rep.get('content', ''))\n cursor = self._get_cursor()\n info = self._request_info.get('call_tip')\n if info and info.id == rep['parent_header']['msg_id'] and \\\n info.pos == cursor.position():\n # Get the information for a call tip. For now we format the call\n # line as string, later we can pass False to format_call and\n # syntax-highlight it ourselves for nicer formatting in the\n # calltip.\n content = rep['content']\n # if this is from pykernel, 'docstring' will be the only key\n if content.get('ismagic', False):\n # Don't generate a call-tip for magics. Ideally, we should\n # generate a tooltip, but not on ( like we do for actual\n # callables.\n call_info, doc = None, None\n else:\n call_info, doc = call_tip(content, format_call=True)\n if call_info or doc:\n self._call_tip_widget.show_call_info(call_info, doc)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhandling display hook output.", "response": "def _handle_pyout(self, msg):\n \"\"\" Handle display hook output.\n \"\"\"\n self.log.debug(\"pyout: %s\", msg.get('content', ''))\n if not self._hidden and self._is_from_this_session(msg):\n text = msg['content']['data']\n self._append_plain_text(text + '\\n', before_prompt=True)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _handle_stream(self, msg):\n self.log.debug(\"stream: %s\", msg.get('content', ''))\n if not self._hidden and self._is_from_this_session(msg):\n # Most consoles treat tabs as being 8 space characters. Convert tabs\n # to spaces so that output looks as expected regardless of this\n # widget's tab width.\n text = msg['content']['data'].expandtabs(8)\n\n self._append_plain_text(text, before_prompt=True)\n self._control.moveCursor(QtGui.QTextCursor.End)", "response": "Handle the stream message."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _handle_shutdown_reply(self, msg):\n self.log.debug(\"shutdown: %s\", msg.get('content', ''))\n if not self._hidden and not self._is_from_this_session(msg):\n if self._local_kernel:\n if not msg['content']['restart']:\n self.exit_requested.emit(self)\n else:\n # we just got notified of a restart!\n time.sleep(0.25) # wait 1/4 sec to reset\n # lest the request for a new prompt\n # goes to the old kernel\n self.reset()\n else: # remote kernel, prompt on Kernel shutdown/reset\n title = self.window().windowTitle()\n if not msg['content']['restart']:\n reply = QtGui.QMessageBox.question(self, title,\n \"Kernel has been shutdown permanently. \"\n \"Close the Console?\",\n QtGui.QMessageBox.Yes,QtGui.QMessageBox.No)\n if reply == QtGui.QMessageBox.Yes:\n self.exit_requested.emit(self)\n else:\n # XXX: remove message box in favor of using the\n # clear_on_kernel_restart setting?\n reply = QtGui.QMessageBox.question(self, title,\n \"Kernel has been reset. Clear the Console?\",\n QtGui.QMessageBox.Yes,QtGui.QMessageBox.No)\n if reply == QtGui.QMessageBox.Yes:\n time.sleep(0.25) # wait 1/4 sec to reset\n # lest the request for a new prompt\n # goes to the old kernel\n self.reset()", "response": "Handle a shutdown signal from the kernel and the kernel itself."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef execute_file(self, path, hidden=False):\n self.execute('execfile(%r)' % path, hidden=hidden)", "response": "Attempts to execute a file with path."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nattempting to interrupt the running kernel.", "response": "def interrupt_kernel(self):\n \"\"\" Attempts to interrupt the running kernel.\n \n Also unsets _reading flag, to avoid runtime errors\n if raw_input is called again.\n \"\"\"\n if self.custom_interrupt:\n self._reading = False\n self.custom_interrupt_requested.emit()\n elif self.kernel_manager.has_kernel:\n self._reading = False\n self.kernel_manager.interrupt_kernel()\n else:\n self._append_plain_text('Kernel process is either remote or '\n 'unspecified. Cannot interrupt.\\n')"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef reset(self, clear=False):\n if self._executing:\n self._executing = False\n self._request_info['execute'] = {}\n self._reading = False\n self._highlighter.highlighting_on = False\n\n if self.clear_on_kernel_restart or clear:\n self._control.clear()\n self._append_plain_text(self.banner)\n else:\n self._append_plain_text(\"# restarting kernel...\")\n self._append_html(\"

\")\n # XXX: Reprinting the full banner may be too much, but once #1680 is\n # addressed, that will mitigate it.\n #self._append_plain_text(self.banner)\n # update output marker for stdout/stderr, so that startup\n # messages appear after banner:\n self._append_before_prompt_pos = self._get_cursor().position()\n self._show_interpreter_prompt()", "response": "Resets the widget to its initial state."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nattempting to restart the running kernel.", "response": "def restart_kernel(self, message, now=False):\n \"\"\" Attempts to restart the running kernel.\n \"\"\"\n # FIXME: now should be configurable via a checkbox in the dialog. Right\n # now at least the heartbeat path sets it to True and the manual restart\n # to False. But those should just be the pre-selected states of a\n # checkbox that the user could override if so desired. But I don't know\n # enough Qt to go implementing the checkbox now.\n\n if self.custom_restart:\n self.custom_restart_requested.emit()\n\n elif self.kernel_manager.has_kernel:\n # Pause the heart beat channel to prevent further warnings.\n self.kernel_manager.hb_channel.pause()\n\n # Prompt the user to restart the kernel. Un-pause the heartbeat if\n # they decline. (If they accept, the heartbeat will be un-paused\n # automatically when the kernel is restarted.)\n if self.confirm_restart:\n buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No\n result = QtGui.QMessageBox.question(self, 'Restart kernel?',\n message, buttons)\n do_restart = result == QtGui.QMessageBox.Yes\n else:\n # confirm_restart is False, so we don't need to ask user\n # anything, just do the restart\n do_restart = True\n if do_restart:\n try:\n self.kernel_manager.restart_kernel(now=now)\n except RuntimeError:\n self._append_plain_text('Kernel started externally. '\n 'Cannot restart.\\n',\n before_prompt=True\n )\n else:\n self.reset()\n else:\n self.kernel_manager.hb_channel.unpause()\n\n else:\n self._append_plain_text('Kernel process is either remote or '\n 'unspecified. Cannot restart.\\n',\n before_prompt=True\n )"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _call_tip(self):\n # Decide if it makes sense to show a call tip\n if not self.enable_calltips:\n return False\n cursor = self._get_cursor()\n cursor.movePosition(QtGui.QTextCursor.Left)\n if cursor.document().characterAt(cursor.position()) != '(':\n return False\n context = self._get_context(cursor)\n if not context:\n return False\n\n # Send the metadata request to the kernel\n name = '.'.join(context)\n msg_id = self.kernel_manager.shell_channel.object_info(name)\n pos = self._get_cursor().position()\n self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)\n return True", "response": "Shows a call tip at the current cursor location."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _complete(self):\n context = self._get_context()\n if context:\n # Send the completion request to the kernel\n msg_id = self.kernel_manager.shell_channel.complete(\n '.'.join(context), # text\n self._get_input_buffer_cursor_line(), # line\n self._get_input_buffer_cursor_column(), # cursor_pos\n self.input_buffer) # block\n pos = self._get_cursor().position()\n info = self._CompletionRequest(msg_id, pos)\n self._request_info['complete'] = info", "response": "Sends a completion request to the kernel."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the context for the specified cursor.", "response": "def _get_context(self, cursor=None):\n \"\"\" Gets the context for the specified cursor (or the current cursor\n if none is specified).\n \"\"\"\n if cursor is None:\n cursor = self._get_cursor()\n cursor.movePosition(QtGui.QTextCursor.StartOfBlock,\n QtGui.QTextCursor.KeepAnchor)\n text = cursor.selection().toPlainText()\n return self._completion_lexer.get_context(text)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _process_execute_error(self, msg):\n content = msg['content']\n # If a SystemExit is passed along, this means exit() was called - also\n # all the ipython %exit magic syntax of '-k' to be used to keep\n # the kernel running\n if content['ename']=='SystemExit':\n keepkernel = content['evalue']=='-k' or content['evalue']=='True'\n self._keep_kernel_on_exit = keepkernel\n self.exit_requested.emit(self)\n else:\n traceback = ''.join(content['traceback'])\n self._append_plain_text(traceback)", "response": "Process a reply to an execute error."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _process_execute_ok(self, msg):\n payload = msg['content']['payload']\n for item in payload:\n if not self._process_execute_payload(item):\n warning = 'Warning: received unknown payload of type %s'\n print(warning % repr(item['source']))", "response": "Process a reply for a successful execution request."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall whenever the document s content changes.", "response": "def _document_contents_change(self, position, removed, added):\n \"\"\" Called whenever the document's content changes. Display a call tip\n if appropriate.\n \"\"\"\n # Calculate where the cursor should be *after* the change:\n position += added\n\n document = self._control.document()\n if position == self._get_cursor().position():\n self._call_tip()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a plugin to my list of plugins to call if it has the attribute I m bound to.", "response": "def addPlugin(self, plugin, call):\n \"\"\"Add plugin to my list of plugins to call, if it has the attribute\n I'm bound to.\n \"\"\"\n meth = getattr(plugin, call, None)\n if meth is not None:\n if call == 'loadTestsFromModule' and \\\n len(inspect.getargspec(meth)[0]) == 2:\n orig_meth = meth\n meth = lambda module, path, **kwargs: orig_meth(module)\n self.plugins.append((plugin, meth))"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalls plugins in a chain where the result of each plugin call is sent to the next plugin as input.", "response": "def chain(self, *arg, **kw):\n \"\"\"Call plugins in a chain, where the result of each plugin call is\n sent to the next plugin as input. The final output result is returned.\n \"\"\"\n result = None\n # extract the static arguments (if any) from arg so they can\n # be passed to each plugin call in the chain\n static = [a for (static, a)\n in zip(getattr(self.method, 'static_args', []), arg)\n if static]\n for p, meth in self.plugins:\n result = meth(*arg, **kw)\n arg = static[:]\n arg.append(result)\n return result"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating all non - None items in each non - None result.", "response": "def generate(self, *arg, **kw):\n \"\"\"Call all plugins, yielding each item in each non-None result.\n \"\"\"\n for p, meth in self.plugins:\n result = None\n try:\n result = meth(*arg, **kw)\n if result is not None:\n for r in result:\n yield r\n except (KeyboardInterrupt, SystemExit):\n raise\n except:\n exc = sys.exc_info()\n yield Failure(*exc)\n continue"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalls all plugins and return the first non - None result.", "response": "def simple(self, *arg, **kw):\n \"\"\"Call all plugins, returning the first non-None result.\n \"\"\"\n for p, meth in self.plugins:\n result = meth(*arg, **kw)\n if result is not None:\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds plugins to the manager.", "response": "def addPlugins(self, plugins=(), extraplugins=()):\n \"\"\"extraplugins are maintained in a separate list and\n re-added by loadPlugins() to prevent their being overwritten\n by plugins added by a subclass of PluginManager\n \"\"\"\n self._extraplugins = extraplugins\n for plug in iterchain(plugins, extraplugins):\n self.addPlugin(plug)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconfigure the set of plugins with the given options and config instance.", "response": "def configure(self, options, config):\n \"\"\"Configure the set of plugins with the given options\n and config instance. After configuration, disabled plugins\n are removed from the plugins list.\n \"\"\"\n log.debug(\"Configuring plugins\")\n self.config = config\n cfg = PluginProxy('configure', self._plugins)\n cfg(options, config)\n enabled = [plug for plug in self._plugins if plug.enabled]\n self.plugins = enabled\n self.sort()\n log.debug(\"Plugins enabled: %s\", enabled)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef loadPlugins(self):\n from pkg_resources import iter_entry_points\n loaded = {}\n for entry_point, adapt in self.entry_points:\n for ep in iter_entry_points(entry_point):\n if ep.name in loaded:\n continue\n loaded[ep.name] = True\n log.debug('%s load plugin %s', self.__class__.__name__, ep)\n try:\n plugcls = ep.load()\n except KeyboardInterrupt:\n raise\n except Exception, e:\n # never want a plugin load to kill the test run\n # but we can't log here because the logger is not yet\n # configured\n warn(\"Unable to load plugin %s: %s\" % (ep, e),\n RuntimeWarning)\n continue\n if adapt:\n plug = adapt(plugcls())\n else:\n plug = plugcls()\n self.addPlugin(plug)\n super(EntryPointPluginManager, self).loadPlugins()", "response": "Load plugins by iterating the entry point."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef loadPlugins(self):\n from nose.plugins import builtin\n for plug in builtin.plugins:\n self.addPlugin(plug())\n super(BuiltinPluginManager, self).loadPlugins()", "response": "Load plugins in nose. plugins. builtin\n "} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nrenders a LaTeX string to PNG.", "response": "def latex_to_png(s, encode=False, backend='mpl'):\n \"\"\"Render a LaTeX string to PNG.\n\n Parameters\n ----------\n s : str\n The raw string containing valid inline LaTeX.\n encode : bool, optional\n Should the PNG data bebase64 encoded to make it JSON'able.\n backend : {mpl, dvipng}\n Backend for producing PNG data.\n\n None is returned when the backend cannot be used.\n\n \"\"\"\n if backend == 'mpl':\n f = latex_to_png_mpl\n elif backend == 'dvipng':\n f = latex_to_png_dvipng\n else:\n raise ValueError('No such backend {0}'.format(backend))\n bin_data = f(s)\n if encode and bin_data:\n bin_data = encodestring(bin_data)\n return bin_data"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef latex_to_html(s, alt='image'):\n base64_data = latex_to_png(s, encode=True)\n if base64_data:\n return _data_uri_template_png % (base64_data, alt)", "response": "Render LaTeX to HTML with embedded PNG data using data URIs."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrendering a math expression into a closely - clipped bounding box and saves it to an image file.", "response": "def math_to_image(s, filename_or_obj, prop=None, dpi=None, format=None):\n \"\"\"\n Given a math expression, renders it in a closely-clipped bounding\n box to an image file.\n\n *s*\n A math expression. The math portion should be enclosed in\n dollar signs.\n\n *filename_or_obj*\n A filepath or writable file-like object to write the image data\n to.\n\n *prop*\n If provided, a FontProperties() object describing the size and\n style of the text.\n\n *dpi*\n Override the output dpi, otherwise use the default associated\n with the output format.\n\n *format*\n The output format, eg. 'svg', 'pdf', 'ps' or 'png'. If not\n provided, will be deduced from the filename.\n \"\"\"\n from matplotlib import figure\n # backend_agg supports all of the core output formats\n from matplotlib.backends import backend_agg\n from matplotlib.font_manager import FontProperties\n from matplotlib.mathtext import MathTextParser\n\n if prop is None:\n prop = FontProperties()\n\n parser = MathTextParser('path')\n width, height, depth, _, _ = parser.parse(s, dpi=72, prop=prop)\n\n fig = figure.Figure(figsize=(width / 72.0, height / 72.0))\n fig.text(0, depth/height, s, fontproperties=prop)\n backend_agg.FigureCanvasAgg(fig)\n fig.savefig(filename_or_obj, dpi=dpi, format=format)\n\n return depth"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses an editable requirement into a requirement and a URL", "response": "def parse_editable(editable_req, default_vcs=None):\n \"\"\"Parses svn+http://blahblah@rev#egg=Foobar into a requirement\n (Foobar) and a URL\"\"\"\n\n url = editable_req\n extras = None\n\n # If a file path is specified with extras, strip off the extras.\n m = re.match(r'^(.+)(\\[[^\\]]+\\])$', url)\n if m:\n url_no_extras = m.group(1)\n extras = m.group(2)\n else:\n url_no_extras = url\n\n if os.path.isdir(url_no_extras):\n if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):\n raise InstallationError(\"Directory %r is not installable. File 'setup.py' not found.\", url_no_extras)\n # Treating it as code that has already been checked out\n url_no_extras = path_to_url(url_no_extras)\n\n if url_no_extras.lower().startswith('file:'):\n if extras:\n return None, url_no_extras, pkg_resources.Requirement.parse('__placeholder__' + extras).extras\n else:\n return None, url_no_extras, None\n\n for version_control in vcs:\n if url.lower().startswith('%s:' % version_control):\n url = '%s+%s' % (version_control, url)\n if '+' not in url:\n if default_vcs:\n url = default_vcs + '+' + url\n else:\n raise InstallationError(\n '--editable=%s should be formatted with svn+URL, git+URL, hg+URL or bzr+URL' % editable_req)\n vc_type = url.split('+', 1)[0].lower()\n if not vcs.get_backend(vc_type):\n error_message = 'For --editable=%s only ' % editable_req + \\\n ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \\\n ' is currently supported'\n raise InstallationError(error_message)\n match = re.search(r'(?:#|#.*?&)egg=([^&]*)', editable_req)\n if (not match or not match.group(1)) and vcs.get_backend(vc_type):\n parts = [p for p in editable_req.split('#', 1)[0].split('/') if p]\n if parts[-2] in ('tags', 'branches', 'tag', 'branch'):\n req = parts[-3]\n elif parts[-1] == 'trunk':\n req = parts[-2]\n else:\n raise InstallationError(\n '--editable=%s is not the right format; it must have #egg=Package'\n % editable_req)\n else:\n req = match.group(1)\n ## FIXME: use package_to_requirement?\n match = re.search(r'^(.*?)(?:-dev|-\\d.*)$', req)\n if match:\n # Strip off -dev, -0.2, etc.\n req = match.group(1)\n return req, url, None"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef check_if_exists(self):\n if self.req is None:\n return False\n try:\n self.satisfied_by = pkg_resources.get_distribution(self.req)\n except pkg_resources.DistributionNotFound:\n return False\n except pkg_resources.VersionConflict:\n existing_dist = pkg_resources.get_distribution(self.req.project_name)\n if self.use_user_site:\n if dist_in_usersite(existing_dist):\n self.conflicts_with = existing_dist\n elif running_under_virtualenv() and dist_in_site_packages(existing_dist):\n raise InstallationError(\"Will not install to the user site because it will lack sys.path precedence to %s in %s\"\n %(existing_dist.project_name, existing_dist.location))\n else:\n self.conflicts_with = existing_dist\n return True", "response": "Find an installed distribution that satisfies or conflicts\n with this requirement and set self. satisfied_by or self. conflicts_with appropriately."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cleanup_files(self, bundle=False):\n logger.notify('Cleaning up...')\n logger.indent += 2\n for req in self.reqs_to_cleanup:\n req.remove_temporary_source()\n\n remove_dir = []\n if self._pip_has_created_build_dir():\n remove_dir.append(self.build_dir)\n\n # The source dir of a bundle can always be removed.\n # FIXME: not if it pre-existed the bundle!\n if bundle:\n remove_dir.append(self.src_dir)\n\n for dir in remove_dir:\n if os.path.exists(dir):\n logger.info('Removing temporary dir %s...' % dir)\n rmtree(dir)\n\n logger.indent -= 2", "response": "Clean up files remove builds."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef install(self, install_options, global_options=()):\n to_install = [r for r in self.requirements.values()\n if not r.satisfied_by]\n\n if to_install:\n logger.notify('Installing collected packages: %s' % ', '.join([req.name for req in to_install]))\n logger.indent += 2\n try:\n for requirement in to_install:\n if requirement.conflicts_with:\n logger.notify('Found existing installation: %s'\n % requirement.conflicts_with)\n logger.indent += 2\n try:\n requirement.uninstall(auto_confirm=True)\n finally:\n logger.indent -= 2\n try:\n requirement.install(install_options, global_options)\n except:\n # if install did not succeed, rollback previous uninstall\n if requirement.conflicts_with and not requirement.install_succeeded:\n requirement.rollback_uninstall()\n raise\n else:\n if requirement.conflicts_with and requirement.install_succeeded:\n requirement.commit_uninstall()\n requirement.remove_temporary_source()\n finally:\n logger.indent -= 2\n self.successfully_installed = to_install", "response": "Install all packages in this set."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a generator yielding a Process class instance for all the processes running on the local machine.", "response": "def process_iter():\n \"\"\"Return a generator yielding a Process class instance for all\n running processes on the local machine.\n\n Every new Process instance is only created once and then cached\n into an internal table which is updated every time this is used.\n\n The sorting order in which processes are yielded is based on\n their PIDs.\n \"\"\"\n def add(pid):\n proc = Process(pid)\n _pmap[proc.pid] = proc\n return proc\n\n def remove(pid):\n _pmap.pop(pid, None)\n\n a = set(get_pid_list())\n b = set(_pmap.keys())\n new_pids = a - b\n gone_pids = b - a\n\n for pid in gone_pids:\n remove(pid)\n for pid, proc in sorted(list(_pmap.items()) + \\\n list(dict.fromkeys(new_pids).items())):\n try:\n if proc is None: # new process\n yield add(pid)\n else:\n # use is_running() to check whether PID has been reused by\n # another process in which case yield a new Process instance\n if proc.is_running():\n yield proc\n else:\n yield add(pid)\n except NoSuchProcess:\n remove(pid)\n except AccessDenied:\n # Process creation time can't be determined hence there's\n # no way to tell whether the pid of the cached process\n # has been reused. Just return the cached version.\n yield proc"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cpu_percent(interval=0.1, percpu=False):\n global _last_cpu_times\n global _last_per_cpu_times\n blocking = interval is not None and interval > 0.0\n\n def calculate(t1, t2):\n t1_all = sum(t1)\n t1_busy = t1_all - t1.idle\n\n t2_all = sum(t2)\n t2_busy = t2_all - t2.idle\n\n # this usually indicates a float precision issue\n if t2_busy <= t1_busy:\n return 0.0\n\n busy_delta = t2_busy - t1_busy\n all_delta = t2_all - t1_all\n busy_perc = (busy_delta / all_delta) * 100\n return round(busy_perc, 1)\n\n # system-wide usage\n if not percpu:\n if blocking:\n t1 = cpu_times()\n time.sleep(interval)\n else:\n t1 = _last_cpu_times\n _last_cpu_times = cpu_times()\n return calculate(t1, _last_cpu_times)\n # per-cpu usage\n else:\n ret = []\n if blocking:\n tot1 = cpu_times(percpu=True)\n time.sleep(interval)\n else:\n tot1 = _last_per_cpu_times\n _last_per_cpu_times = cpu_times(percpu=True)\n for t1, t2 in zip(tot1, _last_per_cpu_times):\n ret.append(calculate(t1, t2))\n return ret", "response": "Return a float representing the current system - wide CPU utilization as a percentage."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef disk_io_counters(perdisk=False):\n rawdict = _psplatform.disk_io_counters()\n if not rawdict:\n raise RuntimeError(\"couldn't find any physical disk\")\n if perdisk:\n for disk, fields in rawdict.items():\n rawdict[disk] = _nt_disk_iostat(*fields)\n return rawdict\n else:\n return _nt_disk_iostat(*[sum(x) for x in zip(*rawdict.values())])", "response": "Return system disk I / O statistics as a namedtuple including the following attributes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn network I/O statistics as a namedtuple including network interface names and statistics.", "response": "def network_io_counters(pernic=False):\n \"\"\"Return network I/O statistics as a namedtuple including\n the following attributes:\n\n - bytes_sent: number of bytes sent\n - bytes_recv: number of bytes received\n - packets_sent: number of packets sent\n - packets_recv: number of packets received\n - errin: total number of errors while receiving\n - errout: total number of errors while sending\n - dropin: total number of incoming packets which were dropped\n - dropout: total number of outgoing packets which were dropped\n (always 0 on OSX and BSD)\n\n If pernic is True return the same information for every\n network interface installed on the system as a dictionary\n with network interface names as the keys and the namedtuple\n described above as the values.\n \"\"\"\n rawdict = _psplatform.network_io_counters()\n if not rawdict:\n raise RuntimeError(\"couldn't find any network interface\")\n if pernic:\n for nic, fields in rawdict.items():\n rawdict[nic] = _nt_net_iostat(*fields)\n return rawdict\n else:\n return _nt_net_iostat(*[sum(x) for x in zip(*rawdict.values())])"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the amount of total used and free physical memory on the system in bytes plus the percentage usage.", "response": "def phymem_usage():\n \"\"\"Return the amount of total, used and free physical memory\n on the system in bytes plus the percentage usage.\n Deprecated by psutil.virtual_memory().\n \"\"\"\n mem = virtual_memory()\n return _nt_sysmeminfo(mem.total, mem.used, mem.free, mem.percent)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the children of this process as a list of Process objects.", "response": "def get_children(self, recursive=False):\n \"\"\"Return the children of this process as a list of Process\n objects.\n If recursive is True return all the parent descendants.\n\n Example (A == this process):\n\n A \u2500\u2510\n \u2502\n \u251c\u2500 B (child) \u2500\u2510\n \u2502 \u2514\u2500 X (grandchild) \u2500\u2510\n \u2502 \u2514\u2500 Y (great grandchild)\n \u251c\u2500 C (child)\n \u2514\u2500 D (child)\n\n >>> p.get_children()\n B, C, D\n >>> p.get_children(recursive=True)\n B, X, Y, C, D\n\n Note that in the example above if process X disappears\n process Y won't be returned either as the reference to\n process A is lost.\n \"\"\"\n if not self.is_running():\n name = self._platform_impl._process_name\n raise NoSuchProcess(self.pid, name)\n\n ret = []\n if not recursive:\n for p in process_iter():\n try:\n if p.ppid == self.pid:\n # if child happens to be older than its parent\n # (self) it means child's PID has been reused\n if self.create_time <= p.create_time:\n ret.append(p)\n except NoSuchProcess:\n pass\n else:\n # construct a dict where 'values' are all the processes\n # having 'key' as their parent\n table = defaultdict(list)\n for p in process_iter():\n try:\n table[p.ppid].append(p)\n except NoSuchProcess:\n pass\n # At this point we have a mapping table where table[self.pid]\n # are the current process's children.\n # Below, we look for all descendants recursively, similarly\n # to a recursive function call.\n checkpids = [self.pid]\n for pid in checkpids:\n for child in table[pid]:\n try:\n # if child happens to be older than its parent\n # (self) it means child's PID has been reused\n intime = self.create_time <= child.create_time\n except NoSuchProcess:\n pass\n else:\n if intime:\n ret.append(child)\n if child.pid not in checkpids:\n checkpids.append(child.pid)\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_cpu_percent(self, interval=0.1):\n blocking = interval is not None and interval > 0.0\n if blocking:\n st1 = sum(cpu_times())\n pt1 = self._platform_impl.get_cpu_times()\n time.sleep(interval)\n st2 = sum(cpu_times())\n pt2 = self._platform_impl.get_cpu_times()\n else:\n st1 = self._last_sys_cpu_times\n pt1 = self._last_proc_cpu_times\n st2 = sum(cpu_times())\n pt2 = self._platform_impl.get_cpu_times()\n if st1 is None or pt1 is None:\n self._last_sys_cpu_times = st2\n self._last_proc_cpu_times = pt2\n return 0.0\n\n delta_proc = (pt2.user - pt1.user) + (pt2.system - pt1.system)\n delta_time = st2 - st1\n # reset values for next call in case of interval == None\n self._last_sys_cpu_times = st2\n self._last_proc_cpu_times = pt2\n\n try:\n # the utilization split between all CPUs\n overall_percent = (delta_proc / delta_time) * 100\n except ZeroDivisionError:\n # interval was too low\n return 0.0\n # the utilization of a single CPU\n single_cpu_percent = overall_percent * NUM_CPUS\n # on posix a percentage > 100 is legitimate\n # http://stackoverflow.com/questions/1032357/comprehending-top-cpu-usage\n # on windows we use this ugly hack to avoid troubles with float\n # precision issues\n if os.name != 'posix':\n if single_cpu_percent > 100.0:\n return 100.0\n return round(single_cpu_percent, 1)", "response": "Return a float representing the current process CPU\n utilization as a percentage."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_memory_percent(self):\n rss = self._platform_impl.get_memory_info()[0]\n try:\n return (rss / float(TOTAL_PHYMEM)) * 100\n except ZeroDivisionError:\n return 0.0", "response": "Compare physical system memory to process resident memory and\n calculate process memory utilization as a percentage."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_memory_maps(self, grouped=True):\n it = self._platform_impl.get_memory_maps()\n if grouped:\n d = {}\n for tupl in it:\n path = tupl[2]\n nums = tupl[3:]\n try:\n d[path] = map(lambda x, y: x+y, d[path], nums)\n except KeyError:\n d[path] = nums\n nt = self._platform_impl.nt_mmap_grouped\n return [nt(path, *d[path]) for path in d]\n else:\n nt = self._platform_impl.nt_mmap_ext\n return [nt(*x) for x in it]", "response": "Return process s mapped memory regions as a list of nameduples\n whose fields are variable depending on the platform."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning whether this process is running.", "response": "def is_running(self):\n \"\"\"Return whether this process is running.\"\"\"\n if self._gone:\n return False\n try:\n # Checking if pid is alive is not enough as the pid might\n # have been reused by another process.\n # pid + creation time, on the other hand, is supposed to\n # identify a process univocally.\n return self.create_time == \\\n self._platform_impl.get_process_create_time()\n except NoSuchProcess:\n self._gone = True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsending a signal to the current process.", "response": "def send_signal(self, sig):\n \"\"\"Send a signal to process (see signal module constants).\n On Windows only SIGTERM is valid and is treated as an alias\n for kill().\n \"\"\"\n # safety measure in case the current process has been killed in\n # meantime and the kernel reused its PID\n if not self.is_running():\n name = self._platform_impl._process_name\n raise NoSuchProcess(self.pid, name)\n if os.name == 'posix':\n try:\n os.kill(self.pid, sig)\n except OSError:\n err = sys.exc_info()[1]\n name = self._platform_impl._process_name\n if err.errno == errno.ESRCH:\n raise NoSuchProcess(self.pid, name)\n if err.errno == errno.EPERM:\n raise AccessDenied(self.pid, name)\n raise\n else:\n if sig == signal.SIGTERM:\n self._platform_impl.kill_process()\n else:\n raise ValueError(\"only SIGTERM is supported on Windows\")"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef suspend(self):\n # safety measure in case the current process has been killed in\n # meantime and the kernel reused its PID\n if not self.is_running():\n name = self._platform_impl._process_name\n raise NoSuchProcess(self.pid, name)\n # windows\n if hasattr(self._platform_impl, \"suspend_process\"):\n self._platform_impl.suspend_process()\n else:\n # posix\n self.send_signal(signal.SIGSTOP)", "response": "Suspends the process execution."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nresume the process execution.", "response": "def resume(self):\n \"\"\"Resume process execution.\"\"\"\n # safety measure in case the current process has been killed in\n # meantime and the kernel reused its PID\n if not self.is_running():\n name = self._platform_impl._process_name\n raise NoSuchProcess(self.pid, name)\n # windows\n if hasattr(self._platform_impl, \"resume_process\"):\n self._platform_impl.resume_process()\n else:\n # posix\n self.send_signal(signal.SIGCONT)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nkilling the current process.", "response": "def kill(self):\n \"\"\"Kill the current process.\"\"\"\n # safety measure in case the current process has been killed in\n # meantime and the kernel reused its PID\n if not self.is_running():\n name = self._platform_impl._process_name\n raise NoSuchProcess(self.pid, name)\n if os.name == 'posix':\n self.send_signal(signal.SIGKILL)\n else:\n self._platform_impl.kill_process()"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef wait(self, timeout=None):\n if timeout is not None and not timeout >= 0:\n raise ValueError(\"timeout must be a positive integer\")\n return self._platform_impl.process_wait(timeout)", "response": "Wait for the current process to terminate and return its exit code."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets or set process niceness ( priority.", "response": "def nice(self):\n \"\"\"Get or set process niceness (priority).\n Deprecated, use get_nice() instead.\n \"\"\"\n msg = \"this property is deprecated; use Process.get_nice() method instead\"\n warnings.warn(msg, category=DeprecationWarning, stacklevel=2)\n return self.get_nice()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _wire_kernel(self):\n self.gtk_main, self.gtk_main_quit = self._hijack_gtk()\n gobject.timeout_add(int(1000*self.kernel._poll_interval),\n self.iterate_kernel)\n return False", "response": "Initializes the kernel inside GTK."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _hijack_gtk(self):\n def dummy(*args, **kw):\n pass\n # save and trap main and main_quit from gtk\n orig_main, gtk.main = gtk.main, dummy\n orig_main_quit, gtk.main_quit = gtk.main_quit, dummy\n return orig_main, orig_main_quit", "response": "Hijack a few key functions in GTK for IPython integration."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nam the given identifier defined in one of the namespaces which shadow the alias and magic namespaces?", "response": "def is_shadowed(identifier, ip):\n \"\"\"Is the given identifier defined in one of the namespaces which shadow\n the alias and magic namespaces? Note that an identifier is different\n than ifun, because it can not contain a '.' character.\"\"\"\n # This is much safer than calling ofind, which can change state\n return (identifier in ip.user_ns \\\n or identifier in ip.user_global_ns \\\n or identifier in ip.ns_table['builtin'])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef init_transformers(self):\n self._transformers = []\n for transformer_cls in _default_transformers:\n transformer_cls(\n shell=self.shell, prefilter_manager=self, config=self.config\n )", "response": "Create the default transformers."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef register_transformer(self, transformer):\n if transformer not in self._transformers:\n self._transformers.append(transformer)\n self.sort_transformers()", "response": "Register a transformer instance."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef unregister_transformer(self, transformer):\n if transformer in self._transformers:\n self._transformers.remove(transformer)", "response": "Unregister a transformer instance."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef init_checkers(self):\n self._checkers = []\n for checker in _default_checkers:\n checker(\n shell=self.shell, prefilter_manager=self, config=self.config\n )", "response": "Create the default checkers."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef register_checker(self, checker):\n if checker not in self._checkers:\n self._checkers.append(checker)\n self.sort_checkers()", "response": "Register a checker instance."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate the default handlers.", "response": "def init_handlers(self):\n \"\"\"Create the default handlers.\"\"\"\n self._handlers = {}\n self._esc_handlers = {}\n for handler in _default_handlers:\n handler(\n shell=self.shell, prefilter_manager=self, config=self.config\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef register_handler(self, name, handler, esc_strings):\n self._handlers[name] = handler\n for esc_str in esc_strings:\n self._esc_handlers[esc_str] = handler", "response": "Register a handler instance by name with esc_strings."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfind a handler for the line_info by trying checkers.", "response": "def find_handler(self, line_info):\n \"\"\"Find a handler for the line_info by trying checkers.\"\"\"\n for checker in self.checkers:\n if checker.enabled:\n handler = checker.check(line_info)\n if handler:\n return handler\n return self.get_handler_by_name('normal')"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef transform_line(self, line, continue_prompt):\n for transformer in self.transformers:\n if transformer.enabled:\n line = transformer.transform(line, continue_prompt)\n return line", "response": "Calls the transformers in order of increasing priority."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef check(self, line_info):\n \"Instances of IPyAutocall in user_ns get autocalled immediately\"\n obj = self.shell.user_ns.get(line_info.ifun, None)\n if isinstance(obj, IPyAutocall):\n obj.set_ip(self.shell)\n return self.prefilter_manager.get_handler_by_name('auto')\n else:\n return None", "response": "Instances of IPyAutocall in user_ns get autocalled immediately"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nallowing and!! in multi - line statements if multi_line_specials is on", "response": "def check(self, line_info):\n \"Allow ! and !! in multi-line statements if multi_line_specials is on\"\n # Note that this one of the only places we check the first character of\n # ifun and *not* the pre_char. Also note that the below test matches\n # both ! and !!.\n if line_info.continue_prompt \\\n and self.prefilter_manager.multi_line_specials:\n if line_info.esc == ESC_MAGIC:\n return self.prefilter_manager.get_handler_by_name('magic')\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef check(self, line_info):\n if line_info.line[-1] == ESC_HELP \\\n and line_info.esc != ESC_SHELL \\\n and line_info.esc != ESC_SH_CAP:\n # the ? can be at the end, but *not* for either kind of shell escape,\n # because a ? can be a vaild final char in a shell cmd\n return self.prefilter_manager.get_handler_by_name('help')\n else:\n if line_info.pre:\n return None\n # This returns None like it should if no handler exists\n return self.prefilter_manager.get_handler_by_esc(line_info.esc)", "response": "Check for escape character and return either a handler to handle it or None."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck if the ifun is magic and if automagic is on run it.", "response": "def check(self, line_info):\n \"\"\"If the ifun is magic, and automagic is on, run it. Note: normal,\n non-auto magic would already have been triggered via '%' in\n check_esc_chars. This just checks for automagic. Also, before\n triggering the magic handler, make sure that there is nothing in the\n user namespace which could shadow it.\"\"\"\n if not self.shell.automagic or not self.shell.find_magic(line_info.ifun):\n return None\n\n # We have a likely magic method. Make sure we should actually call it.\n if line_info.continue_prompt and not self.prefilter_manager.multi_line_specials:\n return None\n\n head = line_info.ifun.split('.',1)[0]\n if is_shadowed(head, self.shell):\n return None\n\n return self.prefilter_manager.get_handler_by_name('magic')"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef check(self, line_info):\n \"Check if the initital identifier on the line is an alias.\"\n # Note: aliases can not contain '.'\n head = line_info.ifun.split('.',1)[0]\n if line_info.ifun not in self.shell.alias_manager \\\n or head not in self.shell.alias_manager \\\n or is_shadowed(head, self.shell):\n return None\n\n return self.prefilter_manager.get_handler_by_name('alias')", "response": "Check if the initital identifier on the line is an alias."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check(self, line_info):\n if line_info.the_rest and line_info.the_rest[0] in '!=()<>,+*/%^&|':\n return self.prefilter_manager.get_handler_by_name('normal')\n else:\n return None", "response": "Check if the line starts with a function call or pretty much\n any python operator."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck if the initial word / function is callable and autocall is on.", "response": "def check(self, line_info):\n \"Check if the initial word/function is callable and autocall is on.\"\n if not self.shell.autocall:\n return None\n\n oinfo = line_info.ofind(self.shell) # This can mutate state via getattr\n if not oinfo['found']:\n return None\n\n if callable(oinfo['obj']) \\\n and (not self.exclude_regexp.match(line_info.the_rest)) \\\n and self.function_name_regexp.match(line_info.ifun):\n return self.prefilter_manager.get_handler_by_name('auto')\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef handle(self, line_info):\n # print \"normal: \", line_info\n \"\"\"Handle normal input lines. Use as a template for handlers.\"\"\"\n\n # With autoindent on, we need some way to exit the input loop, and I\n # don't want to force the user to have to backspace all the way to\n # clear the line. The rule will be in this case, that either two\n # lines of pure whitespace in a row, or a line of pure whitespace but\n # of a size different to the indent level, will exit the input loop.\n line = line_info.line\n continue_prompt = line_info.continue_prompt\n\n if (continue_prompt and\n self.shell.autoindent and\n line.isspace() and\n 0 < abs(len(line) - self.shell.indent_current_nsp) <= 2):\n line = ''\n\n return line", "response": "Handle normal input lines. Use as a template for handlers."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhandling alias input lines.", "response": "def handle(self, line_info):\n \"\"\"Handle alias input lines. \"\"\"\n transformed = self.shell.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest)\n # pre is needed, because it carries the leading whitespace. Otherwise\n # aliases won't work in indented sections.\n line_out = '%sget_ipython().system(%r)' % (line_info.pre_whitespace, transformed)\n\n return line_out"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nexecutes the line in a shell", "response": "def handle(self, line_info):\n \"\"\"Execute the line in a shell, empty return value\"\"\"\n magic_handler = self.prefilter_manager.get_handler_by_name('magic')\n\n line = line_info.line\n if line.lstrip().startswith(ESC_SH_CAP):\n # rewrite LineInfo's line, ifun and the_rest to properly hold the\n # call to %sx and the actual command to be executed, so\n # handle_magic can work correctly. Note that this works even if\n # the line is indented, so it handles multi_line_specials\n # properly.\n new_rest = line.lstrip()[2:]\n line_info.line = '%ssx %s' % (ESC_MAGIC, new_rest)\n line_info.ifun = 'sx'\n line_info.the_rest = new_rest\n return magic_handler.handle(line_info)\n else:\n cmd = line.lstrip().lstrip(ESC_SHELL)\n line_out = '%sget_ipython().system(%r)' % (line_info.pre_whitespace, cmd)\n return line_out"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nhandling lines which can be auto - executed quoting if requested.", "response": "def handle(self, line_info):\n \"\"\"Handle lines which can be auto-executed, quoting if requested.\"\"\"\n line = line_info.line\n ifun = line_info.ifun\n the_rest = line_info.the_rest\n pre = line_info.pre\n esc = line_info.esc\n continue_prompt = line_info.continue_prompt\n obj = line_info.ofind(self.shell)['obj']\n #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest) # dbg\n\n # This should only be active for single-line input!\n if continue_prompt:\n return line\n\n force_auto = isinstance(obj, IPyAutocall)\n\n # User objects sometimes raise exceptions on attribute access other\n # than AttributeError (we've seen it in the past), so it's safest to be\n # ultra-conservative here and catch all.\n try:\n auto_rewrite = obj.rewrite\n except Exception:\n auto_rewrite = True\n\n if esc == ESC_QUOTE:\n # Auto-quote splitting on whitespace\n newcmd = '%s(\"%s\")' % (ifun,'\", \"'.join(the_rest.split()) )\n elif esc == ESC_QUOTE2:\n # Auto-quote whole string\n newcmd = '%s(\"%s\")' % (ifun,the_rest)\n elif esc == ESC_PAREN:\n newcmd = '%s(%s)' % (ifun,\",\".join(the_rest.split()))\n else:\n # Auto-paren. \n if force_auto:\n # Don't rewrite if it is already a call.\n do_rewrite = not the_rest.startswith('(')\n else:\n if not the_rest:\n # We only apply it to argument-less calls if the autocall\n # parameter is set to 2.\n do_rewrite = (self.shell.autocall >= 2)\n elif the_rest.startswith('[') and hasattr(obj, '__getitem__'):\n # Don't autocall in this case: item access for an object\n # which is BOTH callable and implements __getitem__.\n do_rewrite = False\n else:\n do_rewrite = True\n\n # Figure out the rewritten command\n if do_rewrite:\n if the_rest.endswith(';'):\n newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])\n else:\n newcmd = '%s(%s)' % (ifun.rstrip(), the_rest) \n else:\n normal_handler = self.prefilter_manager.get_handler_by_name('normal')\n return normal_handler.handle(line_info)\n \n # Display the rewritten call\n if auto_rewrite:\n self.shell.auto_rewrite_input(newcmd)\n\n return newcmd"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef handle(self, line_info):\n normal_handler = self.prefilter_manager.get_handler_by_name('normal')\n line = line_info.line\n # We need to make sure that we don't process lines which would be\n # otherwise valid python, such as \"x=1 # what?\"\n try:\n codeop.compile_command(line)\n except SyntaxError:\n # We should only handle as help stuff which is NOT valid syntax\n if line[0]==ESC_HELP:\n line = line[1:]\n elif line[-1]==ESC_HELP:\n line = line[:-1]\n if line:\n #print 'line:<%r>' % line # dbg\n self.shell.magic('pinfo %s' % line_info.ifun)\n else:\n self.shell.show_usage()\n return '' # Empty string is needed here!\n except:\n raise\n # Pass any other exceptions through to the normal handler\n return normal_handler.handle(line_info)\n else:\n # If the code compiles ok, we should handle it normally\n return normal_handler.handle(line_info)", "response": "Handle a single line of the log file."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef enterEvent(self, event):\n super(CallTipWidget, self).enterEvent(event)\n self._hide_timer.stop()", "response": "Reimplemented to cancel the hide timer."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef paintEvent(self, event):\n painter = QtGui.QStylePainter(self)\n option = QtGui.QStyleOptionFrame()\n option.initFrom(self)\n painter.drawPrimitive(QtGui.QStyle.PE_PanelTipLabel, option)\n painter.end()\n\n super(CallTipWidget, self).paintEvent(event)", "response": "Reimplemented to paint the background panel."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nshow the call line and docstring at the current cursor position.", "response": "def show_call_info(self, call_line=None, doc=None, maxlines=20):\n \"\"\" Attempts to show the specified call line and docstring at the\n current cursor location. The docstring is possibly truncated for\n length.\n \"\"\"\n if doc:\n match = re.match(\"(?:[^\\n]*\\n){%i}\" % maxlines, doc)\n if match:\n doc = doc[:match.end()] + '\\n[Documentation continues...]'\n else:\n doc = ''\n\n if call_line:\n doc = '\\n\\n'.join([call_line, doc])\n return self.show_tip(doc)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef show_tip(self, tip):\n # Attempt to find the cursor position at which to show the call tip.\n text_edit = self._text_edit\n document = text_edit.document()\n cursor = text_edit.textCursor()\n search_pos = cursor.position() - 1\n self._start_position, _ = self._find_parenthesis(search_pos,\n forward=False)\n if self._start_position == -1:\n return False\n\n # Set the text and resize the widget accordingly.\n self.setText(tip)\n self.resize(self.sizeHint())\n\n # Locate and show the widget. Place the tip below the current line\n # unless it would be off the screen. In that case, decide the best\n # location based trying to minimize the area that goes off-screen.\n padding = 3 # Distance in pixels between cursor bounds and tip box.\n cursor_rect = text_edit.cursorRect(cursor)\n screen_rect = QtGui.qApp.desktop().screenGeometry(text_edit)\n point = text_edit.mapToGlobal(cursor_rect.bottomRight())\n point.setY(point.y() + padding)\n tip_height = self.size().height()\n tip_width = self.size().width()\n\n vertical = 'bottom'\n horizontal = 'Right'\n if point.y() + tip_height > screen_rect.height():\n point_ = text_edit.mapToGlobal(cursor_rect.topRight())\n # If tip is still off screen, check if point is in top or bottom\n # half of screen.\n if point_.y() - tip_height < padding:\n # If point is in upper half of screen, show tip below it.\n # otherwise above it.\n if 2*point.y() < screen_rect.height():\n vertical = 'bottom'\n else:\n vertical = 'top'\n else:\n vertical = 'top'\n if point.x() + tip_width > screen_rect.width():\n point_ = text_edit.mapToGlobal(cursor_rect.topRight())\n # If tip is still off-screen, check if point is in the right or\n # left half of the screen.\n if point_.x() - tip_width < padding:\n if 2*point.x() < screen_rect.width():\n horizontal = 'Right'\n else:\n horizontal = 'Left'\n else:\n horizontal = 'Left'\n pos = getattr(cursor_rect, '%s%s' %(vertical, horizontal))\n point = text_edit.mapToGlobal(pos())\n if vertical == 'top':\n point.setY(point.y() - tip_height - padding)\n if horizontal == 'Left':\n point.setX(point.x() - tip_width - padding)\n\n self.move(point)\n self.show()\n return True", "response": "Show the specified tip at the current cursor location."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _find_parenthesis(self, position, forward=True):\n commas = depth = 0\n document = self._text_edit.document()\n char = document.characterAt(position)\n # Search until a match is found or a non-printable character is\n # encountered.\n while category(char) != 'Cc' and position > 0:\n if char == ',' and depth == 0:\n commas += 1\n elif char == ')':\n if forward and depth == 0:\n break\n depth += 1\n elif char == '(':\n if not forward and depth == 0:\n break\n depth -= 1\n position += 1 if forward else -1\n char = document.characterAt(position)\n else:\n position = -1\n return position, commas", "response": "Find the next parenthesis in the line that contains the given position."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nhiding the tooltip after some time has passed.", "response": "def _leave_event_hide(self):\n \"\"\" Hides the tooltip after some time has passed (assuming the cursor is\n not over the tooltip).\n \"\"\"\n if (not self._hide_timer.isActive() and\n # If Enter events always came after Leave events, we wouldn't need\n # this check. But on Mac OS, it sometimes happens the other way\n # around when the tooltip is created.\n QtGui.qApp.topLevelAt(QtGui.QCursor.pos()) != self):\n self._hide_timer.start(300, self)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _cursor_position_changed(self):\n cursor = self._text_edit.textCursor()\n if cursor.position() <= self._start_position:\n self.hide()\n else:\n position, commas = self._find_parenthesis(self._start_position + 1)\n if position != -1:\n self.hide()", "response": "Updates the tip based on user cursor position."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a property that proxies the local attribute proxied_attr through the local attribute local_attr.", "response": "def proxied_attribute(local_attr, proxied_attr, doc):\n \"\"\"Create a property that proxies attribute ``proxied_attr`` through\n the local attribute ``local_attr``.\n \"\"\"\n def fget(self):\n return getattr(getattr(self, local_attr), proxied_attr)\n def fset(self, value):\n setattr(getattr(self, local_attr), proxied_attr, value)\n def fdel(self):\n delattr(getattr(self, local_attr), proxied_attr)\n return property(fget, fset, fdel, doc)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef canonicalize_path(cwd, path):\n\n if not os.path.isabs(path):\n path = os.path.join(cwd, path)\n\n return os.path.abspath(path)", "response": "Canonicalizes a path relative to a given working directory."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef schema_validate(instance, schema, exc_class, *prefix, **kwargs):\n\n try:\n # Do the validation\n jsonschema.validate(instance, schema)\n except jsonschema.ValidationError as exc:\n # Assemble the path\n path = '/'.join((a if isinstance(a, six.string_types) else '[%d]' % a)\n for a in itertools.chain(prefix, exc.path))\n\n # Construct the message\n message = 'Failed to validate \"%s\": %s' % (path, exc.message)\n\n # Raise the target exception\n raise exc_class(message, **kwargs)", "response": "Validate the object with the given schema."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\niterates over a priority dictionary.", "response": "def iter_prio_dict(prio_dict):\n \"\"\"\n Iterate over a priority dictionary. A priority dictionary is a\n dictionary keyed by integer priority, with the values being lists\n of objects. This generator will iterate over the dictionary in\n priority order (from lowest integer value to highest integer\n value), yielding each object in the lists in turn.\n\n :param prio_dict: A priority dictionary, as described above.\n\n :returns: An iterator that yields each object in the correct\n order, based on priority and ordering within the\n priority values.\n \"\"\"\n\n for _prio, objs in sorted(prio_dict.items(), key=lambda x: x[0]):\n for obj in objs:\n yield obj"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve a read - only subordinate mapping.", "response": "def masked(self):\n \"\"\"\n Retrieve a read-only subordinate mapping. All values are\n stringified, and sensitive values are masked. The subordinate\n mapping implements the context manager protocol for\n convenience.\n \"\"\"\n\n if self._masked is None:\n self._masked = MaskedDict(self)\n\n return self._masked"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nbuild a file path from *paths* and return the contents.", "response": "def read(*paths):\n \"\"\"Build a file path from *paths* and return the contents.\"\"\"\n with open(os.path.join(*paths), 'r') as file_handler:\n return file_handler.read()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef virtualenv_no_global():\n #this mirrors the logic in virtualenv.py for locating the no-global-site-packages.txt file\n site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))\n no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt')\n if running_under_virtualenv() and os.path.isfile(no_global_file):\n return True", "response": "Return True if in a virtualenv and no system site packages."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nparallels word frequency counter.", "response": "def pwordfreq(view, fnames):\n \"\"\"Parallel word frequency counter.\n \n view - An IPython DirectView\n fnames - The filenames containing the split data.\n \"\"\"\n assert len(fnames) == len(view.targets)\n view.scatter('fname', fnames, flatten=True)\n ar = view.apply(wordfreq, Reference('fname'))\n freqs_list = ar.get()\n word_set = set()\n for f in freqs_list:\n word_set.update(f.keys())\n freqs = dict(zip(word_set, repeat(0)))\n for f in freqs_list:\n for word, count in f.iteritems():\n freqs[word] += count\n return freqs"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef view_decorator(function_decorator):\n\n\tdef simple_decorator(View):\n\t\tView.dispatch = method_decorator(function_decorator)(View.dispatch)\n\t\treturn View\n\n\treturn simple_decorator", "response": "Convert a function based decorator into a class based decorator usable\n\ton class based Views."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef default_aliases():\n # Note: the aliases defined here should be safe to use on a kernel\n # regardless of what frontend it is attached to. Frontends that use a\n # kernel in-process can define additional aliases that will only work in\n # their case. For example, things like 'less' or 'clear' that manipulate\n # the terminal should NOT be declared here, as they will only work if the\n # kernel is running inside a true terminal, and not over the network.\n\n if os.name == 'posix':\n default_aliases = [('mkdir', 'mkdir'), ('rmdir', 'rmdir'),\n ('mv', 'mv -i'), ('rm', 'rm -i'), ('cp', 'cp -i'),\n ('cat', 'cat'),\n ]\n # Useful set of ls aliases. The GNU and BSD options are a little\n # different, so we make aliases that provide as similar as possible\n # behavior in ipython, by passing the right flags for each platform\n if sys.platform.startswith('linux'):\n ls_aliases = [('ls', 'ls -F --color'),\n # long ls\n ('ll', 'ls -F -o --color'),\n # ls normal files only\n ('lf', 'ls -F -o --color %l | grep ^-'),\n # ls symbolic links\n ('lk', 'ls -F -o --color %l | grep ^l'),\n # directories or links to directories,\n ('ldir', 'ls -F -o --color %l | grep /$'),\n # things which are executable\n ('lx', 'ls -F -o --color %l | grep ^-..x'),\n ]\n else:\n # BSD, OSX, etc.\n ls_aliases = [('ls', 'ls -F'),\n # long ls\n ('ll', 'ls -F -l'),\n # ls normal files only\n ('lf', 'ls -F -l %l | grep ^-'),\n # ls symbolic links\n ('lk', 'ls -F -l %l | grep ^l'),\n # directories or links to directories,\n ('ldir', 'ls -F -l %l | grep /$'),\n # things which are executable\n ('lx', 'ls -F -l %l | grep ^-..x'),\n ]\n default_aliases = default_aliases + ls_aliases\n elif os.name in ['nt', 'dos']:\n default_aliases = [('ls', 'dir /on'),\n ('ddir', 'dir /ad /on'), ('ldir', 'dir /ad /on'),\n ('mkdir', 'mkdir'), ('rmdir', 'rmdir'),\n ('echo', 'echo'), ('ren', 'ren'), ('copy', 'copy'),\n ]\n else:\n default_aliases = []\n\n return default_aliases", "response": "Return a list of shell aliases to auto - define."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef soft_define_alias(self, name, cmd):\n try:\n self.define_alias(name, cmd)\n except AliasError, e:\n error(\"Invalid alias: %s\" % e)", "response": "Define an alias but don t raise on an AliasError."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef define_alias(self, name, cmd):\n nargs = self.validate_alias(name, cmd)\n self.alias_table[name] = (nargs, cmd)", "response": "Define a new alias after validating it."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef validate_alias(self, name, cmd):\n if name in self.no_alias:\n raise InvalidAliasError(\"The name %s can't be aliased \"\n \"because it is a keyword or builtin.\" % name)\n if not (isinstance(cmd, basestring)):\n raise InvalidAliasError(\"An alias command must be a string, \"\n \"got: %r\" % cmd)\n nargs = cmd.count('%s')\n if nargs>0 and cmd.find('%l')>=0:\n raise InvalidAliasError('The %s and %l specifiers are mutually '\n 'exclusive in alias definitions.')\n return nargs", "response": "Validate an alias and return the its number of arguments."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef call_alias(self, alias, rest=''):\n cmd = self.transform_alias(alias, rest)\n try:\n self.shell.system(cmd)\n except:\n self.shell.showtraceback()", "response": "Call an alias given its name and the rest of the line."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef transform_alias(self, alias,rest=''):\n nargs, cmd = self.alias_table[alias]\n\n if ' ' in cmd and os.path.isfile(cmd):\n cmd = '\"%s\"' % cmd\n\n # Expand the %l special to be the user's input line\n if cmd.find('%l') >= 0:\n cmd = cmd.replace('%l', rest)\n rest = ''\n if nargs==0:\n # Simple, argument-less aliases\n cmd = '%s %s' % (cmd, rest)\n else:\n # Handle aliases with positional arguments\n args = rest.split(None, nargs)\n if len(args) < nargs:\n raise AliasError('Alias <%s> requires %s arguments, %s given.' %\n (alias, nargs, len(args)))\n cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))\n return cmd", "response": "Transform an alias to a system command string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexpanding an alias in the command line", "response": "def expand_alias(self, line):\n \"\"\" Expand an alias in the command line\n\n Returns the provided command line, possibly with the first word\n (command) translated according to alias expansion rules.\n\n [ipython]|16> _ip.expand_aliases(\"np myfile.txt\")\n <16> 'q:/opt/np/notepad++.exe myfile.txt'\n \"\"\"\n\n pre,_,fn,rest = split_user_input(line)\n res = pre + self.expand_aliases(fn, rest)\n return res"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexpanding multiple levels of aliases", "response": "def expand_aliases(self, fn, rest):\n \"\"\"Expand multiple levels of aliases:\n\n if:\n\n alias foo bar /tmp\n alias baz foo\n\n then:\n\n baz huhhahhei -> bar /tmp huhhahhei\n \"\"\"\n line = fn + \" \" + rest\n\n done = set()\n while 1:\n pre,_,fn,rest = split_user_input(line, shell_line_split)\n if fn in self.alias_table:\n if fn in done:\n warn(\"Cyclic alias definition, repeated '%s'\" % fn)\n return \"\"\n done.add(fn)\n\n l2 = self.transform_alias(fn, rest)\n if l2 == line:\n break\n # ls -> ls -F should not recurse forever\n if l2.split(None,1)[0] == line.split(None,1)[0]:\n line = l2\n break\n line=l2\n else:\n break\n\n return line"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef shquote(arg):\n for c in '\"', \"'\", \"\\\\\", \"#\":\n if c in arg: return repr(arg)\n if arg.split()<>[arg]:\n return repr(arg)\n return arg", "response": "Quote an argument for later parsing by shlex. split"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nproducing rst from nose help", "response": "def autohelp_directive(dirname, arguments, options, content, lineno,\n content_offset, block_text, state, state_machine):\n \"\"\"produces rst from nose help\"\"\"\n config = Config(parserClass=OptBucket,\n plugins=BuiltinPluginManager())\n parser = config.getParser(TestProgram.usage())\n rst = ViewList()\n for line in parser.format_help().split('\\n'):\n rst.append(line, '')\n\n rst.append('Options', '')\n rst.append('-------', '')\n rst.append('', '')\n for opt in parser:\n rst.append(opt.options(), '')\n rst.append(' \\n', '')\n rst.append(' ' + opt.help + '\\n', '')\n rst.append('\\n', '') \n node = nodes.section()\n node.document = state.document\n surrounding_title_styles = state.memo.title_styles\n surrounding_section_level = state.memo.section_level\n state.memo.title_styles = []\n state.memo.section_level = 0\n state.nested_parse(rst, 0, node, match_titles=1)\n state.memo.title_styles = surrounding_title_styles\n state.memo.section_level = surrounding_section_level\n\n return node.children"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef reset_sgr(self):\n self.intensity = 0\n self.italic = False\n self.bold = False\n self.underline = False\n self.foreground_color = None\n self.background_color = None", "response": "Reset the graphics attributs to their default values."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nyields substrings for which the same escape code applies.", "response": "def split_string(self, string):\n \"\"\" Yields substrings for which the same escape code applies.\n \"\"\"\n self.actions = []\n start = 0\n\n # strings ending with \\r are assumed to be ending in \\r\\n since\n # \\n is appended to output strings automatically. Accounting\n # for that, here.\n last_char = '\\n' if len(string) > 0 and string[-1] == '\\n' else None\n string = string[:-1] if last_char is not None else string\n\n for match in ANSI_OR_SPECIAL_PATTERN.finditer(string):\n raw = string[start:match.start()]\n substring = SPECIAL_PATTERN.sub(self._replace_special, raw)\n if substring or self.actions:\n yield substring\n self.actions = []\n start = match.end()\n\n groups = filter(lambda x: x is not None, match.groups())\n g0 = groups[0]\n if g0 == '\\a':\n self.actions.append(BeepAction('beep'))\n yield None\n self.actions = []\n elif g0 == '\\r':\n self.actions.append(CarriageReturnAction('carriage-return'))\n yield None\n self.actions = []\n elif g0 == '\\b':\n self.actions.append(BackSpaceAction('backspace'))\n yield None\n self.actions = []\n elif g0 == '\\n' or g0 == '\\r\\n':\n self.actions.append(NewLineAction('newline'))\n yield g0\n self.actions = []\n else:\n params = [ param for param in groups[1].split(';') if param ]\n if g0.startswith('['):\n # Case 1: CSI code.\n try:\n params = map(int, params)\n except ValueError:\n # Silently discard badly formed codes.\n pass\n else:\n self.set_csi_code(groups[2], params)\n\n elif g0.startswith(']'):\n # Case 2: OSC code.\n self.set_osc_code(params)\n\n raw = string[start:]\n substring = SPECIAL_PATTERN.sub(self._replace_special, raw)\n if substring or self.actions:\n yield substring\n\n if last_char is not None:\n self.actions.append(NewLineAction('newline'))\n yield last_char"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_csi_code(self, command, params=[]):\n if command == 'm': # SGR - Select Graphic Rendition\n if params:\n self.set_sgr_code(params)\n else:\n self.set_sgr_code([0])\n\n elif (command == 'J' or # ED - Erase Data\n command == 'K'): # EL - Erase in Line\n code = params[0] if params else 0\n if 0 <= code <= 2:\n area = 'screen' if command == 'J' else 'line'\n if code == 0:\n erase_to = 'end'\n elif code == 1:\n erase_to = 'start'\n elif code == 2:\n erase_to = 'all'\n self.actions.append(EraseAction('erase', area, erase_to))\n\n elif (command == 'S' or # SU - Scroll Up\n command == 'T'): # SD - Scroll Down\n dir = 'up' if command == 'S' else 'down'\n count = params[0] if params else 1\n self.actions.append(ScrollAction('scroll', dir, 'line', count))", "response": "Sets attributes based on CSI Control Sequence Introducer code."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset attributes based on OSC command parameters.", "response": "def set_osc_code(self, params):\n \"\"\" Set attributes based on OSC (Operating System Command) parameters.\n\n Parameters\n ----------\n params : sequence of str\n The parameters for the command.\n \"\"\"\n try:\n command = int(params.pop(0))\n except (IndexError, ValueError):\n return\n\n if command == 4:\n # xterm-specific: set color number to color spec.\n try:\n color = int(params.pop(0))\n spec = params.pop(0)\n self.color_map[color] = self._parse_xterm_color_spec(spec)\n except (IndexError, ValueError):\n pass"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets attributes based on SGR codes.", "response": "def set_sgr_code(self, params):\n \"\"\" Set attributes based on SGR (Select Graphic Rendition) codes.\n\n Parameters\n ----------\n params : sequence of ints\n A list of SGR codes for one or more SGR commands. Usually this\n sequence will have one element per command, although certain\n xterm-specific commands requires multiple elements.\n \"\"\"\n # Always consume the first parameter.\n if not params:\n return\n code = params.pop(0)\n\n if code == 0:\n self.reset_sgr()\n elif code == 1:\n if self.bold_text_enabled:\n self.bold = True\n else:\n self.intensity = 1\n elif code == 2:\n self.intensity = 0\n elif code == 3:\n self.italic = True\n elif code == 4:\n self.underline = True\n elif code == 22:\n self.intensity = 0\n self.bold = False\n elif code == 23:\n self.italic = False\n elif code == 24:\n self.underline = False\n elif code >= 30 and code <= 37:\n self.foreground_color = code - 30\n elif code == 38 and params and params.pop(0) == 5:\n # xterm-specific: 256 color support.\n if params:\n self.foreground_color = params.pop(0)\n elif code == 39:\n self.foreground_color = None\n elif code >= 40 and code <= 47:\n self.background_color = code - 40\n elif code == 48 and params and params.pop(0) == 5:\n # xterm-specific: 256 color support.\n if params:\n self.background_color = params.pop(0)\n elif code == 49:\n self.background_color = None\n\n # Recurse with unconsumed parameters.\n self.set_sgr_code(params)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_color(self, color, intensity=0):\n if color is None:\n return None\n\n # Adjust for intensity, if possible.\n if color < 8 and intensity > 0:\n color += 8\n\n constructor = self.color_map.get(color, None)\n if isinstance(constructor, basestring):\n # If this is an X11 color name, we just hope there is a close SVG\n # color name. We could use QColor's static method\n # 'setAllowX11ColorNames()', but this is global and only available\n # on X11. It seems cleaner to aim for uniformity of behavior.\n return QtGui.QColor(constructor)\n\n elif isinstance(constructor, (tuple, list)):\n return QtGui.QColor(*constructor)\n\n return None", "response": "Returns a QColor object for a given color code and intensity."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_format(self):\n format = QtGui.QTextCharFormat()\n\n # Set foreground color\n qcolor = self.get_color(self.foreground_color, self.intensity)\n if qcolor is not None:\n format.setForeground(qcolor)\n\n # Set background color\n qcolor = self.get_color(self.background_color, self.intensity)\n if qcolor is not None:\n format.setBackground(qcolor)\n\n # Set font weight/style options\n if self.bold:\n format.setFontWeight(QtGui.QFont.Bold)\n else:\n format.setFontWeight(QtGui.QFont.Normal)\n format.setFontItalic(self.italic)\n format.setFontUnderline(self.underline)\n\n return format", "response": "Returns a QTextCharFormat that encodes the current style attributes."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_background_color(self, color):\n # Set a new default color map.\n self.default_color_map = self.darkbg_color_map.copy()\n\n if color.value() >= 127:\n # Colors appropriate for a terminal with a light background. For\n # now, only use non-bright colors...\n for i in xrange(8):\n self.default_color_map[i + 8] = self.default_color_map[i]\n\n # ...and replace white with black.\n self.default_color_map[7] = self.default_color_map[15] = 'black'\n\n # Update the current color map with the new defaults.\n self.color_map.update(self.default_color_map)", "response": "Set the background color of the log entry."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating a one - time jwt with an age in seconds", "response": "def generate(secret, age, **payload):\n \"\"\"Generate a one-time jwt with an age in seconds\"\"\"\n jti = str(uuid.uuid1()) # random id\n if not payload:\n payload = {}\n payload['exp'] = int(time.time() + age)\n payload['jti'] = jti\n return jwt.encode(payload, decode_secret(secret))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nuse a thread lock on current method, if self.lock is defined", "response": "def mutex(func):\n \"\"\"use a thread lock on current method, if self.lock is defined\"\"\"\n def wrapper(*args, **kwargs):\n \"\"\"Decorator Wrapper\"\"\"\n lock = args[0].lock\n lock.acquire(True)\n try:\n return func(*args, **kwargs)\n except:\n raise\n finally:\n lock.release()\n\n return wrapper"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _clean(self):\n now = time.time()\n for jwt in self.jwts.keys():\n if (now - self.jwts[jwt]) > (self.age * 2):\n del self.jwts[jwt]", "response": "Clean the cache of all the user - specified items."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks if a token is already used", "response": "def already_used(self, tok):\n \"\"\"has this jwt been used?\"\"\"\n if tok in self.jwts:\n return True\n self.jwts[tok] = time.time()\n return False"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nam this token valid?", "response": "def valid(self, token):\n \"\"\"is this token valid?\"\"\"\n now = time.time()\n\n if 'Bearer ' in token:\n token = token[7:]\n\n data = None\n for secret in self.secrets:\n try:\n data = jwt.decode(token, secret)\n break\n except jwt.DecodeError:\n continue\n except jwt.ExpiredSignatureError:\n raise JwtFailed(\"Jwt expired\")\n\n if not data:\n raise JwtFailed(\"Jwt cannot be decoded\")\n\n exp = data.get('exp')\n if not exp:\n raise JwtFailed(\"Jwt missing expiration (exp)\")\n\n if now - exp > self.age:\n raise JwtFailed(\"Jwt bad expiration - greater than I want to accept\")\n\n jti = data.get('jti')\n if not jti:\n raise JwtFailed(\"Jwt missing one-time id (jti)\")\n\n if self.already_used(jti):\n raise JwtFailed(\"Jwt re-use disallowed (jti={})\".format(jti))\n\n return data"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef split_lines(nb):\n for ws in nb.worksheets:\n for cell in ws.cells:\n if cell.cell_type == 'code':\n if 'input' in cell and isinstance(cell.input, basestring):\n cell.input = cell.input.splitlines()\n for output in cell.outputs:\n for key in _multiline_outputs:\n item = output.get(key, None)\n if isinstance(item, basestring):\n output[key] = item.splitlines()\n else: # text cell\n for key in ['source', 'rendered']:\n item = cell.get(key, None)\n if isinstance(item, basestring):\n cell[key] = item.splitlines()\n return nb", "response": "split likely multiline text into lists of strings\n \n For file output more friendly to line - based VCS."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write(self, nb, fp, **kwargs):\n return fp.write(self.writes(nb,**kwargs))", "response": "Write a notebook to a file like object"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef semaphore(count: int, bounded: bool=False):\n '''\n use `Semaphore` to keep func access thread-safety.\n\n example:\n\n ``` py\n @semaphore(3)\n def func(): pass\n ```\n '''\n\n lock_type = threading.BoundedSemaphore if bounded else threading.Semaphore\n lock_obj = lock_type(value=count)\n\n return with_it(lock_obj)", "response": "A semaphore function that returns a new object with the given count."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns the pyglet event loop by processing pending events only. This keeps processing pending events until stdin is ready. After processing all pending events, a call to time.sleep is inserted. This is needed, otherwise, CPU usage is at 100%. This sleep time should be tuned though for best performance.", "response": "def inputhook_glut():\n \"\"\"Run the pyglet event loop by processing pending events only.\n\n This keeps processing pending events until stdin is ready. After\n processing all pending events, a call to time.sleep is inserted. This is\n needed, otherwise, CPU usage is at 100%. This sleep time should be tuned\n though for best performance.\n \"\"\"\n # We need to protect against a user pressing Control-C when IPython is\n # idle and this is running. We trap KeyboardInterrupt and pass.\n\n signal.signal(signal.SIGINT, glut_int_handler)\n\n try:\n t = clock()\n\n # Make sure the default window is set after a window has been closed\n if glut.glutGetWindow() == 0:\n glut.glutSetWindow( 1 )\n glutMainLoopEvent()\n return 0\n\n while not stdin_ready():\n glutMainLoopEvent()\n # We need to sleep at this point to keep the idle CPU load\n # low. However, if sleep to long, GUI response is poor. As\n # a compromise, we watch how often GUI events are being processed\n # and switch between a short and long sleep time. Here are some\n # stats useful in helping to tune this.\n # time CPU load\n # 0.001 13%\n # 0.005 3%\n # 0.01 1.5%\n # 0.05 0.5%\n used_time = clock() - t\n if used_time > 5*60.0:\n # print 'Sleep for 5 s' # dbg\n time.sleep(5.0)\n elif used_time > 10.0:\n # print 'Sleep for 1 s' # dbg\n time.sleep(1.0)\n elif used_time > 0.1:\n # Few GUI events coming in, so we can sleep longer\n # print 'Sleep for 0.05 s' # dbg\n time.sleep(0.05)\n else:\n # Many GUI events coming in, so sleep only very little\n time.sleep(0.001)\n except KeyboardInterrupt:\n pass\n return 0"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the longest common prefix for completions", "response": "def commonprefix(items):\n \"\"\"Get common prefix for completions\n\n Return the longest common prefix of a list of strings, but with special\n treatment of escape characters that might precede commands in IPython,\n such as %magic functions. Used in tab completion.\n\n For a more general function, see os.path.commonprefix\n \"\"\"\n # the last item will always have the least leading % symbol\n # min / max are first/last in alphabetical order\n first_match = ESCAPE_RE.match(min(items))\n last_match = ESCAPE_RE.match(max(items))\n # common suffix is (common prefix of reversed items) reversed\n if first_match and last_match:\n prefix = os.path.commonprefix((first_match.group(0)[::-1], last_match.group(0)[::-1]))[::-1]\n else:\n prefix = ''\n\n items = [s.lstrip(ESCAPE_CHARS) for s in items]\n return prefix+os.path.commonprefix(items)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef eventFilter(self, obj, event):\n etype = event.type()\n if etype == QtCore.QEvent.KeyPress:\n\n # Re-map keys for all filtered widgets.\n key = event.key()\n if self._control_key_down(event.modifiers()) and \\\n key in self._ctrl_down_remap:\n new_event = QtGui.QKeyEvent(QtCore.QEvent.KeyPress,\n self._ctrl_down_remap[key],\n QtCore.Qt.NoModifier)\n QtGui.qApp.sendEvent(obj, new_event)\n return True\n\n elif obj == self._control:\n return self._event_filter_console_keypress(event)\n\n elif obj == self._page_control:\n return self._event_filter_page_keypress(event)\n\n # Make middle-click paste safe.\n elif etype == QtCore.QEvent.MouseButtonRelease and \\\n event.button() == QtCore.Qt.MidButton and \\\n obj == self._control.viewport():\n cursor = self._control.cursorForPosition(event.pos())\n self._control.setTextCursor(cursor)\n self.paste(QtGui.QClipboard.Selection)\n return True\n\n # Manually adjust the scrollbars *after* a resize event is dispatched.\n elif etype == QtCore.QEvent.Resize and not self._filter_resize:\n self._filter_resize = True\n QtGui.qApp.sendEvent(obj, event)\n self._adjust_scrollbars()\n self._filter_resize = False\n return True\n\n # Override shortcuts for all filtered widgets.\n elif etype == QtCore.QEvent.ShortcutOverride and \\\n self.override_shortcuts and \\\n self._control_key_down(event.modifiers()) and \\\n event.key() in self._shortcuts:\n event.accept()\n\n # Ensure that drags are safe. The problem is that the drag starting\n # logic, which determines whether the drag is a Copy or Move, is locked\n # down in QTextControl. If the widget is editable, which it must be if\n # we're not executing, the drag will be a Move. The following hack\n # prevents QTextControl from deleting the text by clearing the selection\n # when a drag leave event originating from this widget is dispatched.\n # The fact that we have to clear the user's selection is unfortunate,\n # but the alternative--trying to prevent Qt from using its hardwired\n # drag logic and writing our own--is worse.\n elif etype == QtCore.QEvent.DragEnter and \\\n obj == self._control.viewport() and \\\n event.source() == self._control.viewport():\n self._filter_drag = True\n elif etype == QtCore.QEvent.DragLeave and \\\n obj == self._control.viewport() and \\\n self._filter_drag:\n cursor = self._control.textCursor()\n cursor.clearSelection()\n self._control.setTextCursor(cursor)\n self._filter_drag = False\n\n # Ensure that drops are safe.\n elif etype == QtCore.QEvent.Drop and obj == self._control.viewport():\n cursor = self._control.cursorForPosition(event.pos())\n if self._in_buffer(cursor.position()):\n text = event.mimeData().text()\n self._insert_plain_text_into_buffer(cursor, text)\n\n # Qt is expecting to get something here--drag and drop occurs in its\n # own event loop. Send a DragLeave event to end it.\n QtGui.qApp.sendEvent(obj, QtGui.QDragLeaveEvent())\n return True\n\n # Handle scrolling of the vsplit pager. This hack attempts to solve\n # problems with tearing of the help text inside the pager window. This\n # happens only on Mac OS X with both PySide and PyQt. This fix isn't\n # perfect but makes the pager more usable.\n elif etype in self._pager_scroll_events and \\\n obj == self._page_control:\n self._page_control.repaint()\n return True\n return super(ConsoleWidget, self).eventFilter(obj, event)", "response": "Reimplemented to ensure a console - like behavior in the underlying unicode text widgets."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the size of the current object.", "response": "def sizeHint(self):\n \"\"\" Reimplemented to suggest a size that is 80 characters wide and\n 25 lines high.\n \"\"\"\n font_metrics = QtGui.QFontMetrics(self.font)\n margin = (self._control.frameWidth() +\n self._control.document().documentMargin()) * 2\n style = self.style()\n splitwidth = style.pixelMetric(QtGui.QStyle.PM_SplitterWidth)\n\n # Note 1: Despite my best efforts to take the various margins into\n # account, the width is still coming out a bit too small, so we include\n # a fudge factor of one character here.\n # Note 2: QFontMetrics.maxWidth is not used here or anywhere else due\n # to a Qt bug on certain Mac OS systems where it returns 0.\n width = font_metrics.width(' ') * 81 + margin\n width += style.pixelMetric(QtGui.QStyle.PM_ScrollBarExtent)\n if self.paging == 'hsplit':\n width = width * 2 + splitwidth\n\n height = font_metrics.height() * 25 + margin\n if self.paging == 'vsplit':\n height = height * 2 + splitwidth\n\n return QtCore.QSize(width, height)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef can_cut(self):\n cursor = self._control.textCursor()\n return (cursor.hasSelection() and\n self._in_buffer(cursor.anchor()) and\n self._in_buffer(cursor.position()))", "response": "Returns whether the text can be cut to the clipboard."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef can_paste(self):\n if self._control.textInteractionFlags() & QtCore.Qt.TextEditable:\n return bool(QtGui.QApplication.clipboard().text())\n return False", "response": "Returns whether the text can be pasted from the clipboard."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nclears the console. Parameters: ----------- keep_input : bool, optional (default True) If set, restores the old input buffer if a new prompt is written.", "response": "def clear(self, keep_input=True):\n \"\"\" Clear the console.\n\n Parameters:\n -----------\n keep_input : bool, optional (default True)\n If set, restores the old input buffer if a new prompt is written.\n \"\"\"\n if self._executing:\n self._control.clear()\n else:\n if keep_input:\n input_buffer = self.input_buffer\n self._control.clear()\n self._show_prompt()\n if keep_input:\n self.input_buffer = input_buffer"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cut(self):\n self.copy()\n if self.can_cut():\n self._control.textCursor().removeSelectedText()", "response": "Cuts the currently selected text to the clipboard and deletes it if it s inside the input buffer."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef execute(self, source=None, hidden=False, interactive=False):\n # WARNING: The order in which things happen here is very particular, in\n # large part because our syntax highlighting is fragile. If you change\n # something, test carefully!\n\n # Decide what to execute.\n if source is None:\n source = self.input_buffer\n if not hidden:\n # A newline is appended later, but it should be considered part\n # of the input buffer.\n source += '\\n'\n elif not hidden:\n self.input_buffer = source\n\n # Execute the source or show a continuation prompt if it is incomplete.\n complete = self._is_complete(source, interactive)\n if hidden:\n if complete:\n self._execute(source, hidden)\n else:\n error = 'Incomplete noninteractive input: \"%s\"'\n raise RuntimeError(error % source)\n else:\n if complete:\n self._append_plain_text('\\n')\n self._input_buffer_executing = self.input_buffer\n self._executing = True\n self._prompt_finished()\n\n # The maximum block count is only in effect during execution.\n # This ensures that _prompt_pos does not become invalid due to\n # text truncation.\n self._control.document().setMaximumBlockCount(self.buffer_size)\n\n # Setting a positive maximum block count will automatically\n # disable the undo/redo history, but just to be safe:\n self._control.setUndoRedoEnabled(False)\n\n # Perform actual execution.\n self._execute(source, hidden)\n\n else:\n # Do this inside an edit block so continuation prompts are\n # removed seamlessly via undo/redo.\n cursor = self._get_end_cursor()\n cursor.beginEditBlock()\n cursor.insertText('\\n')\n self._insert_continuation_prompt(cursor)\n cursor.endEditBlock()\n\n # Do not do this inside the edit block. It works as expected\n # when using a QPlainTextEdit control, but does not have an\n # effect when using a QTextEdit. I believe this is a Qt bug.\n self._control.moveCursor(QtGui.QTextCursor.End)\n\n return complete", "response": "Executes the source or the input buffer."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_input_buffer(self, force=False):\n # If we're executing, the input buffer may not even exist anymore due to\n # the limit imposed by 'buffer_size'. Therefore, we store it.\n if self._executing and not force:\n return self._input_buffer_executing\n\n cursor = self._get_end_cursor()\n cursor.setPosition(self._prompt_pos, QtGui.QTextCursor.KeepAnchor)\n input_buffer = cursor.selection().toPlainText()\n\n # Strip out continuation prompts.\n return input_buffer.replace('\\n' + self._continuation_prompt, '\\n')", "response": "Get the text that the user has entered at the current prompt."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _set_input_buffer(self, string):\n # If we're executing, store the text for later.\n if self._executing:\n self._input_buffer_pending = string\n return\n\n # Remove old text.\n cursor = self._get_end_cursor()\n cursor.beginEditBlock()\n cursor.setPosition(self._prompt_pos, QtGui.QTextCursor.KeepAnchor)\n cursor.removeSelectedText()\n\n # Insert new text with continuation prompts.\n self._insert_plain_text_into_buffer(self._get_prompt_cursor(), string)\n cursor.endEditBlock()\n self._control.moveCursor(QtGui.QTextCursor.End)", "response": "Sets the text in the input buffer."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _set_font(self, font):\n font_metrics = QtGui.QFontMetrics(font)\n self._control.setTabStopWidth(self.tab_width * font_metrics.width(' '))\n\n self._completion_widget.setFont(font)\n self._control.document().setDefaultFont(font)\n if self._page_control:\n self._page_control.document().setDefaultFont(font)\n\n self.font_changed.emit(font)", "response": "Sets the base font for the ConsoleWidget to the specified QFont."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef paste(self, mode=QtGui.QClipboard.Clipboard):\n if self._control.textInteractionFlags() & QtCore.Qt.TextEditable:\n # Make sure the paste is safe.\n self._keep_cursor_in_buffer()\n cursor = self._control.textCursor()\n\n # Remove any trailing newline, which confuses the GUI and forces the\n # user to backspace.\n text = QtGui.QApplication.clipboard().text(mode).rstrip()\n self._insert_plain_text_into_buffer(cursor, dedent(text))", "response": "Paste the contents of the clipboard into the input region."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nprinting the contents of the ConsoleWidget to the specified QPrinter.", "response": "def print_(self, printer = None):\n \"\"\" Print the contents of the ConsoleWidget to the specified QPrinter.\n \"\"\"\n if (not printer):\n printer = QtGui.QPrinter()\n if(QtGui.QPrintDialog(printer).exec_() != QtGui.QDialog.Accepted):\n return\n self._control.print_(printer)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef prompt_to_top(self):\n if not self._executing:\n prompt_cursor = self._get_prompt_cursor()\n if self._get_cursor().blockNumber() < prompt_cursor.blockNumber():\n self._set_cursor(prompt_cursor)\n self._set_top_cursor(prompt_cursor)", "response": "Moves the prompt to the top of the viewport."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nresetting the font to the default fixed - width font for this platform.", "response": "def reset_font(self):\n \"\"\" Sets the font to the default fixed-width font for this platform.\n \"\"\"\n if sys.platform == 'win32':\n # Consolas ships with Vista/Win7, fallback to Courier if needed\n fallback = 'Courier'\n elif sys.platform == 'darwin':\n # OSX always has Monaco\n fallback = 'Monaco'\n else:\n # Monospace should always exist\n fallback = 'Monospace'\n font = get_font(self.font_family, fallback)\n if self.font_size:\n font.setPointSize(self.font_size)\n else:\n font.setPointSize(QtGui.qApp.font().pointSize())\n font.setStyleHint(QtGui.QFont.TypeWriter)\n self._set_font(font)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef change_font_size(self, delta):\n font = self.font\n size = max(font.pointSize() + delta, 1) # minimum 1 point\n font.setPointSize(size)\n self._set_font(font)", "response": "Change the font size by the specified amount."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the width of the tab characters for the current locale.", "response": "def _set_tab_width(self, tab_width):\n \"\"\" Sets the width (in terms of space characters) for tab characters.\n \"\"\"\n font_metrics = QtGui.QFontMetrics(self.font)\n self._control.setTabStopWidth(tab_width * font_metrics.width(' '))\n\n self._tab_width = tab_width"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _append_custom(self, insert, input, before_prompt=False):\n # Determine where to insert the content.\n cursor = self._control.textCursor()\n if before_prompt and (self._reading or not self._executing):\n cursor.setPosition(self._append_before_prompt_pos)\n else:\n cursor.movePosition(QtGui.QTextCursor.End)\n start_pos = cursor.position()\n\n # Perform the insertion.\n result = insert(cursor, input)\n\n # Adjust the prompt position if we have inserted before it. This is safe\n # because buffer truncation is disabled when not executing.\n if before_prompt and not self._executing:\n diff = cursor.position() - start_pos\n self._append_before_prompt_pos += diff\n self._prompt_pos += diff\n\n return result", "response": "Internal method to append custom content to the end of the buffer."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _append_html(self, html, before_prompt=False):\n self._append_custom(self._insert_html, html, before_prompt)", "response": "Append HTML at the end of the console buffer."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _append_html_fetching_plain_text(self, html, before_prompt=False):\n return self._append_custom(self._insert_html_fetching_plain_text,\n html, before_prompt)", "response": "Appends HTML then returns the plain text version of it."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nappending plain text to the log.", "response": "def _append_plain_text(self, text, before_prompt=False):\n \"\"\" Appends plain text, processing ANSI codes if enabled.\n \"\"\"\n self._append_custom(self._insert_plain_text, text, before_prompt)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nclearing the temporary text buffer.", "response": "def _clear_temporary_buffer(self):\n \"\"\" Clears the \"temporary text\" buffer, i.e. all the text following\n the prompt region.\n \"\"\"\n # Select and remove all text below the input buffer.\n cursor = self._get_prompt_cursor()\n prompt = self._continuation_prompt.lstrip()\n if(self._temp_buffer_filled):\n self._temp_buffer_filled = False\n while cursor.movePosition(QtGui.QTextCursor.NextBlock):\n temp_cursor = QtGui.QTextCursor(cursor)\n temp_cursor.select(QtGui.QTextCursor.BlockUnderCursor)\n text = temp_cursor.selection().toPlainText().lstrip()\n if not text.startswith(prompt):\n break\n else:\n # We've reached the end of the input buffer and no text follows.\n return\n cursor.movePosition(QtGui.QTextCursor.Left) # Grab the newline.\n cursor.movePosition(QtGui.QTextCursor.End,\n QtGui.QTextCursor.KeepAnchor)\n cursor.removeSelectedText()\n\n # After doing this, we have no choice but to clear the undo/redo\n # history. Otherwise, the text is not \"temporary\" at all, because it\n # can be recalled with undo/redo. Unfortunately, Qt does not expose\n # fine-grained control to the undo/redo system.\n if self._control.isUndoRedoEnabled():\n self._control.setUndoRedoEnabled(False)\n self._control.setUndoRedoEnabled(True)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nperforms completion with items at the specified cursor location.", "response": "def _complete_with_items(self, cursor, items):\n \"\"\" Performs completion with 'items' at the specified cursor location.\n \"\"\"\n self._cancel_completion()\n\n if len(items) == 1:\n cursor.setPosition(self._control.textCursor().position(),\n QtGui.QTextCursor.KeepAnchor)\n cursor.insertText(items[0])\n\n elif len(items) > 1:\n current_pos = self._control.textCursor().position()\n prefix = commonprefix(items)\n if prefix:\n cursor.setPosition(current_pos, QtGui.QTextCursor.KeepAnchor)\n cursor.insertText(prefix)\n current_pos = cursor.position()\n\n cursor.movePosition(QtGui.QTextCursor.Left, n=len(prefix))\n self._completion_widget.show_items(cursor, items)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _fill_temporary_buffer(self, cursor, text, html=False):\n\n current_pos = self._control.textCursor().position()\n\n cursor.beginEditBlock()\n self._append_plain_text('\\n')\n self._page(text, html=html)\n cursor.endEditBlock()\n\n cursor.setPosition(current_pos)\n self._control.moveCursor(QtGui.QTextCursor.End)\n self._control.setTextCursor(cursor)\n\n self._temp_buffer_filled = True", "response": "fill the area below the active editting zone with text"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _context_menu_make(self, pos):\n menu = QtGui.QMenu(self)\n\n self.cut_action = menu.addAction('Cut', self.cut)\n self.cut_action.setEnabled(self.can_cut())\n self.cut_action.setShortcut(QtGui.QKeySequence.Cut)\n\n self.copy_action = menu.addAction('Copy', self.copy)\n self.copy_action.setEnabled(self.can_copy())\n self.copy_action.setShortcut(QtGui.QKeySequence.Copy)\n\n self.paste_action = menu.addAction('Paste', self.paste)\n self.paste_action.setEnabled(self.can_paste())\n self.paste_action.setShortcut(QtGui.QKeySequence.Paste)\n\n menu.addSeparator()\n menu.addAction(self.select_all_action)\n\n menu.addSeparator()\n menu.addAction(self.export_action)\n menu.addAction(self.print_action)\n\n return menu", "response": "Creates a context menu for the given QPoint."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn whether the Control key is down.", "response": "def _control_key_down(self, modifiers, include_command=False):\n \"\"\" Given a KeyboardModifiers flags object, return whether the Control\n key is down.\n\n Parameters:\n -----------\n include_command : bool, optional (default True)\n Whether to treat the Command key as a (mutually exclusive) synonym\n for Control when in Mac OS.\n \"\"\"\n # Note that on Mac OS, ControlModifier corresponds to the Command key\n # while MetaModifier corresponds to the Control key.\n if sys.platform == 'darwin':\n down = include_command and (modifiers & QtCore.Qt.ControlModifier)\n return bool(down) ^ bool(modifiers & QtCore.Qt.MetaModifier)\n else:\n return bool(modifiers & QtCore.Qt.ControlModifier)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates and connects the underlying text widget.", "response": "def _create_control(self):\n \"\"\" Creates and connects the underlying text widget.\n \"\"\"\n # Create the underlying control.\n if self.custom_control:\n control = self.custom_control()\n elif self.kind == 'plain':\n control = QtGui.QPlainTextEdit()\n elif self.kind == 'rich':\n control = QtGui.QTextEdit()\n control.setAcceptRichText(False)\n\n # Install event filters. The filter on the viewport is needed for\n # mouse events and drag events.\n control.installEventFilter(self)\n control.viewport().installEventFilter(self)\n\n # Connect signals.\n control.customContextMenuRequested.connect(\n self._custom_context_menu_requested)\n control.copyAvailable.connect(self.copy_available)\n control.redoAvailable.connect(self.redo_available)\n control.undoAvailable.connect(self.undo_available)\n\n # Hijack the document size change signal to prevent Qt from adjusting\n # the viewport's scrollbar. We are relying on an implementation detail\n # of Q(Plain)TextEdit here, which is potentially dangerous, but without\n # this functionality we cannot create a nice terminal interface.\n layout = control.document().documentLayout()\n layout.documentSizeChanged.disconnect()\n layout.documentSizeChanged.connect(self._adjust_scrollbars)\n\n # Configure the control.\n control.setAttribute(QtCore.Qt.WA_InputMethodEnabled, True)\n control.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)\n control.setReadOnly(True)\n control.setUndoRedoEnabled(False)\n control.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)\n return control"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _create_page_control(self):\n if self.custom_page_control:\n control = self.custom_page_control()\n elif self.kind == 'plain':\n control = QtGui.QPlainTextEdit()\n elif self.kind == 'rich':\n control = QtGui.QTextEdit()\n control.installEventFilter(self)\n viewport = control.viewport()\n viewport.installEventFilter(self)\n control.setReadOnly(True)\n control.setUndoRedoEnabled(False)\n control.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)\n return control", "response": "Creates and connects the underlying paging widget."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfilter the event for the underlying console - like interface.", "response": "def _event_filter_console_keypress(self, event):\n \"\"\" Filter key events for the underlying text widget to create a\n console-like interface.\n \"\"\"\n intercepted = False\n cursor = self._control.textCursor()\n position = cursor.position()\n key = event.key()\n ctrl_down = self._control_key_down(event.modifiers())\n alt_down = event.modifiers() & QtCore.Qt.AltModifier\n shift_down = event.modifiers() & QtCore.Qt.ShiftModifier\n\n #------ Special sequences ----------------------------------------------\n\n if event.matches(QtGui.QKeySequence.Copy):\n self.copy()\n intercepted = True\n\n elif event.matches(QtGui.QKeySequence.Cut):\n self.cut()\n intercepted = True\n\n elif event.matches(QtGui.QKeySequence.Paste):\n self.paste()\n intercepted = True\n\n #------ Special modifier logic -----------------------------------------\n\n elif key in (QtCore.Qt.Key_Return, QtCore.Qt.Key_Enter):\n intercepted = True\n\n # Special handling when tab completing in text mode.\n self._cancel_completion()\n\n if self._in_buffer(position):\n # Special handling when a reading a line of raw input.\n if self._reading:\n self._append_plain_text('\\n')\n self._reading = False\n if self._reading_callback:\n self._reading_callback()\n\n # If the input buffer is a single line or there is only\n # whitespace after the cursor, execute. Otherwise, split the\n # line with a continuation prompt.\n elif not self._executing:\n cursor.movePosition(QtGui.QTextCursor.End,\n QtGui.QTextCursor.KeepAnchor)\n at_end = len(cursor.selectedText().strip()) == 0\n single_line = (self._get_end_cursor().blockNumber() ==\n self._get_prompt_cursor().blockNumber())\n if (at_end or shift_down or single_line) and not ctrl_down:\n self.execute(interactive = not shift_down)\n else:\n # Do this inside an edit block for clean undo/redo.\n cursor.beginEditBlock()\n cursor.setPosition(position)\n cursor.insertText('\\n')\n self._insert_continuation_prompt(cursor)\n cursor.endEditBlock()\n\n # Ensure that the whole input buffer is visible.\n # FIXME: This will not be usable if the input buffer is\n # taller than the console widget.\n self._control.moveCursor(QtGui.QTextCursor.End)\n self._control.setTextCursor(cursor)\n\n #------ Control/Cmd modifier -------------------------------------------\n\n elif ctrl_down:\n if key == QtCore.Qt.Key_G:\n self._keyboard_quit()\n intercepted = True\n\n elif key == QtCore.Qt.Key_K:\n if self._in_buffer(position):\n cursor.clearSelection()\n cursor.movePosition(QtGui.QTextCursor.EndOfLine,\n QtGui.QTextCursor.KeepAnchor)\n if not cursor.hasSelection():\n # Line deletion (remove continuation prompt)\n cursor.movePosition(QtGui.QTextCursor.NextBlock,\n QtGui.QTextCursor.KeepAnchor)\n cursor.movePosition(QtGui.QTextCursor.Right,\n QtGui.QTextCursor.KeepAnchor,\n len(self._continuation_prompt))\n self._kill_ring.kill_cursor(cursor)\n self._set_cursor(cursor)\n intercepted = True\n\n elif key == QtCore.Qt.Key_L:\n self.prompt_to_top()\n intercepted = True\n\n elif key == QtCore.Qt.Key_O:\n if self._page_control and self._page_control.isVisible():\n self._page_control.setFocus()\n intercepted = True\n\n elif key == QtCore.Qt.Key_U:\n if self._in_buffer(position):\n cursor.clearSelection()\n start_line = cursor.blockNumber()\n if start_line == self._get_prompt_cursor().blockNumber():\n offset = len(self._prompt)\n else:\n offset = len(self._continuation_prompt)\n cursor.movePosition(QtGui.QTextCursor.StartOfBlock,\n QtGui.QTextCursor.KeepAnchor)\n cursor.movePosition(QtGui.QTextCursor.Right,\n QtGui.QTextCursor.KeepAnchor, offset)\n self._kill_ring.kill_cursor(cursor)\n self._set_cursor(cursor)\n intercepted = True\n\n elif key == QtCore.Qt.Key_Y:\n self._keep_cursor_in_buffer()\n self._kill_ring.yank()\n intercepted = True\n\n elif key in (QtCore.Qt.Key_Backspace, QtCore.Qt.Key_Delete):\n if key == QtCore.Qt.Key_Backspace:\n cursor = self._get_word_start_cursor(position)\n else: # key == QtCore.Qt.Key_Delete\n cursor = self._get_word_end_cursor(position)\n cursor.setPosition(position, QtGui.QTextCursor.KeepAnchor)\n self._kill_ring.kill_cursor(cursor)\n intercepted = True\n\n elif key == QtCore.Qt.Key_D:\n if len(self.input_buffer) == 0:\n self.exit_requested.emit(self)\n else:\n new_event = QtGui.QKeyEvent(QtCore.QEvent.KeyPress,\n QtCore.Qt.Key_Delete,\n QtCore.Qt.NoModifier)\n QtGui.qApp.sendEvent(self._control, new_event)\n intercepted = True\n\n #------ Alt modifier ---------------------------------------------------\n\n elif alt_down:\n if key == QtCore.Qt.Key_B:\n self._set_cursor(self._get_word_start_cursor(position))\n intercepted = True\n\n elif key == QtCore.Qt.Key_F:\n self._set_cursor(self._get_word_end_cursor(position))\n intercepted = True\n\n elif key == QtCore.Qt.Key_Y:\n self._kill_ring.rotate()\n intercepted = True\n\n elif key == QtCore.Qt.Key_Backspace:\n cursor = self._get_word_start_cursor(position)\n cursor.setPosition(position, QtGui.QTextCursor.KeepAnchor)\n self._kill_ring.kill_cursor(cursor)\n intercepted = True\n\n elif key == QtCore.Qt.Key_D:\n cursor = self._get_word_end_cursor(position)\n cursor.setPosition(position, QtGui.QTextCursor.KeepAnchor)\n self._kill_ring.kill_cursor(cursor)\n intercepted = True\n\n elif key == QtCore.Qt.Key_Delete:\n intercepted = True\n\n elif key == QtCore.Qt.Key_Greater:\n self._control.moveCursor(QtGui.QTextCursor.End)\n intercepted = True\n\n elif key == QtCore.Qt.Key_Less:\n self._control.setTextCursor(self._get_prompt_cursor())\n intercepted = True\n\n #------ No modifiers ---------------------------------------------------\n\n else:\n if shift_down:\n anchormode = QtGui.QTextCursor.KeepAnchor\n else:\n anchormode = QtGui.QTextCursor.MoveAnchor\n\n if key == QtCore.Qt.Key_Escape:\n self._keyboard_quit()\n intercepted = True\n\n elif key == QtCore.Qt.Key_Up:\n if self._reading or not self._up_pressed(shift_down):\n intercepted = True\n else:\n prompt_line = self._get_prompt_cursor().blockNumber()\n intercepted = cursor.blockNumber() <= prompt_line\n\n elif key == QtCore.Qt.Key_Down:\n if self._reading or not self._down_pressed(shift_down):\n intercepted = True\n else:\n end_line = self._get_end_cursor().blockNumber()\n intercepted = cursor.blockNumber() == end_line\n\n elif key == QtCore.Qt.Key_Tab:\n if not self._reading:\n if self._tab_pressed():\n # real tab-key, insert four spaces\n cursor.insertText(' '*4)\n intercepted = True\n\n elif key == QtCore.Qt.Key_Left:\n\n # Move to the previous line\n line, col = cursor.blockNumber(), cursor.columnNumber()\n if line > self._get_prompt_cursor().blockNumber() and \\\n col == len(self._continuation_prompt):\n self._control.moveCursor(QtGui.QTextCursor.PreviousBlock,\n mode=anchormode)\n self._control.moveCursor(QtGui.QTextCursor.EndOfBlock,\n mode=anchormode)\n intercepted = True\n\n # Regular left movement\n else:\n intercepted = not self._in_buffer(position - 1)\n\n elif key == QtCore.Qt.Key_Right:\n original_block_number = cursor.blockNumber()\n cursor.movePosition(QtGui.QTextCursor.Right,\n mode=anchormode)\n if cursor.blockNumber() != original_block_number:\n cursor.movePosition(QtGui.QTextCursor.Right,\n n=len(self._continuation_prompt),\n mode=anchormode)\n self._set_cursor(cursor)\n intercepted = True\n\n elif key == QtCore.Qt.Key_Home:\n start_line = cursor.blockNumber()\n if start_line == self._get_prompt_cursor().blockNumber():\n start_pos = self._prompt_pos\n else:\n cursor.movePosition(QtGui.QTextCursor.StartOfBlock,\n QtGui.QTextCursor.KeepAnchor)\n start_pos = cursor.position()\n start_pos += len(self._continuation_prompt)\n cursor.setPosition(position)\n if shift_down and self._in_buffer(position):\n cursor.setPosition(start_pos, QtGui.QTextCursor.KeepAnchor)\n else:\n cursor.setPosition(start_pos)\n self._set_cursor(cursor)\n intercepted = True\n\n elif key == QtCore.Qt.Key_Backspace:\n\n # Line deletion (remove continuation prompt)\n line, col = cursor.blockNumber(), cursor.columnNumber()\n if not self._reading and \\\n col == len(self._continuation_prompt) and \\\n line > self._get_prompt_cursor().blockNumber():\n cursor.beginEditBlock()\n cursor.movePosition(QtGui.QTextCursor.StartOfBlock,\n QtGui.QTextCursor.KeepAnchor)\n cursor.removeSelectedText()\n cursor.deletePreviousChar()\n cursor.endEditBlock()\n intercepted = True\n\n # Regular backwards deletion\n else:\n anchor = cursor.anchor()\n if anchor == position:\n intercepted = not self._in_buffer(position - 1)\n else:\n intercepted = not self._in_buffer(min(anchor, position))\n\n elif key == QtCore.Qt.Key_Delete:\n\n # Line deletion (remove continuation prompt)\n if not self._reading and self._in_buffer(position) and \\\n cursor.atBlockEnd() and not cursor.hasSelection():\n cursor.movePosition(QtGui.QTextCursor.NextBlock,\n QtGui.QTextCursor.KeepAnchor)\n cursor.movePosition(QtGui.QTextCursor.Right,\n QtGui.QTextCursor.KeepAnchor,\n len(self._continuation_prompt))\n cursor.removeSelectedText()\n intercepted = True\n\n # Regular forwards deletion:\n else:\n anchor = cursor.anchor()\n intercepted = (not self._in_buffer(anchor) or\n not self._in_buffer(position))\n\n # Don't move the cursor if Control/Cmd is pressed to allow copy-paste\n # using the keyboard in any part of the buffer. Also, permit scrolling\n # with Page Up/Down keys. Finally, if we're executing, don't move the\n # cursor (if even this made sense, we can't guarantee that the prompt\n # position is still valid due to text truncation).\n if not (self._control_key_down(event.modifiers(), include_command=True)\n or key in (QtCore.Qt.Key_PageUp, QtCore.Qt.Key_PageDown)\n or (self._executing and not self._reading)):\n self._keep_cursor_in_buffer()\n\n return intercepted"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _event_filter_page_keypress(self, event):\n key = event.key()\n ctrl_down = self._control_key_down(event.modifiers())\n alt_down = event.modifiers() & QtCore.Qt.AltModifier\n\n if ctrl_down:\n if key == QtCore.Qt.Key_O:\n self._control.setFocus()\n intercept = True\n\n elif alt_down:\n if key == QtCore.Qt.Key_Greater:\n self._page_control.moveCursor(QtGui.QTextCursor.End)\n intercepted = True\n\n elif key == QtCore.Qt.Key_Less:\n self._page_control.moveCursor(QtGui.QTextCursor.Start)\n intercepted = True\n\n elif key in (QtCore.Qt.Key_Q, QtCore.Qt.Key_Escape):\n if self._splitter:\n self._page_control.hide()\n self._control.setFocus()\n else:\n self.layout().setCurrentWidget(self._control)\n return True\n\n elif key in (QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return,\n QtCore.Qt.Key_Tab):\n new_event = QtGui.QKeyEvent(QtCore.QEvent.KeyPress,\n QtCore.Qt.Key_PageDown,\n QtCore.Qt.NoModifier)\n QtGui.qApp.sendEvent(self._page_control, new_event)\n return True\n\n elif key == QtCore.Qt.Key_Backspace:\n new_event = QtGui.QKeyEvent(QtCore.QEvent.KeyPress,\n QtCore.Qt.Key_PageUp,\n QtCore.Qt.NoModifier)\n QtGui.qApp.sendEvent(self._page_control, new_event)\n return True\n\n return False", "response": "Filter key events for the paging widget to create console - like object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _format_as_columns(self, items, separator=' '):\n # Calculate the number of characters available.\n width = self._control.viewport().width()\n char_width = QtGui.QFontMetrics(self.font).width(' ')\n displaywidth = max(10, (width / char_width) - 1)\n\n return columnize(items, separator, displaywidth)", "response": "Transform a list of strings into a single string with columns."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_block_plain_text(self, block):\n cursor = QtGui.QTextCursor(block)\n cursor.movePosition(QtGui.QTextCursor.StartOfBlock)\n cursor.movePosition(QtGui.QTextCursor.EndOfBlock,\n QtGui.QTextCursor.KeepAnchor)\n return cursor.selection().toPlainText()", "response": "Given a QTextBlock return its unformatted text."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_end_cursor(self):\n cursor = self._control.textCursor()\n cursor.movePosition(QtGui.QTextCursor.End)\n return cursor", "response": "Returns a cursor for the last character in the control."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the column number of the cursor in the input buffer excluding the the contribution by the prompt.", "response": "def _get_input_buffer_cursor_column(self):\n \"\"\" Returns the column of the cursor in the input buffer, excluding the\n contribution by the prompt, or -1 if there is no such column.\n \"\"\"\n prompt = self._get_input_buffer_cursor_prompt()\n if prompt is None:\n return -1\n else:\n cursor = self._control.textCursor()\n return cursor.columnNumber() - len(prompt)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the line of the input buffer that contains the cursor.", "response": "def _get_input_buffer_cursor_line(self):\n \"\"\" Returns the text of the line of the input buffer that contains the\n cursor, or None if there is no such line.\n \"\"\"\n prompt = self._get_input_buffer_cursor_prompt()\n if prompt is None:\n return None\n else:\n cursor = self._control.textCursor()\n text = self._get_block_plain_text(cursor.block())\n return text[len(prompt):]"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the prompt for the line that contains the cursor.", "response": "def _get_input_buffer_cursor_prompt(self):\n \"\"\" Returns the (plain text) prompt for line of the input buffer that\n contains the cursor, or None if there is no such line.\n \"\"\"\n if self._executing:\n return None\n cursor = self._control.textCursor()\n if cursor.position() >= self._prompt_pos:\n if cursor.blockNumber() == self._get_prompt_cursor().blockNumber():\n return self._prompt\n else:\n return self._continuation_prompt\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_prompt_cursor(self):\n cursor = self._control.textCursor()\n cursor.setPosition(self._prompt_pos)\n return cursor", "response": "Returns a cursor for the prompt position."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_selection_cursor(self, start, end):\n cursor = self._control.textCursor()\n cursor.setPosition(start)\n cursor.setPosition(end, QtGui.QTextCursor.KeepAnchor)\n return cursor", "response": "Returns a cursor with text selected between the positions start and end."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind the start of the word to the left the given position.", "response": "def _get_word_start_cursor(self, position):\n \"\"\" Find the start of the word to the left the given position. If a\n sequence of non-word characters precedes the first word, skip over\n them. (This emulates the behavior of bash, emacs, etc.)\n \"\"\"\n document = self._control.document()\n position -= 1\n while position >= self._prompt_pos and \\\n not is_letter_or_number(document.characterAt(position)):\n position -= 1\n while position >= self._prompt_pos and \\\n is_letter_or_number(document.characterAt(position)):\n position -= 1\n cursor = self._control.textCursor()\n cursor.setPosition(position + 1)\n return cursor"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfinding the end of the word to the right the given position.", "response": "def _get_word_end_cursor(self, position):\n \"\"\" Find the end of the word to the right the given position. If a\n sequence of non-word characters precedes the first word, skip over\n them. (This emulates the behavior of bash, emacs, etc.)\n \"\"\"\n document = self._control.document()\n end = self._get_end_cursor().position()\n while position < end and \\\n not is_letter_or_number(document.characterAt(position)):\n position += 1\n while position < end and \\\n is_letter_or_number(document.characterAt(position)):\n position += 1\n cursor = self._control.textCursor()\n cursor.setPosition(position)\n return cursor"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _insert_continuation_prompt(self, cursor):\n if self._continuation_prompt_html is None:\n self._insert_plain_text(cursor, self._continuation_prompt)\n else:\n self._continuation_prompt = self._insert_html_fetching_plain_text(\n cursor, self._continuation_prompt_html)", "response": "Inserts new continuation prompt using the specified cursor."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninserts the specified HTML into the specified cursor.", "response": "def _insert_html(self, cursor, html):\n \"\"\" Inserts HTML using the specified cursor in such a way that future\n formatting is unaffected.\n \"\"\"\n cursor.beginEditBlock()\n cursor.insertHtml(html)\n\n # After inserting HTML, the text document \"remembers\" it's in \"html\n # mode\", which means that subsequent calls adding plain text will result\n # in unwanted formatting, lost tab characters, etc. The following code\n # hacks around this behavior, which I consider to be a bug in Qt, by\n # (crudely) resetting the document's style state.\n cursor.movePosition(QtGui.QTextCursor.Left,\n QtGui.QTextCursor.KeepAnchor)\n if cursor.selection().toPlainText() == ' ':\n cursor.removeSelectedText()\n else:\n cursor.movePosition(QtGui.QTextCursor.Right)\n cursor.insertText(' ', QtGui.QTextCharFormat())\n cursor.endEditBlock()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninsert the specified cursor and returns its plain text version.", "response": "def _insert_html_fetching_plain_text(self, cursor, html):\n \"\"\" Inserts HTML using the specified cursor, then returns its plain text\n version.\n \"\"\"\n cursor.beginEditBlock()\n cursor.removeSelectedText()\n\n start = cursor.position()\n self._insert_html(cursor, html)\n end = cursor.position()\n cursor.setPosition(start, QtGui.QTextCursor.KeepAnchor)\n text = cursor.selection().toPlainText()\n\n cursor.setPosition(end)\n cursor.endEditBlock()\n return text"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninsert plain text using the specified cursor.", "response": "def _insert_plain_text(self, cursor, text):\n \"\"\" Inserts plain text using the specified cursor, processing ANSI codes\n if enabled.\n \"\"\"\n cursor.beginEditBlock()\n if self.ansi_codes:\n for substring in self._ansi_processor.split_string(text):\n for act in self._ansi_processor.actions:\n\n # Unlike real terminal emulators, we don't distinguish\n # between the screen and the scrollback buffer. A screen\n # erase request clears everything.\n if act.action == 'erase' and act.area == 'screen':\n cursor.select(QtGui.QTextCursor.Document)\n cursor.removeSelectedText()\n\n # Simulate a form feed by scrolling just past the last line.\n elif act.action == 'scroll' and act.unit == 'page':\n cursor.insertText('\\n')\n cursor.endEditBlock()\n self._set_top_cursor(cursor)\n cursor.joinPreviousEditBlock()\n cursor.deletePreviousChar()\n\n elif act.action == 'carriage-return':\n cursor.movePosition(\n cursor.StartOfLine, cursor.KeepAnchor)\n\n elif act.action == 'beep':\n QtGui.qApp.beep()\n\n elif act.action == 'backspace':\n if not cursor.atBlockStart():\n cursor.movePosition(\n cursor.PreviousCharacter, cursor.KeepAnchor)\n\n elif act.action == 'newline':\n cursor.movePosition(cursor.EndOfLine)\n\n format = self._ansi_processor.get_format()\n\n selection = cursor.selectedText()\n if len(selection) == 0:\n cursor.insertText(substring, format)\n elif substring is not None:\n # BS and CR are treated as a change in print\n # position, rather than a backwards character\n # deletion for output equivalence with (I)Python\n # terminal.\n if len(substring) >= len(selection):\n cursor.insertText(substring, format)\n else:\n old_text = selection[len(substring):]\n cursor.insertText(substring + old_text, format)\n cursor.movePosition(cursor.PreviousCharacter,\n cursor.KeepAnchor, len(old_text))\n else:\n cursor.insertText(text)\n cursor.endEditBlock()"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninserts text into the input buffer.", "response": "def _insert_plain_text_into_buffer(self, cursor, text):\n \"\"\" Inserts text into the input buffer using the specified cursor (which\n must be in the input buffer), ensuring that continuation prompts are\n inserted as necessary.\n \"\"\"\n lines = text.splitlines(True)\n if lines:\n cursor.beginEditBlock()\n cursor.insertText(lines[0])\n for line in lines[1:]:\n if self._continuation_prompt_html is None:\n cursor.insertText(self._continuation_prompt)\n else:\n self._continuation_prompt = \\\n self._insert_html_fetching_plain_text(\n cursor, self._continuation_prompt_html)\n cursor.insertText(line)\n cursor.endEditBlock()"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _in_buffer(self, position=None):\n cursor = self._control.textCursor()\n if position is None:\n position = cursor.position()\n else:\n cursor.setPosition(position)\n line = cursor.blockNumber()\n prompt_line = self._get_prompt_cursor().blockNumber()\n if line == prompt_line:\n return position >= self._prompt_pos\n elif line > prompt_line:\n cursor.movePosition(QtGui.QTextCursor.StartOfBlock)\n prompt_pos = cursor.position() + len(self._continuation_prompt)\n return position >= prompt_pos\n return False", "response": "Returns True if the current cursor is inside the editing region."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _keep_cursor_in_buffer(self):\n moved = not self._in_buffer()\n if moved:\n cursor = self._control.textCursor()\n cursor.movePosition(QtGui.QTextCursor.End)\n self._control.setTextCursor(cursor)\n return moved", "response": "Keeps the cursor in the buffer. Returns True if the cursor was moved False otherwise."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _keyboard_quit(self):\n if self._temp_buffer_filled :\n self._cancel_completion()\n self._clear_temporary_buffer()\n else:\n self.input_buffer = ''", "response": "Cancels the current editing task ala Ctrl - G in Emacs."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndisplays text using the pager if it exceeds the height of the viewport.", "response": "def _page(self, text, html=False):\n \"\"\" Displays text using the pager if it exceeds the height of the\n viewport.\n\n Parameters:\n -----------\n html : bool, optional (default False)\n If set, the text will be interpreted as HTML instead of plain text.\n \"\"\"\n line_height = QtGui.QFontMetrics(self.font).height()\n minlines = self._control.viewport().height() / line_height\n if self.paging != 'none' and \\\n re.match(\"(?:[^\\n]*\\n){%i}\" % minlines, text):\n if self.paging == 'custom':\n self.custom_page_requested.emit(text)\n else:\n self._page_control.clear()\n cursor = self._page_control.textCursor()\n if html:\n self._insert_html(cursor, text)\n else:\n self._insert_plain_text(cursor, text)\n self._page_control.moveCursor(QtGui.QTextCursor.Start)\n\n self._page_control.viewport().resize(self._control.size())\n if self._splitter:\n self._page_control.show()\n self._page_control.setFocus()\n else:\n self.layout().setCurrentWidget(self._page_control)\n elif html:\n self._append_html(text)\n else:\n self._append_plain_text(text)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _prompt_started(self):\n # Temporarily disable the maximum block count to permit undo/redo and\n # to ensure that the prompt position does not change due to truncation.\n self._control.document().setMaximumBlockCount(0)\n self._control.setUndoRedoEnabled(True)\n\n # Work around bug in QPlainTextEdit: input method is not re-enabled\n # when read-only is disabled.\n self._control.setReadOnly(False)\n self._control.setAttribute(QtCore.Qt.WA_InputMethodEnabled, True)\n\n if not self._reading:\n self._executing = False\n self._prompt_started_hook()\n\n # If the input buffer has changed while executing, load it.\n if self._input_buffer_pending:\n self.input_buffer = self._input_buffer_pending\n self._input_buffer_pending = ''\n\n self._control.moveCursor(QtGui.QTextCursor.End)", "response": "Called immediately after a new prompt is displayed."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _readline(self, prompt='', callback=None):\n if self._reading:\n raise RuntimeError('Cannot read a line. Widget is already reading.')\n\n if not callback and not self.isVisible():\n # If the user cannot see the widget, this function cannot return.\n raise RuntimeError('Cannot synchronously read a line if the widget '\n 'is not visible!')\n\n self._reading = True\n self._show_prompt(prompt, newline=False)\n\n if callback is None:\n self._reading_callback = None\n while self._reading:\n QtCore.QCoreApplication.processEvents()\n return self._get_input_buffer(force=True).rstrip('\\n')\n\n else:\n self._reading_callback = lambda: \\\n callback(self._get_input_buffer(force=True).rstrip('\\n'))", "response": "Reads one line of input from the user."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the continuation prompt.", "response": "def _set_continuation_prompt(self, prompt, html=False):\n \"\"\" Sets the continuation prompt.\n\n Parameters\n ----------\n prompt : str\n The prompt to show when more input is needed.\n\n html : bool, optional (default False)\n If set, the prompt will be inserted as formatted HTML. Otherwise,\n the prompt will be treated as plain text, though ANSI color codes\n will be handled.\n \"\"\"\n if html:\n self._continuation_prompt_html = prompt\n else:\n self._continuation_prompt = prompt\n self._continuation_prompt_html = None"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the cursor to the top of the screen.", "response": "def _set_top_cursor(self, cursor):\n \"\"\" Scrolls the viewport so that the specified cursor is at the top.\n \"\"\"\n scrollbar = self._control.verticalScrollBar()\n scrollbar.setValue(scrollbar.maximum())\n original_cursor = self._control.textCursor()\n self._control.setTextCursor(cursor)\n self._control.ensureCursorVisible()\n self._control.setTextCursor(original_cursor)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a new prompt at the end of the buffer.", "response": "def _show_prompt(self, prompt=None, html=False, newline=True):\n \"\"\" Writes a new prompt at the end of the buffer.\n\n Parameters\n ----------\n prompt : str, optional\n The prompt to show. If not specified, the previous prompt is used.\n\n html : bool, optional (default False)\n Only relevant when a prompt is specified. If set, the prompt will\n be inserted as formatted HTML. Otherwise, the prompt will be treated\n as plain text, though ANSI color codes will be handled.\n\n newline : bool, optional (default True)\n If set, a new line will be written before showing the prompt if\n there is not already a newline at the end of the buffer.\n \"\"\"\n # Save the current end position to support _append*(before_prompt=True).\n cursor = self._get_end_cursor()\n self._append_before_prompt_pos = cursor.position()\n\n # Insert a preliminary newline, if necessary.\n if newline and cursor.position() > 0:\n cursor.movePosition(QtGui.QTextCursor.Left,\n QtGui.QTextCursor.KeepAnchor)\n if cursor.selection().toPlainText() != '\\n':\n self._append_plain_text('\\n')\n\n # Write the prompt.\n self._append_plain_text(self._prompt_sep)\n if prompt is None:\n if self._prompt_html is None:\n self._append_plain_text(self._prompt)\n else:\n self._append_html(self._prompt_html)\n else:\n if html:\n self._prompt = self._append_html_fetching_plain_text(prompt)\n self._prompt_html = prompt\n else:\n self._append_plain_text(prompt)\n self._prompt = prompt\n self._prompt_html = None\n\n self._prompt_pos = self._get_end_cursor().position()\n self._prompt_started()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadjust the vertical scrollbar to the range set by Qt.", "response": "def _adjust_scrollbars(self):\n \"\"\" Expands the vertical scrollbar beyond the range set by Qt.\n \"\"\"\n # This code is adapted from _q_adjustScrollbars in qplaintextedit.cpp\n # and qtextedit.cpp.\n document = self._control.document()\n scrollbar = self._control.verticalScrollBar()\n viewport_height = self._control.viewport().height()\n if isinstance(self._control, QtGui.QPlainTextEdit):\n maximum = max(0, document.lineCount() - 1)\n step = viewport_height / self._control.fontMetrics().lineSpacing()\n else:\n # QTextEdit does not do line-based layout and blocks will not in\n # general have the same height. Therefore it does not make sense to\n # attempt to scroll in line height increments.\n maximum = document.size().height()\n step = viewport_height\n diff = maximum - scrollbar.maximum()\n scrollbar.setRange(0, maximum)\n scrollbar.setPageStep(step)\n\n # Compensate for undesirable scrolling that occurs automatically due to\n # maximumBlockCount() text truncation.\n if diff < 0 and document.blockCount() == document.maximumBlockCount():\n scrollbar.setValue(scrollbar.value() + diff)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _custom_context_menu_requested(self, pos):\n menu = self._context_menu_make(pos)\n menu.exec_(self._control.mapToGlobal(pos))", "response": "Shows a context menu at the given QPoint."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if given Distribution is installed in user site.", "response": "def dist_in_usersite(dist):\n \"\"\"\n Return True if given Distribution is installed in user site.\n \"\"\"\n if user_site:\n return normalize_path(dist_location(dist)).startswith(normalize_path(user_site))\n else:\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprints the informations from installed distributions found.", "response": "def print_results(distributions, list_all_files):\n \"\"\"\n Print the informations from installed distributions found.\n \"\"\"\n results_printed = False\n for dist in distributions:\n results_printed = True\n logger.info(\"---\")\n logger.info(\"Name: %s\" % dist['name'])\n logger.info(\"Version: %s\" % dist['version'])\n logger.info(\"Location: %s\" % dist['location'])\n logger.info(\"Requires: %s\" % ', '.join(dist['requires']))\n if list_all_files:\n logger.info(\"Files:\")\n if dist['files'] is not None:\n for line in dist['files']:\n logger.info(\" %s\" % line.strip())\n else:\n logger.info(\"Cannot locate installed-files.txt\")\n return results_printed"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef main(args=None):\n options, paths = _parse_options(args)\n format = getattr(options, 'output', 'simple')\n formatter = _FORMATTERS[format](options)\n\n for path in paths:\n meta = get_metadata(path, options.metadata_version)\n if meta is None:\n continue\n\n if options.download_url_prefix:\n if meta.download_url is None:\n filename = os.path.basename(path)\n meta.download_url = '%s/%s' % (options.download_url_prefix,\n filename)\n\n formatter(meta)\n\n formatter.finish()", "response": "Entry point for pkginfo tool\n "} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the context for the specified string.", "response": "def get_context(self, string):\n \"\"\" Assuming the cursor is at the end of the specified string, get the\n context (a list of names) for the symbol at cursor position.\n \"\"\"\n context = []\n reversed_tokens = list(self._lexer.get_tokens(string))\n reversed_tokens.reverse()\n\n # Pygments often tacks on a newline when none is specified in the input.\n # Remove this newline.\n if reversed_tokens and reversed_tokens[0][1].endswith('\\n') and \\\n not string.endswith('\\n'):\n reversed_tokens.pop(0)\n\n current_op = ''\n for token, text in reversed_tokens:\n\n if is_token_subtype(token, Token.Name):\n\n # Handle a trailing separator, e.g 'foo.bar.'\n if current_op in self._name_separators:\n if not context:\n context.insert(0, '')\n\n # Handle non-separator operators and punction.\n elif current_op:\n break\n\n context.insert(0, text)\n current_op = ''\n\n # Pygments doesn't understand that, e.g., '->' is a single operator\n # in C++. This is why we have to build up an operator from\n # potentially several tokens.\n elif token is Token.Operator or token is Token.Punctuation:\n current_op = text + current_op\n\n # Break on anything that is not a Operator, Punctuation, or Name.\n else:\n break\n\n return context"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef copy_config_file(self, config_file, path=None, overwrite=False):\n dst = os.path.join(self.location, config_file)\n if os.path.isfile(dst) and not overwrite:\n return False\n if path is None:\n path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')\n src = os.path.join(path, config_file)\n shutil.copy(src, dst)\n return True", "response": "Copy a default config file into the active profile."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a profile dir by name and path.", "response": "def create_profile_dir_by_name(cls, path, name=u'default', config=None):\n \"\"\"Create a profile dir by profile name and path.\n\n Parameters\n ----------\n path : unicode\n The path (directory) to put the profile directory in.\n name : unicode\n The name of the profile. The name of the profile directory will\n be \"profile_\".\n \"\"\"\n if not os.path.isdir(path):\n raise ProfileDirError('Directory not found: %s' % path)\n profile_dir = os.path.join(path, u'profile_' + name)\n return cls(location=profile_dir, config=config)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinds an existing profile dir by profile name return its ProfileDir.", "response": "def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):\n \"\"\"Find an existing profile dir by profile name, return its ProfileDir.\n\n This searches through a sequence of paths for a profile dir. If it\n is not found, a :class:`ProfileDirError` exception will be raised.\n\n The search path algorithm is:\n 1. ``os.getcwdu()``\n 2. ``ipython_dir``\n\n Parameters\n ----------\n ipython_dir : unicode or str\n The IPython directory to use.\n name : unicode or str\n The name of the profile. The name of the profile directory\n will be \"profile_\".\n \"\"\"\n dirname = u'profile_' + name\n paths = [os.getcwdu(), ipython_dir]\n for p in paths:\n profile_dir = os.path.join(p, dirname)\n if os.path.isdir(profile_dir):\n return cls(location=profile_dir, config=config)\n else:\n raise ProfileDirError('Profile directory not found in paths: %s' % dirname)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef find_profile_dir(cls, profile_dir, config=None):\n profile_dir = expand_path(profile_dir)\n if not os.path.isdir(profile_dir):\n raise ProfileDirError('Profile directory not found: %s' % profile_dir)\n return cls(location=profile_dir, config=config)", "response": "Find a profile dir and return its ProfileDir."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cmp_to_key(mycmp):\n 'Convert a cmp= function into a key= function'\n class Key(object):\n def __init__(self, obj):\n self.obj = obj\n def __lt__(self, other):\n return mycmp(self.obj, other.obj) < 0\n def __gt__(self, other):\n return mycmp(self.obj, other.obj) > 0\n def __eq__(self, other):\n return mycmp(self.obj, other.obj) == 0\n return Key", "response": "Convert a cmp = function into a key = function"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads a file and close it. Returns the file source.", "response": "def file_read(filename):\n \"\"\"Read a file and close it. Returns the file source.\"\"\"\n fobj = open(filename,'r');\n source = fobj.read();\n fobj.close()\n return source"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef file_readlines(filename):\n fobj = open(filename,'r');\n lines = fobj.readlines();\n fobj.close()\n return lines", "response": "Read a file and close it. Returns the file source using readlines."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):\n\n try:\n if header:\n header += '\\n'\n lines = [raw_input(header + ps1)]\n except EOFError:\n return []\n terminate = [terminate_str]\n try:\n while lines[-1:] != terminate:\n new_line = raw_input(ps1)\n while new_line.endswith('\\\\'):\n new_line = new_line[:-1] + raw_input(ps2)\n lines.append(new_line)\n\n return lines[:-1] # don't return the termination command\n except EOFError:\n print()\n return lines", "response": "Take multiple lines of input."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nasking a question and returns a boolean ( y or n answer.", "response": "def ask_yes_no(prompt,default=None):\n \"\"\"Asks a question and returns a boolean (y/n) answer.\n\n If default is given (one of 'y','n'), it is used if the user input is\n empty. Otherwise the question is repeated until an answer is given.\n\n An EOF is treated as the default answer. If there is no default, an\n exception is raised to prevent infinite loops.\n\n Valid answers are: y/yes/n/no (match is not case sensitive).\"\"\"\n\n answers = {'y':True,'n':False,'yes':True,'no':False}\n ans = None\n while ans not in answers.keys():\n try:\n ans = raw_input(prompt+' ').lower()\n if not ans: # response was an empty string\n ans = default\n except KeyboardInterrupt:\n pass\n except EOFError:\n if default in answers.keys():\n ans = default\n print()\n else:\n raise\n\n return answers[ans]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef temp_pyfile(src, ext='.py'):\n fname = tempfile.mkstemp(ext)[1]\n f = open(fname,'w')\n f.write(src)\n f.flush()\n return fname, f", "response": "Make a temporary python file and return filename and filehandle."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef raw_print(*args, **kw):\n\n print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\\n'),\n file=sys.__stdout__)\n sys.__stdout__.flush()", "response": "Raw print to sys. stdout if it s not already set."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef close(self):\n self.flush()\n setattr(sys, self.channel, self.ostream)\n self.file.close()\n self._closed = True", "response": "Close the file and restore the channel."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write(self, data):\n self.file.write(data)\n self.ostream.write(data)\n self.ostream.flush()", "response": "Write data to both channels."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef show(self):\n sys.stdout.write(self.stdout)\n sys.stderr.write(self.stderr)\n sys.stdout.flush()\n sys.stderr.flush()", "response": "write my output to sys. stdout and err as appropriate"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_new_heart_handler(self, handler):\n self.log.debug(\"heartbeat::new_heart_handler: %s\", handler)\n self._new_handlers.add(handler)", "response": "add a new handler for new hearts"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd a new handler for heart failure", "response": "def add_heart_failure_handler(self, handler):\n \"\"\"add a new handler for heart failure\"\"\"\n self.log.debug(\"heartbeat::new heart failure handler: %s\", handler)\n self._failure_handlers.add(handler)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nhandles a PING message.", "response": "def handle_pong(self, msg):\n \"a heart just beat\"\n current = str_to_bytes(str(self.lifetime))\n last = str_to_bytes(str(self.last_ping))\n if msg[1] == current:\n delta = time.time()-self.tic\n # self.log.debug(\"heartbeat::heart %r took %.2f ms to respond\"%(msg[0], 1000*delta))\n self.responses.add(msg[0])\n elif msg[1] == last:\n delta = time.time()-self.tic + (self.lifetime-self.last_ping)\n self.log.warn(\"heartbeat::heart %r missed a beat, and took %.2f ms to respond\", msg[0], 1000*delta)\n self.responses.add(msg[0])\n else:\n self.log.warn(\"heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)\", msg[1], self.lifetime)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncatch exceptions and returns the expected output of a function", "response": "def catch(fcn, *args, **kwargs):\n '''try:\n retrun fcn(*args, **kwargs)\n except:\n print traceback\n if 'spit' in kwargs.keys():\n return kwargs['spit']\n\n Parameters\n ----------\n fcn : function\n *args : unnamed parameters of fcn\n **kwargs : named parameters of fcn\n spit : returns the parameter named return in the exception\n\n Returns\n -------\n The expected output of fcn or prints the exception traceback\n\n '''\n\n try:\n # remove the special kwargs key \"spit\" and use it to return if it exists\n spit = kwargs.pop('spit')\n except:\n spit = None\n\n try:\n results = fcn(*args, **kwargs)\n if results:\n return results\n except:\n print traceback.format_exc()\n if spit:\n return spit"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts a list into a list of lists with equal batch_size.", "response": "def batch_list(sequence, batch_size, mod = 0, randomize = False):\n '''\n Converts a list into a list of lists with equal batch_size.\n\n Parameters\n ----------\n sequence : list\n list of items to be placed in batches\n batch_size : int\n length of each sub list\n mod : int\n remainder of list length devided by batch_size\n mod = len(sequence) % batch_size\n randomize = bool\n should the initial sequence be randomized before being batched\n \n '''\n\n if randomize:\n sequence = random.sample(sequence, len(sequence))\n\n return [sequence[x:x + batch_size] for x in xrange(0, len(sequence)-mod, batch_size)]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef to_pickle(obj, filename, clean_memory=False):\n '''http://stackoverflow.com/questions/7900944/read-write-classes-to-files-in-an-efficent-way'''\n\n path, filename = path_to_filename(filename)\n create_dir(path)\n\n with open(path + filename, \"wb\") as output:\n pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL)\n\n if clean_memory:\n obj = None\n\n # setting the global object to None requires a return assignment\n return obj", "response": "Pickle a object to a file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef path_to_filename(pathfile):\n '''\n Takes a path filename string and returns the split between the path and the filename\n\n if filename is not given, filename = ''\n if path is not given, path = './'\n\n '''\n\n path = pathfile[:pathfile.rfind('/') + 1]\n if path == '':\n path = './'\n\n filename = pathfile[pathfile.rfind('/') + 1:len(pathfile)]\n if '.' not in filename:\n path = pathfile\n filename = ''\n\n if (filename == '') and (path[len(path) - 1] != '/'):\n path += '/'\n\n return path, filename", "response": "Takes a path filename string and returns the split between the path and the filename"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_dir(path, dir_dict={}):\n '''\n Tries to create a new directory in the given path.\n **create_dir** can also create subfolders according to the dictionnary given as second argument.\n\n Parameters\n ----------\n path : string\n string giving the path of the location to create the directory, either absolute or relative.\n dir_dict : dictionary, optional \n Dictionary ordering the creation of subfolders. Keys must be strings, and values either None or path dictionaries.\n the default is {}, which means that no subfolders will be created\n\n Examples\n --------\n\n >>> path = './project'\n >>> dir_dict = {'dir1':None, 'dir2':{'subdir21':None}}\n >>> utils.create_dir(path, dir_dict)\n\n will create:\n\n * *project/dir1*\n * *project/dir2/subdir21*\n\n in your parent directory.\n\n '''\n\n folders = path.split('/')\n folders = [i for i in folders if i != '']\n rootPath = ''\n if folders[0] == 'C:':\n folders = folders[1:]\n count = 0\n for directory in folders:\n count += 1\n\n # required to handle the dot operators\n if (directory[0] == '.') & (count == 1):\n rootPath = directory\n else:\n rootPath = rootPath + '/' + directory\n try:\n os.makedirs(rootPath)\n # If the file already exists (EEXIST), raise exception and do nothing\n except OSError as exception:\n if exception.errno != errno.EEXIST:\n raise\n\n for key in dir_dict.keys():\n rootPath = path + \"/\" + key\n try:\n os.makedirs(rootPath)\n # If the file already exists (EEXIST), raise exception and do nothing\n except OSError as exception:\n if exception.errno != errno.EEXIST:\n raise\n if dir_dict[key] is not None:\n create_dir(rootPath, dir_dict[key])", "response": "Creates a new directory in the given path."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Walk(root='.', recurse=True, pattern='*'):\n ''' \n Generator for walking a directory tree.\n Starts at specified root folder, returning files that match our pattern. \n Optionally will also recurse through sub-folders.\n\n Parameters\n ----------\n root : string (default is *'.'*)\n Path for the root folder to look in.\n recurse : bool (default is *True*)\n If *True*, will also look in the subfolders.\n pattern : string (default is :emphasis:`'*'`, which means all the files are concerned)\n The pattern to look for in the files' name.\n\n Returns\n -------\n generator\n **Walk** yields a generator from the matching files paths.\n\n '''\n\n for path, subdirs, files in os.walk(root):\n for name in files:\n if fnmatch.fnmatch(name, pattern):\n yield os.path.join(path, name)\n if not recurse:\n break", "response": "A generator for walking a directory tree."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwalks a directory tree and returns a list of all the file paths that match the given pattern.", "response": "def scan_path(root='.', recurse=False, pattern='*'):\n '''\n Runs a loop over the :doc:`Walk` Generator\n to find all file paths in the root directory with the given\n pattern. If recurse is *True*: matching paths are identified\n for all sub directories.\n\n Parameters\n ----------\n root : string (default is *'.'*)\n Path for the root folder to look in.\n recurse : bool (default is *True*)\n If *True*, will also look in the subfolders.\n pattern : string (default is :emphasis:`'*'`, which means all the files are concerned)\n The pattern to look for in the files' name.\n\n Returns\n -------\n path_list : list\n list of all the matching files paths.\n\n '''\n\n path_list = []\n for path in Walk(root=root, recurse=recurse, pattern=pattern):\n path_list.append(path)\n return path_list"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef displayAll(elapsed, display_amt, est_end, nLoops, count, numPrints):\n '''Displays time if verbose is true and count is within the display amount'''\n\n if numPrints > nLoops:\n display_amt = 1\n else:\n display_amt = round(nLoops / numPrints)\n\n if count % display_amt == 0:\n\n avg = elapsed / count\n est_end = round(avg * nLoops)\n\n (disp_elapsed,\n disp_avg,\n disp_est) = timeUnit(int(round(elapsed)),\n int(round(avg)),\n int(round(est_end)))\n\n print \"%s%%\" % str(round(count / float(nLoops) * 100)), \"@\" + str(count),\n\n totalTime = disp_est[0]\n unit = disp_est[1]\n\n if str(unit) == \"secs\":\n remain = totalTime - round(elapsed)\n remainUnit = \"secs\"\n elif str(unit) == \"mins\":\n remain = totalTime - round(elapsed) / 60\n remainUnit = \"mins\"\n elif str(unit) == \"hr\":\n remain = totalTime - round(elapsed) / 3600\n remainUnit = \"hr\"\n\n print \"ETA: %s %s\" % (str(remain), remainUnit)\n print\n\n return", "response": "Displays the time if verbose is true and count is within the display amount"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalculates unit of time to display", "response": "def timeUnit(elapsed, avg, est_end):\n '''calculates unit of time to display'''\n minute = 60\n hr = 3600\n day = 86400\n\n if elapsed <= 3 * minute:\n unit_elapsed = (elapsed, \"secs\")\n if elapsed > 3 * minute:\n unit_elapsed = ((elapsed / 60), \"mins\")\n if elapsed > 3 * hr:\n unit_elapsed = ((elapsed / 3600), \"hr\")\n\n if avg <= 3 * minute:\n unit_avg = (avg, \"secs\")\n if avg > 3 * minute:\n unit_avg = ((avg / 60), \"mins\")\n if avg > 3 * hr:\n unit_avg = ((avg / 3600), \"hr\")\n\n if est_end <= 3 * minute:\n unit_estEnd = (est_end, \"secs\")\n if est_end > 3 * minute:\n unit_estEnd = ((est_end / 60), \"mins\")\n if est_end > 3 * hr:\n unit_estEnd = ((est_end / 3600), \"hr\")\n\n return [unit_elapsed, unit_avg, unit_estEnd]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntrack the time in a loop. The estimated time to completion can be calculated and if verbose is set to *True*, the object will print estimated time to completion, and percent complete. Actived in every loop to keep track", "response": "def loop(self):\n '''\n Tracks the time in a loop. The estimated time to completion\n can be calculated and if verbose is set to *True*, the object will print\n estimated time to completion, and percent complete.\n Actived in every loop to keep track'''\n\n self.count += 1\n self.tf = time.time()\n self.elapsed = self.tf - self.ti\n\n if self.verbose:\n displayAll(self.elapsed, self.display_amt, self.est_end,\n self.nLoops, self.count, self.numPrints)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd a path to sys. path.", "response": "def add_path(path, config=None):\n \"\"\"Ensure that the path, or the root of the current package (if\n path is in a package), is in sys.path.\n \"\"\"\n\n # FIXME add any src-looking dirs seen too... need to get config for that\n \n log.debug('Add path %s' % path) \n if not path:\n return []\n added = []\n parent = os.path.dirname(path)\n if (parent\n and os.path.exists(os.path.join(path, '__init__.py'))):\n added.extend(add_path(parent, config))\n elif not path in sys.path:\n log.debug(\"insert %s into sys.path\", path)\n sys.path.insert(0, path)\n added.append(path)\n if config and config.srcDirs:\n for dirname in config.srcDirs:\n dirpath = os.path.join(path, dirname)\n if os.path.isdir(dirpath):\n sys.path.insert(0, dirpath)\n added.append(dirpath)\n return added"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nimports a dotted - name package whose tail is at path and return the full name of the module.", "response": "def importFromPath(self, path, fqname):\n \"\"\"Import a dotted-name package whose tail is at path. In other words,\n given foo.bar and path/to/foo/bar.py, import foo from path/to/foo then\n bar from path/to/foo/bar, returning bar.\n \"\"\"\n # find the base dir of the package\n path_parts = os.path.normpath(os.path.abspath(path)).split(os.sep)\n name_parts = fqname.split('.')\n if path_parts[-1].startswith('__init__'):\n path_parts.pop()\n path_parts = path_parts[:-(len(name_parts))]\n dir_path = os.sep.join(path_parts)\n # then import fqname starting from that dir\n return self.importFromDir(dir_path, fqname)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef importFromDir(self, dir, fqname):\n dir = os.path.normpath(os.path.abspath(dir))\n log.debug(\"Import %s from %s\", fqname, dir)\n\n # FIXME reimplement local per-dir cache?\n \n # special case for __main__\n if fqname == '__main__':\n return sys.modules[fqname]\n \n if self.config.addPaths:\n add_path(dir, self.config)\n \n path = [dir]\n parts = fqname.split('.')\n part_fqname = ''\n mod = parent = fh = None\n\n for part in parts:\n if part_fqname == '':\n part_fqname = part\n else:\n part_fqname = \"%s.%s\" % (part_fqname, part)\n try:\n acquire_lock()\n log.debug(\"find module part %s (%s) in %s\",\n part, part_fqname, path)\n fh, filename, desc = find_module(part, path)\n old = sys.modules.get(part_fqname)\n if old is not None:\n # test modules frequently have name overlap; make sure\n # we get a fresh copy of anything we are trying to load\n # from a new path\n log.debug(\"sys.modules has %s as %s\", part_fqname, old)\n if (self.sameModule(old, filename)\n or (self.config.firstPackageWins and\n getattr(old, '__path__', None))):\n mod = old\n else:\n del sys.modules[part_fqname]\n mod = load_module(part_fqname, fh, filename, desc)\n else:\n mod = load_module(part_fqname, fh, filename, desc)\n finally:\n if fh:\n fh.close()\n release_lock()\n if parent:\n setattr(parent, part, mod)\n if hasattr(mod, '__path__'):\n path = mod.__path__\n parent = mod\n return mod", "response": "Import a module from a directory."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nextract the wininst configuration data from a bdist_wininst. exe file.", "response": "def extract_wininst_cfg(dist_filename):\n \"\"\"Extract configuration data from a bdist_wininst .exe\n\n Returns a ConfigParser.RawConfigParser, or None\n \"\"\"\n f = open(dist_filename,'rb')\n try:\n endrec = zipfile._EndRecData(f)\n if endrec is None:\n return None\n\n prepended = (endrec[9] - endrec[5]) - endrec[6]\n if prepended < 12: # no wininst data here\n return None\n f.seek(prepended-12)\n\n import struct, StringIO, ConfigParser\n tag, cfglen, bmlen = struct.unpack(\"= (2,6):\n null_byte = bytes([0])\n else:\n null_byte = chr(0)\n config = part.split(null_byte, 1)[0]\n # Now the config is in bytes, but on Python 3, it must be\n # unicode for the RawConfigParser, so decode it. Is this the\n # right encoding?\n config = config.decode('ascii')\n cfg.readfp(StringIO.StringIO(config))\n except ConfigParser.Error:\n return None\n if not cfg.has_section('metadata') or not cfg.has_section('Setup'):\n return None\n return cfg\n\n finally:\n f.close()"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a header line from a script text.", "response": "def get_script_header(script_text, executable=sys_executable, wininst=False):\n \"\"\"Create a #! line, getting options (if any) from script_text\"\"\"\n from distutils.command.build_scripts import first_line_re\n\n # first_line_re in Python >=3.1.4 and >=3.2.1 is a bytes pattern.\n if not isinstance(first_line_re.pattern, str):\n first_line_re = re.compile(first_line_re.pattern.decode())\n\n first = (script_text+'\\n').splitlines()[0]\n match = first_line_re.match(first)\n options = ''\n if match:\n options = match.group(1) or ''\n if options: options = ' '+options\n if wininst:\n executable = \"python.exe\"\n else:\n executable = nt_quote_arg(executable)\n hdr = \"#!%(executable)s%(options)s\\n\" % locals()\n if not isascii(hdr):\n # Non-ascii path to sys.executable, use -x to prevent warnings\n if options:\n if options.strip().startswith('-'):\n options = ' -x'+options.strip()[1:]\n # else: punt, we can't do it, let the warning happen anyway\n else:\n options = ' -x'\n executable = fix_jython_executable(executable, options)\n hdr = \"#!%(executable)s%(options)s\\n\" % locals()\n return hdr"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef uncache_zipdir(path):\n from zipimport import _zip_directory_cache as zdc\n _uncache(path, zdc)\n _uncache(path, sys.path_importer_cache)", "response": "Ensure that the importer caches dont have stale info for path"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_sh(executable):\n try:\n fp = open(executable)\n magic = fp.read(2)\n fp.close()\n except (OSError,IOError): return executable\n return magic == '#!'", "response": "Determine if the specified executable is a. sh."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nquote a command line argument according to Windows parsing rules", "response": "def nt_quote_arg(arg):\n \"\"\"Quote a command line argument according to Windows parsing rules\"\"\"\n\n result = []\n needquote = False\n nb = 0\n\n needquote = (\" \" in arg) or (\"\\t\" in arg)\n if needquote:\n result.append('\"')\n\n for c in arg:\n if c == '\\\\':\n nb += 1\n elif c == '\"':\n # double preceding backslashes, then add a \\\"\n result.append('\\\\' * (nb*2) + '\\\\\"')\n nb = 0\n else:\n if nb:\n result.append('\\\\' * nb)\n nb = 0\n result.append(c)\n\n if nb:\n result.append('\\\\' * nb)\n\n if needquote:\n result.append('\\\\' * nb) # double the trailing backslashes\n result.append('\"')\n\n return ''.join(result)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_script_args(dist, executable=sys_executable, wininst=False):\n spec = str(dist.as_requirement())\n header = get_script_header(\"\", executable, wininst)\n for group in 'console_scripts', 'gui_scripts':\n for name, ep in dist.get_entry_map(group).items():\n script_text = (\n \"# EASY-INSTALL-ENTRY-SCRIPT: %(spec)r,%(group)r,%(name)r\\n\"\n \"__requires__ = %(spec)r\\n\"\n \"import sys\\n\"\n \"from pkg_resources import load_entry_point\\n\"\n \"\\n\"\n \"if __name__ == '__main__':\"\n \"\\n\"\n \" sys.exit(\\n\"\n \" load_entry_point(%(spec)r, %(group)r, %(name)r)()\\n\"\n \" )\\n\"\n ) % locals()\n if sys.platform=='win32' or wininst:\n # On Windows/wininst, add a .py extension and an .exe launcher\n if group=='gui_scripts':\n ext, launcher = '-script.pyw', 'gui.exe'\n old = ['.pyw']\n new_header = re.sub('(?i)python.exe','pythonw.exe',header)\n else:\n ext, launcher = '-script.py', 'cli.exe'\n old = ['.py','.pyc','.pyo']\n new_header = re.sub('(?i)pythonw.exe','python.exe',header)\n if is_64bit():\n launcher = launcher.replace(\".\", \"-64.\")\n else:\n launcher = launcher.replace(\".\", \"-32.\")\n if os.path.exists(new_header[2:-1]) or sys.platform!='win32':\n hdr = new_header\n else:\n hdr = header\n yield (name+ext, hdr+script_text, 't', [name+x for x in old])\n yield (\n name+'.exe', resource_string('setuptools', launcher),\n 'b' # write in binary mode\n )\n else:\n # On other platforms, we assume the right thing to do is to\n # just write the stub with no extension.\n yield (name, header+script_text)", "response": "Yields a list of write_script arguments for a distribution s entrypoints."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pseudo_tempname(self):\n try:\n pid = os.getpid()\n except:\n pid = random.randint(0,sys.maxint)\n return os.path.join(self.install_dir, \"test-easy-install-%s\" % pid)", "response": "Return a pseudo - tempdir base in the install directory."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef install_script(self, dist, script_name, script_text, dev_path=None):\n spec = str(dist.as_requirement())\n is_script = is_python_script(script_text, script_name)\n\n def get_template(filename):\n \"\"\"\n There are a couple of template scripts in the package. This\n function loads one of them and prepares it for use.\n\n These templates use triple-quotes to escape variable\n substitutions so the scripts get the 2to3 treatment when build\n on Python 3. The templates cannot use triple-quotes naturally.\n \"\"\"\n raw_bytes = resource_string('setuptools', template_name)\n template_str = raw_bytes.decode('utf-8')\n clean_template = template_str.replace('\"\"\"', '')\n return clean_template\n\n if is_script:\n template_name = 'script template.py'\n if dev_path:\n template_name = template_name.replace('.py', ' (dev).py')\n script_text = (get_script_header(script_text) +\n get_template(template_name) % locals())\n self.write_script(script_name, _to_ascii(script_text), 'b')", "response": "Generate a legacy script wrapper and install it"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef check_conflicts(self, dist):\n\n return dist # XXX temporarily disable until new strategy is stable\n from imp import find_module, get_suffixes\n from glob import glob\n\n blockers = []\n names = dict.fromkeys(dist._get_metadata('top_level.txt')) # XXX private attr\n\n exts = {'.pyc':1, '.pyo':1} # get_suffixes() might leave one out\n for ext,mode,typ in get_suffixes():\n exts[ext] = 1\n\n for path,files in expand_paths([self.install_dir]+self.all_site_dirs):\n for filename in files:\n base,ext = os.path.splitext(filename)\n if base in names:\n if not ext:\n # no extension, check for package\n try:\n f, filename, descr = find_module(base, [path])\n except ImportError:\n continue\n else:\n if f: f.close()\n if filename not in blockers:\n blockers.append(filename)\n elif ext in exts and base!='site': # XXX ugh\n blockers.append(os.path.join(path,filename))\n if blockers:\n self.found_conflicts(dist, blockers)\n\n return dist", "response": "Verify that there are no conflicting old - style packages"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the fetcher options for the current locale.", "response": "def _set_fetcher_options(self, base):\n \"\"\"\n When easy_install is about to run bdist_egg on a source dist, that\n source dist might have 'setup_requires' directives, requiring\n additional fetching. Ensure the fetcher options given to easy_install\n are available to that command as well.\n \"\"\"\n # find the fetch options from easy_install and write them out\n # to the setup.cfg file.\n ei_opts = self.distribution.get_option_dict('easy_install').copy()\n fetch_directives = (\n 'find_links', 'site_dirs', 'index_url', 'optimize',\n 'site_dirs', 'allow_hosts',\n )\n fetch_options = {}\n for key, val in ei_opts.iteritems():\n if key not in fetch_directives: continue\n fetch_options[key.replace('_', '-')] = val[1]\n # create a settings dictionary suitable for `edit_config`\n settings = dict(easy_install=fetch_options)\n cfg_filename = os.path.join(base, 'setup.cfg')\n setopt.edit_config(cfg_filename, settings)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates directories under ~.", "response": "def create_home_path(self):\n \"\"\"Create directories under ~.\"\"\"\n if not self.user:\n return\n home = convert_path(os.path.expanduser(\"~\"))\n for name, path in self.config_vars.iteritems():\n if path.startswith(home) and not os.path.isdir(path):\n self.debug_print(\"os.makedirs('%s', 0700)\" % path)\n os.makedirs(path, 0700)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_archive_file(name):\n archives = (\n '.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.whl'\n )\n ext = splitext(name)[1].lower()\n if ext in archives:\n return True\n return False", "response": "Return True if name is a considered as an archive file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a mutable proxy for the obj.", "response": "def mutable(obj):\n '''\n return a mutable proxy for the `obj`.\n\n all modify on the proxy will not apply on origin object.\n '''\n base_cls = type(obj)\n\n class Proxy(base_cls):\n def __getattribute__(self, name):\n try:\n return super().__getattribute__(name)\n except AttributeError:\n return getattr(obj, name)\n\n update_wrapper(Proxy, base_cls, updated = ())\n return Proxy()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef readonly(obj, *, error_on_set = False):\n '''\n return a readonly proxy for the `obj`.\n\n all modify on the proxy will not apply on origin object.\n '''\n base_cls = type(obj)\n\n class ReadonlyProxy(base_cls):\n def __getattribute__(self, name):\n return getattr(obj, name)\n\n def __setattr__(self, name, value):\n if error_on_set:\n raise AttributeError('cannot set readonly object.')\n\n update_wrapper(ReadonlyProxy, base_cls, updated = ())\n return ReadonlyProxy()", "response": "Returns a readonly proxy for the obj."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a new output node with input and output", "response": "def new_output(output_type=None, output_text=None, output_png=None,\n output_html=None, output_svg=None, output_latex=None, output_json=None,\n output_javascript=None, output_jpeg=None, prompt_number=None,\n etype=None, evalue=None, traceback=None):\n \"\"\"Create a new code cell with input and output\"\"\"\n output = NotebookNode()\n if output_type is not None:\n output.output_type = unicode(output_type)\n\n if output_type != 'pyerr':\n if output_text is not None:\n output.text = unicode(output_text)\n if output_png is not None:\n output.png = bytes(output_png)\n if output_jpeg is not None:\n output.jpeg = bytes(output_jpeg)\n if output_html is not None:\n output.html = unicode(output_html)\n if output_svg is not None:\n output.svg = unicode(output_svg)\n if output_latex is not None:\n output.latex = unicode(output_latex)\n if output_json is not None:\n output.json = unicode(output_json)\n if output_javascript is not None:\n output.javascript = unicode(output_javascript)\n\n if output_type == u'pyout':\n if prompt_number is not None:\n output.prompt_number = int(prompt_number)\n\n if output_type == u'pyerr':\n if etype is not None:\n output.etype = unicode(etype)\n if evalue is not None:\n output.evalue = unicode(evalue)\n if traceback is not None:\n output.traceback = [unicode(frame) for frame in list(traceback)]\n\n return output"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a new code cell with input and output", "response": "def new_code_cell(input=None, prompt_number=None, outputs=None,\n language=u'python', collapsed=False, metadata=None):\n \"\"\"Create a new code cell with input and output\"\"\"\n cell = NotebookNode()\n cell.cell_type = u'code'\n if language is not None:\n cell.language = unicode(language)\n if input is not None:\n cell.input = unicode(input)\n if prompt_number is not None:\n cell.prompt_number = int(prompt_number)\n if outputs is None:\n cell.outputs = []\n else:\n cell.outputs = outputs\n if collapsed is not None:\n cell.collapsed = bool(collapsed)\n cell.metadata = NotebookNode(metadata or {})\n\n return cell"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a new text cell.", "response": "def new_text_cell(cell_type, source=None, rendered=None, metadata=None):\n \"\"\"Create a new text cell.\"\"\"\n cell = NotebookNode()\n # VERSIONHACK: plaintext -> raw\n # handle never-released plaintext name for raw cells\n if cell_type == 'plaintext':\n cell_type = 'raw'\n if source is not None:\n cell.source = unicode(source)\n if rendered is not None:\n cell.rendered = unicode(rendered)\n cell.metadata = NotebookNode(metadata or {})\n cell.cell_type = cell_type\n return cell"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a new section cell with a given integer level.", "response": "def new_heading_cell(source=None, rendered=None, level=1, metadata=None):\n \"\"\"Create a new section cell with a given integer level.\"\"\"\n cell = NotebookNode()\n cell.cell_type = u'heading'\n if source is not None:\n cell.source = unicode(source)\n if rendered is not None:\n cell.rendered = unicode(rendered)\n cell.level = int(level)\n cell.metadata = NotebookNode(metadata or {})\n return cell"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef new_notebook(name=None, metadata=None, worksheets=None):\n nb = NotebookNode()\n nb.nbformat = nbformat\n nb.nbformat_minor = nbformat_minor\n if worksheets is None:\n nb.worksheets = []\n else:\n nb.worksheets = list(worksheets)\n if metadata is None:\n nb.metadata = new_metadata()\n else:\n nb.metadata = NotebookNode(metadata)\n if name is not None:\n nb.metadata.name = unicode(name)\n return nb", "response": "Create a notebook by name id and a list of worksheets."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef new_metadata(name=None, authors=None, license=None, created=None,\n modified=None, gistid=None):\n \"\"\"Create a new metadata node.\"\"\"\n metadata = NotebookNode()\n if name is not None:\n metadata.name = unicode(name)\n if authors is not None:\n metadata.authors = list(authors)\n if created is not None:\n metadata.created = unicode(created)\n if modified is not None:\n metadata.modified = unicode(modified)\n if license is not None:\n metadata.license = unicode(license)\n if gistid is not None:\n metadata.gistid = unicode(gistid)\n return metadata", "response": "Create a new metadata node."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef new_author(name=None, email=None, affiliation=None, url=None):\n author = NotebookNode()\n if name is not None:\n author.name = unicode(name)\n if email is not None:\n author.email = unicode(email)\n if affiliation is not None:\n author.affiliation = unicode(affiliation)\n if url is not None:\n author.url = unicode(url)\n return author", "response": "Create a new author."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nembeds and start an IPython kernel in a given scope.", "response": "def embed_kernel(module=None, local_ns=None, **kwargs):\n \"\"\"Embed and start an IPython kernel in a given scope.\n \n Parameters\n ----------\n module : ModuleType, optional\n The module to load into IPython globals (default: caller)\n local_ns : dict, optional\n The namespace to load into IPython user namespace (default: caller)\n \n kwargs : various, optional\n Further keyword args are relayed to the KernelApp constructor,\n allowing configuration of the Kernel. Will only have an effect\n on the first embed_kernel call for a given process.\n \n \"\"\"\n \n (caller_module, caller_locals) = extract_module_locals(1)\n if module is None:\n module = caller_module\n if local_ns is None:\n local_ns = caller_locals\n \n # Only import .zmq when we really need it\n from .zmq.ipkernel import embed_kernel as real_embed_kernel\n real_embed_kernel(module=module, local_ns=local_ns, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef unquote_filename(name, win32=(sys.platform=='win32')):\n if win32:\n if name.startswith((\"'\", '\"')) and name.endswith((\"'\", '\"')):\n name = name[1:-1]\n return name", "response": "On Windows, remove leading and trailing quotes from filenames."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a valid python filename in the current directory.", "response": "def get_py_filename(name, force_win32=None):\n \"\"\"Return a valid python filename in the current directory.\n\n If the given name is not a file, it adds '.py' and searches again.\n Raises IOError with an informative message if the file isn't found.\n\n On Windows, apply Windows semantics to the filename. In particular, remove\n any quoting that has been applied to it. This option can be forced for\n testing purposes.\n \"\"\"\n\n name = os.path.expanduser(name)\n if force_win32 is None:\n win32 = (sys.platform == 'win32')\n else:\n win32 = force_win32\n name = unquote_filename(name, win32=win32)\n if not os.path.isfile(name) and not name.endswith('.py'):\n name += '.py'\n if os.path.isfile(name):\n return name\n else:\n raise IOError,'File `%r` not found.' % name"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef filefind(filename, path_dirs=None):\n\n # If paths are quoted, abspath gets confused, strip them...\n filename = filename.strip('\"').strip(\"'\")\n # If the input is an absolute path, just check it exists\n if os.path.isabs(filename) and os.path.isfile(filename):\n return filename\n\n if path_dirs is None:\n path_dirs = (\"\",)\n elif isinstance(path_dirs, basestring):\n path_dirs = (path_dirs,)\n\n for path in path_dirs:\n if path == '.': path = os.getcwdu()\n testname = expand_path(os.path.join(path, filename))\n if os.path.isfile(testname):\n return os.path.abspath(testname)\n\n raise IOError(\"File %r does not exist in any of the search paths: %r\" %\n (filename, path_dirs) )", "response": "Find a file in a set of paths and return the absolute path to the file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_home_dir(require_writable=False):\n\n # first, check py2exe distribution root directory for _ipython.\n # This overrides all. Normally does not exist.\n\n if hasattr(sys, \"frozen\"): #Is frozen by py2exe\n if '\\\\library.zip\\\\' in IPython.__file__.lower():#libraries compressed to zip-file\n root, rest = IPython.__file__.lower().split('library.zip')\n else:\n root=os.path.join(os.path.split(IPython.__file__)[0],\"../../\")\n root=os.path.abspath(root).rstrip('\\\\')\n if _writable_dir(os.path.join(root, '_ipython')):\n os.environ[\"IPYKITROOT\"] = root\n return py3compat.cast_unicode(root, fs_encoding)\n \n homedir = os.path.expanduser('~')\n # Next line will make things work even when /home/ is a symlink to\n # /usr/home as it is on FreeBSD, for example\n homedir = os.path.realpath(homedir)\n \n if not _writable_dir(homedir) and os.name == 'nt':\n # expanduser failed, use the registry to get the 'My Documents' folder.\n try:\n import _winreg as wreg\n key = wreg.OpenKey(\n wreg.HKEY_CURRENT_USER,\n \"Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders\"\n )\n homedir = wreg.QueryValueEx(key,'Personal')[0]\n key.Close()\n except:\n pass\n \n if (not require_writable) or _writable_dir(homedir):\n return py3compat.cast_unicode(homedir, fs_encoding)\n else:\n raise HomeDirError('%s is not a writable dir, '\n 'set $HOME environment variable to override' % homedir)", "response": "Get the home directory for the current user."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the XDG_CONFIG_HOME if it is defined and exists else None.", "response": "def get_xdg_dir():\n \"\"\"Return the XDG_CONFIG_HOME, if it is defined and exists, else None.\n\n This is only for non-OS X posix (Linux,Unix,etc.) systems.\n \"\"\"\n\n env = os.environ\n\n if os.name == 'posix' and sys.platform != 'darwin':\n # Linux, Unix, AIX, etc.\n # use ~/.config if empty OR not set\n xdg = env.get(\"XDG_CONFIG_HOME\", None) or os.path.join(get_home_dir(), '.config')\n if xdg and _writable_dir(xdg):\n return py3compat.cast_unicode(xdg, fs_encoding)\n\n return None"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_ipython_dir():\n\n env = os.environ\n pjoin = os.path.join\n\n\n ipdir_def = '.ipython'\n xdg_def = 'ipython'\n\n home_dir = get_home_dir()\n xdg_dir = get_xdg_dir()\n \n # import pdb; pdb.set_trace() # dbg\n if 'IPYTHON_DIR' in env:\n warnings.warn('The environment variable IPYTHON_DIR is deprecated. '\n 'Please use IPYTHONDIR instead.')\n ipdir = env.get('IPYTHONDIR', env.get('IPYTHON_DIR', None))\n if ipdir is None:\n # not set explicitly, use XDG_CONFIG_HOME or HOME\n home_ipdir = pjoin(home_dir, ipdir_def)\n if xdg_dir:\n # use XDG, as long as the user isn't already\n # using $HOME/.ipython and *not* XDG/ipython\n\n xdg_ipdir = pjoin(xdg_dir, xdg_def)\n\n if _writable_dir(xdg_ipdir) or not _writable_dir(home_ipdir):\n ipdir = xdg_ipdir\n\n if ipdir is None:\n # not using XDG\n ipdir = home_ipdir\n\n ipdir = os.path.normpath(os.path.expanduser(ipdir))\n\n if os.path.exists(ipdir) and not _writable_dir(ipdir):\n # ipdir exists, but is not writable\n warnings.warn(\"IPython dir '%s' is not a writable location,\"\n \" using a temp directory.\"%ipdir)\n ipdir = tempfile.mkdtemp()\n elif not os.path.exists(ipdir):\n parent = ipdir.rsplit(os.path.sep, 1)[0]\n if not _writable_dir(parent):\n # ipdir does not exist and parent isn't writable\n warnings.warn(\"IPython parent '%s' is not a writable location,\"\n \" using a temp directory.\"%parent)\n ipdir = tempfile.mkdtemp()\n\n return py3compat.cast_unicode(ipdir, fs_encoding)", "response": "Get the IPython directory for this platform and user."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_ipython_package_dir():\n ipdir = os.path.dirname(IPython.__file__)\n return py3compat.cast_unicode(ipdir, fs_encoding)", "response": "Get the base directory where IPython itself is installed."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinds the path to an IPython module in this version of IPython.", "response": "def get_ipython_module_path(module_str):\n \"\"\"Find the path to an IPython module in this version of IPython.\n\n This will always find the version of the module that is in this importable\n IPython package. This will always return the path to the ``.py``\n version of the module.\n \"\"\"\n if module_str == 'IPython':\n return os.path.join(get_ipython_package_dir(), '__init__.py')\n mod = import_item(module_str)\n the_path = mod.__file__.replace('.pyc', '.py')\n the_path = the_path.replace('.pyo', '.py')\n return py3compat.cast_unicode(the_path, fs_encoding)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef locate_profile(profile='default'):\n from IPython.core.profiledir import ProfileDir, ProfileDirError\n try:\n pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)\n except ProfileDirError:\n # IOError makes more sense when people are expecting a path\n raise IOError(\"Couldn't find profile %r\" % profile)\n return pd.location", "response": "Find the path to the folder associated with a given profile."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nexpands variables and ~names in a string like a shell", "response": "def expand_path(s):\n \"\"\"Expand $VARS and ~names in a string, like a shell\n\n :Examples:\n\n In [2]: os.environ['FOO']='test'\n\n In [3]: expand_path('variable FOO is $FOO')\n Out[3]: 'variable FOO is test'\n \"\"\"\n # This is a pretty subtle hack. When expand user is given a UNC path\n # on Windows (\\\\server\\share$\\%username%), os.path.expandvars, removes\n # the $ to get (\\\\server\\share\\%username%). I think it considered $\n # alone an empty var. But, we need the $ to remains there (it indicates\n # a hidden share).\n if os.name=='nt':\n s = s.replace('$\\\\', 'IPYTHON_TEMP')\n s = os.path.expandvars(os.path.expanduser(s))\n if os.name=='nt':\n s = s.replace('IPYTHON_TEMP', '$\\\\')\n return s"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef target_outdated(target,deps):\n try:\n target_time = os.path.getmtime(target)\n except os.error:\n return 1\n for dep in deps:\n dep_time = os.path.getmtime(dep)\n if dep_time > target_time:\n #print \"For target\",target,\"Dep failed:\",dep # dbg\n #print \"times (dep,tar):\",dep_time,target_time # dbg\n return 1\n return 0", "response": "Determine whether a target is out of date."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmaking an MD5 hash of a file ignoring any differences in line ending characters.", "response": "def filehash(path):\n \"\"\"Make an MD5 hash of a file, ignoring any differences in line\n ending characters.\"\"\"\n with open(path, \"rU\") as f:\n return md5(py3compat.str_to_bytes(f.read())).hexdigest()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef check_for_old_config(ipython_dir=None):\n if ipython_dir is None:\n ipython_dir = get_ipython_dir()\n\n old_configs = ['ipy_user_conf.py', 'ipythonrc', 'ipython_config.py']\n warned = False\n for cfg in old_configs:\n f = os.path.join(ipython_dir, cfg)\n if os.path.exists(f):\n if filehash(f) == old_config_md5.get(cfg, ''):\n os.unlink(f)\n else:\n warnings.warn(\"Found old IPython config file %r (modified by user)\"%f)\n warned = True\n\n if warned:\n warnings.warn(\"\"\"\n The IPython configuration system has changed as of 0.11, and these files will\n be ignored. See http://ipython.github.com/ipython-doc/dev/config for details\n of the new config system.\n To start configuring IPython, do `ipython profile create`, and edit\n `ipython_config.py` in /profile_default.\n If you need to leave the old config files in place for an older version of\n IPython and want to suppress this warning message, set\n `c.InteractiveShellApp.ignore_old_config=True` in the new config.\"\"\")", "response": "Check for old config files and present a warning if they exist."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_security_file(filename, profile='default'):\n # import here, because profiledir also imports from utils.path\n from IPython.core.profiledir import ProfileDir\n try:\n pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)\n except Exception:\n # will raise ProfileDirError if no such profile\n raise IOError(\"Profile %r not found\")\n return filefind(filename, ['.', pd.security_dir])", "response": "Returns the absolute path of a security file given by filename and profile"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update_suggestions_dictionary(request, object):\n if request.user.is_authenticated():\n user = request.user\n content_type = ContentType.objects.get_for_model(type(object))\n try:\n # Check if the user has visited this page before\n ObjectView.objects.get(\n user=user, object_id=object.id, content_type=content_type)\n except:\n ObjectView.objects.create(user=user, content_object=object)\n # Get a list of all the objects a user has visited\n viewed = ObjectView.objects.filter(user=user)\n\n else:\n update_dict_for_guests(request, object, content_type)\n return\n\n if viewed:\n for obj in viewed:\n if content_type == obj.content_type:\n if not exists_in_dictionary(request, object,\n content_type,\n obj, True):\n # Create an entry if it's non existent\n if object.id != obj.object_id:\n ObjectViewDictionary.objects.create(\n current_object=object,\n visited_before_object=obj.content_object)\n if not exists_in_dictionary(request, obj,\n obj.content_type,\n object, False):\n ObjectViewDictionary.objects.create(\n current_object=obj.content_object,\n visited_before_object=object)\n return", "response": "Updates the suggestions dictionary for an object upon visiting its page."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_suggestions_with_size(object, size):\n content_type = ContentType.objects.get_for_model(type(object))\n try:\n return ObjectViewDictionary.objects.filter(\n current_object_id=object.id,\n current_content_type=content_type).extra(\n order_by=['-visits'])[:size]\n except:\n return ObjectViewDictionary.objects.filter(\n current_object_id=object.id,\n current_content_type=content_type).extra(order_by=['-visits'])", "response": "Gets a list with a certain size of suggestions for an object"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_suggestions(object):\n content_type = ContentType.objects.get_for_model(type(object))\n return ObjectViewDictionary.objects.filter(\n current_object_id=object.id,\n current_content_type=content_type).extra(order_by=['-visits'])", "response": "Gets a list of all suggestions for an object"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning this path as a relative path based from the current working directory.", "response": "def relpath(self):\n \"\"\" Return this path as a relative path,\n based from the current working directory.\n \"\"\"\n cwd = self.__class__(os.getcwdu())\n return cwd.relpathto(self)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a list of path objects that match the pattern.", "response": "def glob(self, pattern):\n \"\"\" Return a list of path objects that match the pattern.\n\n pattern - a path relative to this directory, with wildcards.\n\n For example, path('/users').glob('*/bin/*') returns a list\n of all the files users have in their bin directories.\n \"\"\"\n cls = self.__class__\n return [cls(s) for s in glob.glob(unicode(self / pattern))]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_md5(self):\n f = self.open('rb')\n try:\n m = md5()\n while True:\n d = f.read(8192)\n if not d:\n break\n m.update(d)\n finally:\n f.close()\n return m.digest()", "response": "Calculate the md5 hash for this file. This reads through the entire file and returns the md5 hash of the entire file."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef begin(self):\n if not self.available():\n return\n self._create_pfile()\n self.prof = hotshot.Profile(self.pfile)", "response": "Create profile stats file and load profiler."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef configure(self, options, conf):\n if not self.available():\n self.enabled = False\n return\n Plugin.configure(self, options, conf)\n self.conf = conf\n if options.profile_stats_file:\n self.pfile = options.profile_stats_file\n self.clean_stats_file = False\n else:\n self.pfile = None\n self.clean_stats_file = True\n self.fileno = None\n self.sort = options.profile_sort\n self.restrict = tolist(options.profile_restrict)", "response": "Configure the object based on the options."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef report(self, stream):\n log.debug('printing profiler report')\n self.prof.close()\n prof_stats = stats.load(self.pfile)\n prof_stats.sort_stats(self.sort)\n\n # 2.5 has completely different stream handling from 2.4 and earlier.\n # Before 2.5, stats objects have no stream attribute; in 2.5 and later\n # a reference sys.stdout is stored before we can tweak it.\n compat_25 = hasattr(prof_stats, 'stream')\n if compat_25:\n tmp = prof_stats.stream\n prof_stats.stream = stream\n else:\n tmp = sys.stdout\n sys.stdout = stream\n try:\n if self.restrict:\n log.debug('setting profiler restriction to %s', self.restrict)\n prof_stats.print_stats(*self.restrict)\n else:\n prof_stats.print_stats()\n finally:\n if compat_25:\n prof_stats.stream = tmp\n else:\n sys.stdout = tmp", "response": "Print a report of the current profiler state."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef finalize(self, result):\n if not self.available():\n return\n try:\n self.prof.close()\n except AttributeError:\n # TODO: is this trying to catch just the case where not\n # hasattr(self.prof, \"close\")? If so, the function call should be\n # moved out of the try: suite.\n pass\n if self.clean_stats_file:\n if self.fileno:\n try:\n os.close(self.fileno)\n except OSError:\n pass\n try:\n os.unlink(self.pfile)\n except OSError:\n pass\n return None", "response": "Clean up stats file if configured to do so."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhandling the heartbeat command.", "response": "def handle(self, *args, **options):\n \"\"\"Handle CLI command\"\"\"\n try:\n while True:\n Channel(HEARTBEAT_CHANNEL).send({'time':time.time()})\n time.sleep(HEARTBEAT_FREQUENCY)\n except KeyboardInterrupt:\n print(\"Received keyboard interrupt, exiting...\")"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef enable_wx(self, app=None):\n import wx\n \n wx_version = V(wx.__version__).version\n \n if wx_version < [2, 8]:\n raise ValueError(\"requires wxPython >= 2.8, but you have %s\" % wx.__version__)\n \n from IPython.lib.inputhookwx import inputhook_wx\n self.set_inputhook(inputhook_wx)\n self._current_gui = GUI_WX\n import wx\n if app is None:\n app = wx.GetApp()\n if app is None:\n app = wx.App(redirect=False, clearSigInt=False)\n app._in_event_loop = True\n self._apps[GUI_WX] = app\n return app", "response": "Enables event loop integration with wxPython."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndisables event loop integration with wxPython.", "response": "def disable_wx(self):\n \"\"\"Disable event loop integration with wxPython.\n\n This merely sets PyOS_InputHook to NULL.\n \"\"\"\n if self._apps.has_key(GUI_WX):\n self._apps[GUI_WX]._in_event_loop = False\n self.clear_inputhook()"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nenabling event loop integration with PyQt4.", "response": "def enable_qt4(self, app=None):\n \"\"\"Enable event loop integration with PyQt4.\n \n Parameters\n ----------\n app : Qt Application, optional.\n Running application to use. If not given, we probe Qt for an\n existing application object, and create a new one if none is found.\n\n Notes\n -----\n This methods sets the PyOS_InputHook for PyQt4, which allows\n the PyQt4 to integrate with terminal based applications like\n IPython.\n\n If ``app`` is not given we probe for an existing one, and return it if\n found. If no existing app is found, we create an :class:`QApplication`\n as follows::\n\n from PyQt4 import QtCore\n app = QtGui.QApplication(sys.argv)\n \"\"\"\n from IPython.lib.inputhookqt4 import create_inputhook_qt4\n app, inputhook_qt4 = create_inputhook_qt4(self, app)\n self.set_inputhook(inputhook_qt4)\n\n self._current_gui = GUI_QT4\n app._in_event_loop = True\n self._apps[GUI_QT4] = app\n return app"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef disable_qt4(self):\n if self._apps.has_key(GUI_QT4):\n self._apps[GUI_QT4]._in_event_loop = False\n self.clear_inputhook()", "response": "Disable event loop integration with PyQt4."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nenable event loop integration with PyGTK.", "response": "def enable_gtk(self, app=None):\n \"\"\"Enable event loop integration with PyGTK.\n\n Parameters\n ----------\n app : ignored\n Ignored, it's only a placeholder to keep the call signature of all\n gui activation methods consistent, which simplifies the logic of\n supporting magics.\n\n Notes\n -----\n This methods sets the PyOS_InputHook for PyGTK, which allows\n the PyGTK to integrate with terminal based applications like\n IPython.\n \"\"\"\n import gtk\n try:\n gtk.set_interactive(True)\n self._current_gui = GUI_GTK\n except AttributeError:\n # For older versions of gtk, use our own ctypes version\n from IPython.lib.inputhookgtk import inputhook_gtk\n self.set_inputhook(inputhook_gtk)\n self._current_gui = GUI_GTK"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nenables event loop integration with Tk.", "response": "def enable_tk(self, app=None):\n \"\"\"Enable event loop integration with Tk.\n\n Parameters\n ----------\n app : toplevel :class:`Tkinter.Tk` widget, optional.\n Running toplevel widget to use. If not given, we probe Tk for an\n existing one, and create a new one if none is found.\n\n Notes\n -----\n If you have already created a :class:`Tkinter.Tk` object, the only\n thing done by this method is to register with the\n :class:`InputHookManager`, since creating that object automatically\n sets ``PyOS_InputHook``.\n \"\"\"\n self._current_gui = GUI_TK\n if app is None:\n import Tkinter\n app = Tkinter.Tk()\n app.withdraw()\n self._apps[GUI_TK] = app\n return app"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef enable_pyglet(self, app=None):\n import pyglet\n from IPython.lib.inputhookpyglet import inputhook_pyglet\n self.set_inputhook(inputhook_pyglet)\n self._current_gui = GUI_PYGLET\n return app", "response": "Enable event loop integration with pyglet."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nenabling event loop integration with Gtk3.", "response": "def enable_gtk3(self, app=None):\n \"\"\"Enable event loop integration with Gtk3 (gir bindings).\n\n Parameters\n ----------\n app : ignored\n Ignored, it's only a placeholder to keep the call signature of all\n gui activation methods consistent, which simplifies the logic of\n supporting magics.\n\n Notes\n -----\n This methods sets the PyOS_InputHook for Gtk3, which allows\n the Gtk3 to integrate with terminal based applications like\n IPython.\n \"\"\"\n from IPython.lib.inputhookgtk3 import inputhook_gtk3\n self.set_inputhook(inputhook_gtk3)\n self._current_gui = GUI_GTK"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a partitioner in the engine namespace", "response": "def setup_partitioner(index, num_procs, gnum_cells, parts):\n \"\"\"create a partitioner in the engine namespace\"\"\"\n global partitioner\n p = MPIRectPartitioner2D(my_id=index, num_procs=num_procs)\n p.redim(global_num_cells=gnum_cells, num_parts=parts)\n p.prepare_communication()\n # put the partitioner into the global namespace:\n partitioner=p"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wave_saver(u, x, y, t):\n global u_hist\n global t_hist\n t_hist.append(t)\n u_hist.append(1.0*u)", "response": "save the wave log"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntake a string of history ranges and returns a list of tuples.", "response": "def extract_hist_ranges(ranges_str):\n \"\"\"Turn a string of history ranges into 3-tuples of (session, start, stop).\n\n Examples\n --------\n list(extract_input_ranges(\"~8/5-~7/4 2\"))\n [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]\n \"\"\"\n for range_str in ranges_str.split():\n rmatch = range_re.match(range_str)\n if not rmatch:\n continue\n start = int(rmatch.group(\"start\"))\n end = rmatch.group(\"end\")\n end = int(end) if end else start+1 # If no end specified, get (a, a+1)\n if rmatch.group(\"sep\") == \"-\": # 1-3 == 1:4 --> [1, 2, 3]\n end += 1\n startsess = rmatch.group(\"startsess\") or \"0\"\n endsess = rmatch.group(\"endsess\") or startsess\n startsess = int(startsess.replace(\"~\",\"-\"))\n endsess = int(endsess.replace(\"~\",\"-\"))\n assert endsess >= startsess\n\n if endsess == startsess:\n yield (startsess, start, end)\n continue\n # Multiple sessions in one range:\n yield (startsess, start, None)\n for sess in range(startsess+1, endsess):\n yield (sess, 1, None)\n yield (endsess, 1, end)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconnect to the database and create tables if necessary.", "response": "def init_db(self):\n \"\"\"Connect to the database, and create tables if necessary.\"\"\"\n # use detect_types so that timestamps return datetime objects\n self.db = sqlite3.connect(self.hist_file,\n detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)\n self.db.execute(\"\"\"CREATE TABLE IF NOT EXISTS sessions (session integer\n primary key autoincrement, start timestamp,\n end timestamp, num_cmds integer, remark text)\"\"\")\n self.db.execute(\"\"\"CREATE TABLE IF NOT EXISTS history\n (session integer, line integer, source text, source_raw text,\n PRIMARY KEY (session, line))\"\"\")\n # Output history is optional, but ensure the table's there so it can be\n # enabled later.\n self.db.execute(\"\"\"CREATE TABLE IF NOT EXISTS output_history\n (session integer, line integer, output text,\n PRIMARY KEY (session, line))\"\"\")\n self.db.commit()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\npreparing and runs a SQL query for the history database.", "response": "def _run_sql(self, sql, params, raw=True, output=False):\n \"\"\"Prepares and runs an SQL query for the history database.\n\n Parameters\n ----------\n sql : str\n Any filtering expressions to go after SELECT ... FROM ...\n params : tuple\n Parameters passed to the SQL query (to replace \"?\")\n raw, output : bool\n See :meth:`get_range`\n\n Returns\n -------\n Tuples as :meth:`get_range`\n \"\"\"\n toget = 'source_raw' if raw else 'source'\n sqlfrom = \"history\"\n if output:\n sqlfrom = \"history LEFT JOIN output_history USING (session, line)\"\n toget = \"history.%s, output_history.output\" % toget\n cur = self.db.execute(\"SELECT session, line, %s FROM %s \" %\\\n (toget, sqlfrom) + sql, params)\n if output: # Regroup into 3-tuples, and parse JSON\n return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)\n return cur"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_session_info(self, session=0):\n\n if session <= 0:\n session += self.session_number\n\n query = \"SELECT * from sessions where session == ?\"\n return self.db.execute(query, (session,)).fetchone()", "response": "get info about a session"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_tail(self, n=10, raw=True, output=False, include_latest=False):\n self.writeout_cache()\n if not include_latest:\n n += 1\n cur = self._run_sql(\"ORDER BY session DESC, line DESC LIMIT ?\",\n (n,), raw=raw, output=output)\n if not include_latest:\n return reversed(list(cur)[1:])\n return reversed(list(cur))", "response": "Get the last n lines from the history database."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef search(self, pattern=\"*\", raw=True, search_raw=True,\n output=False):\n \"\"\"Search the database using unix glob-style matching (wildcards\n * and ?).\n\n Parameters\n ----------\n pattern : str\n The wildcarded pattern to match when searching\n search_raw : bool\n If True, search the raw input, otherwise, the parsed input\n raw, output : bool\n See :meth:`get_range`\n\n Returns\n -------\n Tuples as :meth:`get_range`\n \"\"\"\n tosearch = \"source_raw\" if search_raw else \"source\"\n if output:\n tosearch = \"history.\" + tosearch\n self.writeout_cache()\n return self._run_sql(\"WHERE %s GLOB ?\" % tosearch, (pattern,),\n raw=raw, output=output)", "response": "Search the database using unix glob - style matching."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_range(self, session, start=1, stop=None, raw=True,output=False):\n if stop:\n lineclause = \"line >= ? AND line < ?\"\n params = (session, start, stop)\n else:\n lineclause = \"line>=?\"\n params = (session, start)\n\n return self._run_sql(\"WHERE session==? AND %s\"\"\" % lineclause,\n params, raw=raw, output=output)", "response": "Retrieve input from previous log file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_range_by_str(self, rangestr, raw=True, output=False):\n for sess, s, e in extract_hist_ranges(rangestr):\n for line in self.get_range(sess, s, e, raw=raw, output=output):\n yield line", "response": "Get lines of history from a string of ranges."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_hist_file_name(self, profile=None):\n profile_dir = self.shell.profile_dir.location\n return os.path.join(profile_dir, 'history.sqlite')", "response": "Get the default history file name based on the profile.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef new_session(self, conn=None):\n if conn is None:\n conn = self.db\n \n with conn:\n cur = conn.execute(\"\"\"INSERT INTO sessions VALUES (NULL, ?, NULL,\n NULL, \"\") \"\"\", (datetime.datetime.now(),))\n self.session_number = cur.lastrowid", "response": "Get a new session number."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef end_session(self):\n self.writeout_cache()\n with self.db:\n self.db.execute(\"\"\"UPDATE sessions SET end=?, num_cmds=? WHERE\n session==?\"\"\", (datetime.datetime.now(),\n len(self.input_hist_parsed)-1, self.session_number))\n self.session_number = 0", "response": "Close the database session."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngive the current session a name in the history database.", "response": "def name_session(self, name):\n \"\"\"Give the current session a name in the history database.\"\"\"\n with self.db:\n self.db.execute(\"UPDATE sessions SET remark=? WHERE session==?\",\n (name, self.session_number))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef reset(self, new_session=True):\n self.output_hist.clear()\n # The directory history can't be completely empty\n self.dir_hist[:] = [os.getcwdu()]\n \n if new_session:\n if self.session_number:\n self.end_session()\n self.input_hist_parsed[:] = [\"\"]\n self.input_hist_raw[:] = [\"\"]\n self.new_session()", "response": "Clear the session history releasing all object references and and\n optionally open a new session."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_range_session(self, start=1, stop=None, raw=True, output=False):\n input_hist = self.input_hist_raw if raw else self.input_hist_parsed\n \n n = len(input_hist)\n if start < 0:\n start += n\n if not stop or (stop > n):\n stop = n\n elif stop < 0:\n stop += n\n \n for i in range(start, stop):\n if output:\n line = (input_hist[i], self.output_hist_reprs.get(i))\n else:\n line = input_hist[i]\n yield (0, i, line)", "response": "Get input and output history from the current session. Called by get_range and takes similar parameters."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nretrieve input by session.", "response": "def get_range(self, session=0, start=1, stop=None, raw=True,output=False):\n \"\"\"Retrieve input by session.\n \n Parameters\n ----------\n session : int\n Session number to retrieve. The current session is 0, and negative\n numbers count back from current session, so -1 is previous session.\n start : int\n First line to retrieve.\n stop : int\n End of line range (excluded from output itself). If None, retrieve\n to the end of the session.\n raw : bool\n If True, return untranslated input\n output : bool\n If True, attempt to include output. This will be 'real' Python\n objects for the current session, or text reprs from previous\n sessions if db_log_output was enabled at the time. Where no output\n is found, None is used.\n \n Returns\n -------\n An iterator over the desired lines. Each line is a 3-tuple, either\n (session, line, input) if output is False, or\n (session, line, (input, output)) if output is True.\n \"\"\"\n if session <= 0:\n session += self.session_number\n if session==self.session_number: # Current session\n return self._get_range_session(start, stop, raw, output)\n return super(HistoryManager, self).get_range(session, start, stop, raw,\n output)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nstore source and raw input in history and create input cache variables _i1 _i2... dynamically", "response": "def store_inputs(self, line_num, source, source_raw=None):\n \"\"\"Store source and raw input in history and create input cache\n variables _i*.\n\n Parameters\n ----------\n line_num : int\n The prompt number of this input.\n\n source : str\n Python input.\n\n source_raw : str, optional\n If given, this is the raw input without any IPython transformations\n applied to it. If not given, ``source`` is used.\n \"\"\"\n if source_raw is None:\n source_raw = source\n source = source.rstrip('\\n')\n source_raw = source_raw.rstrip('\\n')\n\n # do not store exit/quit commands\n if self._exit_re.match(source_raw.strip()):\n return\n\n self.input_hist_parsed.append(source)\n self.input_hist_raw.append(source_raw)\n\n with self.db_input_cache_lock:\n self.db_input_cache.append((line_num, source, source_raw))\n # Trigger to flush cache and write to DB.\n if len(self.db_input_cache) >= self.db_cache_size:\n self.save_flag.set()\n\n # update the auto _i variables\n self._iii = self._ii\n self._ii = self._i\n self._i = self._i00\n self._i00 = source_raw\n\n # hackish access to user namespace to create _i1,_i2... dynamically\n new_i = '_i%s' % line_num\n to_main = {'_i': self._i,\n '_ii': self._ii,\n '_iii': self._iii,\n new_i : self._i00 }\n\n self.shell.push(to_main, interactive=False)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef store_output(self, line_num):\n if (not self.db_log_output) or (line_num not in self.output_hist_reprs):\n return\n output = self.output_hist_reprs[line_num]\n\n with self.db_output_cache_lock:\n self.db_output_cache.append((line_num, output))\n if self.db_cache_size <= 1:\n self.save_flag.set()", "response": "Stores the output of a given line number into the database."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef writeout_cache(self, conn=None):\n if conn is None:\n conn = self.db\n\n with self.db_input_cache_lock:\n try:\n self._writeout_input_cache(conn)\n except sqlite3.IntegrityError:\n self.new_session(conn)\n print(\"ERROR! Session/line number was not unique in\",\n \"database. History logging moved to new session\",\n self.session_number)\n try:\n # Try writing to the new session. If this fails, don't\n # recurse\n self._writeout_input_cache(conn)\n except sqlite3.IntegrityError:\n pass\n finally:\n self.db_input_cache = []\n\n with self.db_output_cache_lock:\n try:\n self._writeout_output_cache(conn)\n except sqlite3.IntegrityError:\n print(\"!! Session/line number for output was not unique\",\n \"in database. Output will not be stored.\")\n finally:\n self.db_output_cache = []", "response": "Write any entries in the cache to the database."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_boot_time():\n f = open('/proc/stat', 'r')\n try:\n for line in f:\n if line.startswith('btime'):\n return float(line.strip().split()[1])\n raise RuntimeError(\"line not found\")\n finally:\n f.close()", "response": "Return system boot time in seconds"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_num_cpus():\n # we try to determine num CPUs by using different approaches.\n # SC_NPROCESSORS_ONLN seems to be the safer and it is also\n # used by multiprocessing module\n try:\n return os.sysconf(\"SC_NPROCESSORS_ONLN\")\n except ValueError:\n # as a second fallback we try to parse /proc/cpuinfo\n num = 0\n f = open('/proc/cpuinfo', 'r')\n try:\n lines = f.readlines()\n finally:\n f.close()\n for line in lines:\n if line.lower().startswith('processor'):\n num += 1\n\n # unknown format (e.g. amrel/sparc architectures), see:\n # http://code.google.com/p/psutil/issues/detail?id=200\n # try to parse /proc/stat as a last resort\n if num == 0:\n f = open('/proc/stat', 'r')\n try:\n lines = f.readlines()\n finally:\n f.close()\n search = re.compile('cpu\\d')\n for line in lines:\n line = line.split(' ')[0]\n if search.match(line):\n num += 1\n\n if num == 0:\n raise RuntimeError(\"can't determine number of CPUs\")\n return num", "response": "Return the number of CPUs on the system"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_system_cpu_times():\n f = open('/proc/stat', 'r')\n try:\n values = f.readline().split()\n finally:\n f.close()\n\n values = values[1:8]\n values = tuple([float(x) / _CLOCK_TICKS for x in values])\n return nt_sys_cputimes(*values[:7])", "response": "Return a named tuple representing the following CPU times for the system."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of namedtuple representing the CPU times for every CPU available on the system.", "response": "def get_system_per_cpu_times():\n \"\"\"Return a list of namedtuple representing the CPU times\n for every CPU available on the system.\n \"\"\"\n cpus = []\n f = open('/proc/stat', 'r')\n # get rid of the first line who refers to system wide CPU stats\n try:\n f.readline()\n for line in f.readlines():\n if line.startswith('cpu'):\n values = line.split()[1:8]\n values = tuple([float(x) / _CLOCK_TICKS for x in values])\n entry = nt_sys_cputimes(*values[:7])\n cpus.append(entry)\n return cpus\n finally:\n f.close()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef disk_partitions(all=False):\n phydevs = []\n f = open(\"/proc/filesystems\", \"r\")\n try:\n for line in f:\n if not line.startswith(\"nodev\"):\n phydevs.append(line.strip())\n finally:\n f.close()\n\n retlist = []\n partitions = _psutil_linux.get_disk_partitions()\n for partition in partitions:\n device, mountpoint, fstype, opts = partition\n if device == 'none':\n device = ''\n if not all:\n if device == '' or fstype not in phydevs:\n continue\n ntuple = nt_partition(device, mountpoint, fstype, opts)\n retlist.append(ntuple)\n return retlist", "response": "Return mounted disk partitions as a list of nameduples"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_system_users():\n retlist = []\n rawlist = _psutil_linux.get_system_users()\n for item in rawlist:\n user, tty, hostname, tstamp, user_process = item\n # XXX the underlying C function includes entries about\n # system boot, run level and others. We might want\n # to use them in the future.\n if not user_process:\n continue\n if hostname == ':0.0':\n hostname = 'localhost'\n nt = nt_user(user, tty or None, hostname, tstamp)\n retlist.append(nt)\n return retlist", "response": "Return currently connected users as a list of namedtuples."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of PIDs currently running on the system.", "response": "def get_pid_list():\n \"\"\"Returns a list of PIDs currently running on the system.\"\"\"\n pids = [int(x) for x in os.listdir('/proc') if x.isdigit()]\n return pids"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns statistics for every network interface installed on the system as a dict of raw tuples.", "response": "def network_io_counters():\n \"\"\"Return network I/O statistics for every network interface\n installed on the system as a dict of raw tuples.\n \"\"\"\n f = open(\"/proc/net/dev\", \"r\")\n try:\n lines = f.readlines()\n finally:\n f.close()\n\n retdict = {}\n for line in lines[2:]:\n colon = line.find(':')\n assert colon > 0, line\n name = line[:colon].strip()\n fields = line[colon+1:].strip().split()\n bytes_recv = int(fields[0])\n packets_recv = int(fields[1])\n errin = int(fields[2])\n dropin = int(fields[2])\n bytes_sent = int(fields[8])\n packets_sent = int(fields[9])\n errout = int(fields[10])\n dropout = int(fields[11])\n retdict[name] = (bytes_sent, bytes_recv, packets_sent, packets_recv,\n errin, errout, dropin, dropout)\n return retdict"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef disk_io_counters():\n # man iostat states that sectors are equivalent with blocks and\n # have a size of 512 bytes since 2.4 kernels. This value is\n # needed to calculate the amount of disk I/O in bytes.\n SECTOR_SIZE = 512\n\n # determine partitions we want to look for\n partitions = []\n f = open(\"/proc/partitions\", \"r\")\n try:\n lines = f.readlines()[2:]\n finally:\n f.close()\n for line in lines:\n _, _, _, name = line.split()\n if name[-1].isdigit():\n partitions.append(name)\n #\n retdict = {}\n f = open(\"/proc/diskstats\", \"r\")\n try:\n lines = f.readlines()\n finally:\n f.close()\n for line in lines:\n _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = \\\n line.split()[:11]\n if name in partitions:\n rbytes = int(rbytes) * SECTOR_SIZE\n wbytes = int(wbytes) * SECTOR_SIZE\n reads = int(reads)\n writes = int(writes)\n # TODO: times are expressed in milliseconds while OSX/BSD has\n # these expressed in nanoseconds; figure this out.\n rtime = int(rtime)\n wtime = int(wtime)\n retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime)\n return retdict", "response": "Return disk I/O statistics for every disk installed on the\n system as a dict of raw tuples."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef wrap_exceptions(callable):\n def wrapper(self, *args, **kwargs):\n try:\n return callable(self, *args, **kwargs)\n except EnvironmentError:\n # ENOENT (no such file or directory) gets raised on open().\n # ESRCH (no such process) can get raised on read() if\n # process is gone in meantime.\n err = sys.exc_info()[1]\n if err.errno in (errno.ENOENT, errno.ESRCH):\n raise NoSuchProcess(self.pid, self._process_name)\n if err.errno in (errno.EPERM, errno.EACCES):\n raise AccessDenied(self.pid, self._process_name)\n raise\n return wrapper", "response": "Wrap exceptions in a try / except clause and translate EPERM and EPERM in NoSuchProcess or AccessDenied exceptions."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_memory_maps(self):\n f = None\n try:\n f = open(\"/proc/%s/smaps\" % self.pid)\n first_line = f.readline()\n current_block = [first_line]\n\n def get_blocks():\n data = {}\n for line in f:\n fields = line.split(None, 5)\n if len(fields) >= 5:\n yield (current_block.pop(), data)\n current_block.append(line)\n else:\n data[fields[0]] = int(fields[1]) * 1024\n yield (current_block.pop(), data)\n\n if first_line: # smaps file can be empty\n for header, data in get_blocks():\n hfields = header.split(None, 5)\n try:\n addr, perms, offset, dev, inode, path = hfields\n except ValueError:\n addr, perms, offset, dev, inode, path = hfields + ['']\n if not path:\n path = '[anon]'\n else:\n path = path.strip()\n yield (addr, perms, path,\n data['Rss:'],\n data['Size:'],\n data.get('Pss:', 0),\n data['Shared_Clean:'], data['Shared_Clean:'],\n data['Private_Clean:'], data['Private_Dirty:'],\n data['Referenced:'],\n data['Anonymous:'],\n data['Swap:'])\n f.close()\n except EnvironmentError:\n # XXX - Can't use wrap_exceptions decorator as we're\n # returning a generator; this probably needs some\n # refactoring in order to avoid this code duplication.\n if f is not None:\n f.close()\n err = sys.exc_info()[1]\n if err.errno in (errno.ENOENT, errno.ESRCH):\n raise NoSuchProcess(self.pid, self._process_name)\n if err.errno in (errno.EPERM, errno.EACCES):\n raise AccessDenied(self.pid, self._process_name)\n raise\n except:\n if f is not None:\n f.close()\n raise", "response": "Return a generator of nameduples for the process s mapped memory regions."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of namedtuples that can be used to create a new process.", "response": "def get_connections(self, kind='inet'):\n \"\"\"Return connections opened by process as a list of namedtuples.\n The kind parameter filters for connections that fit the following\n criteria:\n\n Kind Value Number of connections using\n inet IPv4 and IPv6\n inet4 IPv4\n inet6 IPv6\n tcp TCP\n tcp4 TCP over IPv4\n tcp6 TCP over IPv6\n udp UDP\n udp4 UDP over IPv4\n udp6 UDP over IPv6\n all the sum of all the possible families and protocols\n \"\"\"\n # Note: in case of UNIX sockets we're only able to determine the\n # local bound path while the remote endpoint is not retrievable:\n # http://goo.gl/R3GHM\n inodes = {}\n # os.listdir() is gonna raise a lot of access denied\n # exceptions in case of unprivileged user; that's fine:\n # lsof does the same so it's unlikely that we can to better.\n for fd in os.listdir(\"/proc/%s/fd\" % self.pid):\n try:\n inode = os.readlink(\"/proc/%s/fd/%s\" % (self.pid, fd))\n except OSError:\n continue\n if inode.startswith('socket:['):\n # the process is using a socket\n inode = inode[8:][:-1]\n inodes[inode] = fd\n\n if not inodes:\n # no connections for this process\n return []\n\n def process(file, family, type_):\n retlist = []\n try:\n f = open(file, 'r')\n except IOError:\n # IPv6 not supported on this platform\n err = sys.exc_info()[1]\n if err.errno == errno.ENOENT and file.endswith('6'):\n return []\n else:\n raise\n try:\n f.readline() # skip the first line\n for line in f:\n # IPv4 / IPv6\n if family in (socket.AF_INET, socket.AF_INET6):\n _, laddr, raddr, status, _, _, _, _, _, inode = \\\n line.split()[:10]\n if inode in inodes:\n laddr = self._decode_address(laddr, family)\n raddr = self._decode_address(raddr, family)\n if type_ == socket.SOCK_STREAM:\n status = _TCP_STATES_TABLE[status]\n else:\n status = \"\"\n fd = int(inodes[inode])\n conn = nt_connection(fd, family, type_, laddr,\n raddr, status)\n retlist.append(conn)\n elif family == socket.AF_UNIX:\n tokens = line.split()\n _, _, _, _, type_, _, inode = tokens[0:7]\n if inode in inodes:\n\n if len(tokens) == 8:\n path = tokens[-1]\n else:\n path = \"\"\n fd = int(inodes[inode])\n type_ = int(type_)\n conn = nt_connection(fd, family, type_, path,\n None, \"\")\n retlist.append(conn)\n else:\n raise ValueError(family)\n return retlist\n finally:\n f.close()\n\n tcp4 = (\"tcp\" , socket.AF_INET , socket.SOCK_STREAM)\n tcp6 = (\"tcp6\", socket.AF_INET6, socket.SOCK_STREAM)\n udp4 = (\"udp\" , socket.AF_INET , socket.SOCK_DGRAM)\n udp6 = (\"udp6\", socket.AF_INET6, socket.SOCK_DGRAM)\n unix = (\"unix\", socket.AF_UNIX, None)\n\n tmap = {\n \"all\" : (tcp4, tcp6, udp4, udp6, unix),\n \"tcp\" : (tcp4, tcp6),\n \"tcp4\" : (tcp4,),\n \"tcp6\" : (tcp6,),\n \"udp\" : (udp4, udp6),\n \"udp4\" : (udp4,),\n \"udp6\" : (udp6,),\n \"unix\" : (unix,),\n \"inet\" : (tcp4, tcp6, udp4, udp6),\n \"inet4\": (tcp4, udp4),\n \"inet6\": (tcp6, udp6),\n }\n if kind not in tmap:\n raise ValueError(\"invalid %r kind argument; choose between %s\"\n % (kind, ', '.join([repr(x) for x in tmap])))\n ret = []\n for f, family, type_ in tmap[kind]:\n ret += process(\"/proc/net/%s\" % f, family, type_)\n # raise NSP if the process disappeared on us\n os.stat('/proc/%s' % self.pid)\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\naccepts an IP address as displayed in / proc / net/* and convert it into a human readable form.", "response": "def _decode_address(addr, family):\n \"\"\"Accept an \"ip:port\" address as displayed in /proc/net/*\n and convert it into a human readable form, like:\n\n \"0500000A:0016\" -> (\"10.0.0.5\", 22)\n \"0000000000000000FFFF00000100007F:9E49\" -> (\"::ffff:127.0.0.1\", 40521)\n\n The IP address portion is a little or big endian four-byte\n hexadecimal number; that is, the least significant byte is listed\n first, so we need to reverse the order of the bytes to convert it\n to an IP address.\n The port is represented as a two-byte hexadecimal number.\n\n Reference:\n http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html\n \"\"\"\n ip, port = addr.split(':')\n port = int(port, 16)\n if PY3:\n ip = ip.encode('ascii')\n # this usually refers to a local socket in listen mode with\n # no end-points connected\n if not port:\n return ()\n if family == socket.AF_INET:\n # see: http://code.google.com/p/psutil/issues/detail?id=201\n if sys.byteorder == 'little':\n ip = socket.inet_ntop(family, base64.b16decode(ip)[::-1])\n else:\n ip = socket.inet_ntop(family, base64.b16decode(ip))\n else: # IPv6\n # old version - let's keep it, just in case...\n #ip = ip.decode('hex')\n #return socket.inet_ntop(socket.AF_INET6,\n # ''.join(ip[i:i+4][::-1] for i in xrange(0, 16, 4)))\n ip = base64.b16decode(ip)\n # see: http://code.google.com/p/psutil/issues/detail?id=201\n if sys.byteorder == 'little':\n ip = socket.inet_ntop(socket.AF_INET6,\n struct.pack('>4I', *struct.unpack('<4I', ip)))\n else:\n ip = socket.inet_ntop(socket.AF_INET6,\n struct.pack('<4I', *struct.unpack('<4I', ip)))\n return (ip, port)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmake a nice string representation of a pair of numbers.", "response": "def nice_pair(pair):\n \"\"\"Make a nice string representation of a pair of numbers.\n\n If the numbers are equal, just return the number, otherwise return the pair\n with a dash between them, indicating the range.\n\n \"\"\"\n start, end = pair\n if start == end:\n return \"%d\" % start\n else:\n return \"%d-%d\" % (start, end)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a string summarizing the call stack.", "response": "def short_stack():\n \"\"\"Return a string summarizing the call stack.\"\"\"\n stack = inspect.stack()[:0:-1]\n return \"\\n\".join([\"%30s : %s @%d\" % (t[3],t[1],t[2]) for t in stack])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncombining a list of regexes into one that matches any of them.", "response": "def join_regex(regexes):\n \"\"\"Combine a list of regexes into one that matches any of them.\"\"\"\n if len(regexes) > 1:\n return \"|\".join([\"(%s)\" % r for r in regexes])\n elif regexes:\n return regexes[0]\n else:\n return \"\""} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef file_be_gone(path):\n try:\n os.remove(path)\n except OSError:\n _, e, _ = sys.exc_info()\n if e.errno != errno.ENOENT:\n raise", "response": "Remove a file and don t get annoyed if it doesn t exist."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self, v):\n self.md5.update(to_bytes(str(type(v))))\n if isinstance(v, string_class):\n self.md5.update(to_bytes(v))\n elif v is None:\n pass\n elif isinstance(v, (int, float)):\n self.md5.update(to_bytes(str(v)))\n elif isinstance(v, (tuple, list)):\n for e in v:\n self.update(e)\n elif isinstance(v, dict):\n keys = v.keys()\n for k in sorted(keys):\n self.update(k)\n self.update(v[k])\n else:\n for k in dir(v):\n if k.startswith('__'):\n continue\n a = getattr(v, k)\n if inspect.isroutine(a):\n continue\n self.update(k)\n self.update(a)", "response": "Add v to the hash recursively if needed."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update_profiles(self):\n for path in [get_ipython_dir(), os.getcwdu()]:\n for profile in list_profiles_in(path):\n pd = self.get_profile_dir(profile, path)\n if profile not in self.profiles:\n self.log.debug(\"Adding cluster profile '%s'\" % profile)\n self.profiles[profile] = {\n 'profile': profile,\n 'profile_dir': pd,\n 'status': 'stopped'\n }", "response": "List all profiles in the ipython_dir and cwd."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef start_cluster(self, profile, n=None):\n self.check_profile(profile)\n data = self.profiles[profile]\n if data['status'] == 'running':\n raise web.HTTPError(409, u'cluster already running')\n cl, esl, default_n = self.build_launchers(data['profile_dir'])\n n = n if n is not None else default_n\n def clean_data():\n data.pop('controller_launcher',None)\n data.pop('engine_set_launcher',None)\n data.pop('n',None)\n data['status'] = 'stopped'\n def engines_stopped(r):\n self.log.debug('Engines stopped')\n if cl.running:\n cl.stop()\n clean_data()\n esl.on_stop(engines_stopped)\n def controller_stopped(r):\n self.log.debug('Controller stopped')\n if esl.running:\n esl.stop()\n clean_data()\n cl.on_stop(controller_stopped)\n\n dc = ioloop.DelayedCallback(lambda: cl.start(), 0, self.loop)\n dc.start()\n dc = ioloop.DelayedCallback(lambda: esl.start(n), 1000*self.delay, self.loop)\n dc.start()\n\n self.log.debug('Cluster started')\n data['controller_launcher'] = cl\n data['engine_set_launcher'] = esl\n data['n'] = n\n data['status'] = 'running'\n return self.profile_info(profile)", "response": "Start a cluster for a given profile."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nstopping a cluster for a given profile.", "response": "def stop_cluster(self, profile):\n \"\"\"Stop a cluster for a given profile.\"\"\"\n self.check_profile(profile)\n data = self.profiles[profile]\n if data['status'] == 'stopped':\n raise web.HTTPError(409, u'cluster not running')\n data = self.profiles[profile]\n cl = data['controller_launcher']\n esl = data['engine_set_launcher']\n if cl.running:\n cl.stop()\n if esl.running:\n esl.stop()\n # Return a temp info dict, the real one is updated in the on_stop\n # logic above.\n result = {\n 'profile': data['profile'],\n 'profile_dir': data['profile_dir'],\n 'status': 'stopped'\n }\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _propagate_url(self):\n if self.url:\n iface = self.url.split('://',1)\n if len(iface) == 2:\n self.transport,iface = iface\n iface = iface.split(':')\n self.ip = iface[0]\n if iface[1]:\n self.regport = int(iface[1])", "response": "Ensure self. url contains full transport. interface. port"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _find_cmd(cmd):\n try:\n from win32api import SearchPath\n except ImportError:\n raise ImportError('you need to have pywin32 installed for this to work')\n else:\n PATH = os.environ['PATH']\n extensions = ['.exe', '.com', '.bat', '.py']\n path = None\n for ext in extensions:\n try:\n path = SearchPath(PATH, cmd + ext)[0]\n except:\n pass\n if path is None:\n raise OSError(\"command %r not found\" % cmd)\n else:\n return path", "response": "Find the full path to a. bat or. py using the win32api module."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef system(cmd):\n # The controller provides interactivity with both\n # stdin and stdout\n #import _process_win32_controller\n #_process_win32_controller.system(cmd)\n\n with AvoidUNCPath() as path:\n if path is not None:\n cmd = '\"pushd %s &&\"%s' % (path, cmd)\n return process_handler(cmd, _system_body)", "response": "Win32 version of os. system that works with network shares."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getoutput(cmd):\n\n with AvoidUNCPath() as path:\n if path is not None:\n cmd = '\"pushd %s &&\"%s' % (path, cmd)\n out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)\n\n if out is None:\n out = b''\n return py3compat.bytes_to_str(out)", "response": "Returns the standard output of executing a command in a system shell."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a partitioner in the engine namespace", "response": "def setup_partitioner(comm, addrs, index, num_procs, gnum_cells, parts):\n \"\"\"create a partitioner in the engine namespace\"\"\"\n global partitioner\n p = ZMQRectPartitioner2D(comm, addrs, my_id=index, num_procs=num_procs)\n p.redim(global_num_cells=gnum_cells, num_parts=parts)\n p.prepare_communication()\n # put the partitioner into the global namespace:\n partitioner=p"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of tuples for the given codes", "response": "def get_translations(codes):\n \"\"\" Returns a list of (code, translation) tuples for codes \"\"\"\n codes = codes or self.codes\n return self._get_priority_translations(priority, codes)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a list of tuples for codes sorted by priority", "response": "def get_translations_sorted(codes):\n \"\"\" Returns a sorted list of (code, translation) tuples for codes \"\"\"\n codes = codes or self.codes\n return self._get_priority_translations(priority, codes)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_priority_translations(priority, codes):\n priority = priority or self.priority\n codes = codes or self.codes\n return self._get_priority_translations(priority, codes)", "response": "Returns a list of tuples for priority and codes"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find_code_units(self, morfs):\n morfs = morfs or self.coverage.data.measured_files()\n file_locator = self.coverage.file_locator\n self.code_units = code_unit_factory(morfs, file_locator)\n\n if self.config.include:\n patterns = prep_patterns(self.config.include)\n filtered = []\n for cu in self.code_units:\n for pattern in patterns:\n if fnmatch.fnmatch(cu.filename, pattern):\n filtered.append(cu)\n break\n self.code_units = filtered\n\n if self.config.omit:\n patterns = prep_patterns(self.config.omit)\n filtered = []\n for cu in self.code_units:\n for pattern in patterns:\n if fnmatch.fnmatch(cu.filename, pattern):\n break\n else:\n filtered.append(cu)\n self.code_units = filtered\n\n self.code_units.sort()", "response": "Find the code units we ll report on."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef report_files(self, report_fn, morfs, directory=None):\n self.find_code_units(morfs)\n\n if not self.code_units:\n raise CoverageException(\"No data to report.\")\n\n self.directory = directory\n if self.directory and not os.path.exists(self.directory):\n os.makedirs(self.directory)\n\n for cu in self.code_units:\n try:\n report_fn(cu, self.coverage._analyze(cu))\n except NoSource:\n if not self.config.ignore_errors:\n raise\n except NotPython:\n # Only report errors for .py files, and only if we didn't\n # explicitly suppress those errors.\n if cu.should_be_python() and not self.config.ignore_errors:\n raise", "response": "Run a reporting function on a number of relative morfs."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrap a test decorator so as to properly replicate metadata of the decorated function, including nose's additional stuff (namely, setup and teardown).", "response": "def make_decorator(func):\n \"\"\"\n Wraps a test decorator so as to properly replicate metadata\n of the decorated function, including nose's additional stuff\n (namely, setup and teardown).\n \"\"\"\n def decorate(newfunc):\n if hasattr(func, 'compat_func_name'):\n name = func.compat_func_name\n else:\n name = func.__name__\n newfunc.__dict__ = func.__dict__\n newfunc.__doc__ = func.__doc__\n newfunc.__module__ = func.__module__\n if not hasattr(newfunc, 'compat_co_firstlineno'):\n newfunc.compat_co_firstlineno = func.func_code.co_firstlineno\n try:\n newfunc.__name__ = name\n except TypeError:\n # can't set func name in 2.3\n newfunc.compat_func_name = name\n return newfunc\n return decorate"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntesting must raise one of expected exceptions to pass. Example use:: @raises(TypeError, ValueError) def test_raises_type_error(): raise TypeError(\"This test passes\") @raises(Exception) def test_that_fails_by_passing(): pass If you want to test many assertions about exceptions in a single test, you may want to use `assert_raises` instead.", "response": "def raises(*exceptions):\n \"\"\"Test must raise one of expected exceptions to pass.\n\n Example use::\n\n @raises(TypeError, ValueError)\n def test_raises_type_error():\n raise TypeError(\"This test passes\")\n\n @raises(Exception)\n def test_that_fails_by_passing():\n pass\n\n If you want to test many assertions about exceptions in a single test,\n you may want to use `assert_raises` instead.\n \"\"\"\n valid = ' or '.join([e.__name__ for e in exceptions])\n def decorate(func):\n name = func.__name__\n def newfunc(*arg, **kw):\n try:\n func(*arg, **kw)\n except exceptions:\n pass\n except:\n raise\n else:\n message = \"%s() did not raise %s\" % (name, valid)\n raise AssertionError(message)\n newfunc = make_decorator(func)(newfunc)\n return newfunc\n return decorate"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncall pdb. set_trace in the calling frame first restoring sys. stdout to the real output stream.", "response": "def set_trace():\n \"\"\"Call pdb.set_trace in the calling frame, first restoring\n sys.stdout to the real output stream. Note that sys.stdout is NOT\n reset to whatever it was before the call once pdb is done!\n \"\"\"\n import pdb\n import sys\n stdout = sys.stdout\n sys.stdout = sys.__stdout__\n pdb.Pdb().set_trace(sys._getframe().f_back)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntest must finish within specified time limit to pass. Example use:: @timed(.1) def test_that_fails(): time.sleep(.2)", "response": "def timed(limit):\n \"\"\"Test must finish within specified time limit to pass.\n\n Example use::\n\n @timed(.1)\n def test_that_fails():\n time.sleep(.2)\n \"\"\"\n def decorate(func):\n def newfunc(*arg, **kw):\n start = time.time()\n func(*arg, **kw)\n end = time.time()\n if end - start > limit:\n raise TimeExpired(\"Time limit (%s) exceeded\" % limit)\n newfunc = make_decorator(func)(newfunc)\n return newfunc\n return decorate"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef with_setup(setup=None, teardown=None):\n def decorate(func, setup=setup, teardown=teardown):\n if setup:\n if hasattr(func, 'setup'):\n _old_s = func.setup\n def _s():\n setup()\n _old_s()\n func.setup = _s\n else:\n func.setup = setup\n if teardown:\n if hasattr(func, 'teardown'):\n _old_t = func.teardown\n def _t():\n _old_t()\n teardown()\n func.teardown = _t\n else:\n func.teardown = teardown\n return func\n return decorate", "response": "Decorator to add setup and teardown methods to a test function."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef init_gui_pylab(self):\n if self.gui or self.pylab:\n shell = self.shell\n try:\n if self.pylab:\n gui, backend = pylabtools.find_gui_and_backend(self.pylab)\n self.log.info(\"Enabling GUI event loop integration, \"\n \"toolkit=%s, pylab=%s\" % (gui, self.pylab))\n shell.enable_pylab(gui, import_all=self.pylab_import_all)\n else:\n self.log.info(\"Enabling GUI event loop integration, \"\n \"toolkit=%s\" % self.gui)\n shell.enable_gui(self.gui)\n except Exception:\n self.log.warn(\"GUI event loop or pylab initialization failed\")\n self.shell.showtraceback()", "response": "Enable GUI event loop integration taking pylab into account."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrun the pre - flight code specified via exec_lines", "response": "def init_code(self):\n \"\"\"run the pre-flight code, specified via exec_lines\"\"\"\n self._run_startup_files()\n self._run_exec_lines()\n self._run_exec_files()\n self._run_cmd_line_code()\n self._run_module()\n \n # flush output, so itwon't be attached to the first cell\n sys.stdout.flush()\n sys.stderr.flush()\n \n # Hide variables defined here from %who etc.\n self.shell.user_ns_hidden.update(self.shell.user_ns)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _run_exec_lines(self):\n if not self.exec_lines:\n return\n try:\n self.log.debug(\"Running code from IPythonApp.exec_lines...\")\n for line in self.exec_lines:\n try:\n self.log.info(\"Running code in user namespace: %s\" %\n line)\n self.shell.run_cell(line, store_history=False)\n except:\n self.log.warn(\"Error in executing line in user \"\n \"namespace: %s\" % line)\n self.shell.showtraceback()\n except:\n self.log.warn(\"Unknown error in handling IPythonApp.exec_lines:\")\n self.shell.showtraceback()", "response": "Run lines of code in IPythonApp. exec_lines in the user s namespace."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _run_startup_files(self):\n startup_dir = self.profile_dir.startup_dir\n startup_files = glob.glob(os.path.join(startup_dir, '*.py'))\n startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))\n if not startup_files:\n return\n \n self.log.debug(\"Running startup files from %s...\", startup_dir)\n try:\n for fname in sorted(startup_files):\n self._exec_file(fname)\n except:\n self.log.warn(\"Unknown error in handling startup files:\")\n self.shell.showtraceback()", "response": "Run files from profile startup directory"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _run_exec_files(self):\n if not self.exec_files:\n return\n\n self.log.debug(\"Running files in IPythonApp.exec_files...\")\n try:\n for fname in self.exec_files:\n self._exec_file(fname)\n except:\n self.log.warn(\"Unknown error in handling IPythonApp.exec_files:\")\n self.shell.showtraceback()", "response": "Run files from IPythonApp. exec_files"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun code or file specified at the command - line.", "response": "def _run_cmd_line_code(self):\n \"\"\"Run code or file specified at the command-line\"\"\"\n if self.code_to_run:\n line = self.code_to_run\n try:\n self.log.info(\"Running code given at command line (c=): %s\" %\n line)\n self.shell.run_cell(line, store_history=False)\n except:\n self.log.warn(\"Error in executing line in user namespace: %s\" %\n line)\n self.shell.showtraceback()\n\n # Like Python itself, ignore the second if the first of these is present\n elif self.file_to_run:\n fname = self.file_to_run\n try:\n self._exec_file(fname)\n except:\n self.log.warn(\"Error in executing file in user namespace: %s\" %\n fname)\n self.shell.showtraceback()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun the module specified at the command - line.", "response": "def _run_module(self):\n \"\"\"Run module specified at the command-line.\"\"\"\n if self.module_to_run:\n # Make sure that the module gets a proper sys.argv as if it were\n # run using `python -m`.\n save_argv = sys.argv\n sys.argv = [sys.executable] + self.extra_args\n try:\n self.shell.safe_run_module(self.module_to_run,\n self.shell.user_ns)\n finally:\n sys.argv = save_argv"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef generic(func):\n\n _sentinel = object()\n\n def _by_class(*args, **kw):\n cls = args[0].__class__\n for t in type(cls.__name__, (cls,object), {}).__mro__:\n f = _gbt(t, _sentinel)\n if f is not _sentinel:\n return f(*args, **kw)\n else:\n return func(*args, **kw)\n\n _by_type = {object: func}\n try:\n _by_type[InstanceType] = _by_class\n except NameError: # Python 3\n pass\n \n _gbt = _by_type.get\n\n def when_type(*types):\n \"\"\"Decorator to add a method that will be called for the given types\"\"\"\n for t in types:\n if not isinstance(t, classtypes):\n raise TypeError(\n \"%r is not a type or class\" % (t,)\n )\n def decorate(f):\n for t in types:\n if _by_type.setdefault(t,f) is not f:\n raise TypeError(\n \"%r already has method for type %r\" % (func, t)\n )\n return f\n return decorate\n\n\n\n\n _by_object = {}\n _gbo = _by_object.get\n\n def when_object(*obs):\n \"\"\"Decorator to add a method to be called for the given object(s)\"\"\"\n def decorate(f):\n for o in obs:\n if _by_object.setdefault(id(o), (o,f))[1] is not f:\n raise TypeError(\n \"%r already has method for object %r\" % (func, o)\n )\n return f\n return decorate\n\n\n def dispatch(*args, **kw):\n f = _gbo(id(args[0]), _sentinel)\n if f is _sentinel:\n for t in type(args[0]).__mro__:\n f = _gbt(t, _sentinel)\n if f is not _sentinel:\n return f(*args, **kw)\n else:\n return func(*args, **kw)\n else:\n return f[1](*args, **kw)\n\n dispatch.__name__ = func.__name__\n dispatch.__dict__ = func.__dict__.copy()\n dispatch.__doc__ = func.__doc__\n dispatch.__module__ = func.__module__\n\n dispatch.when_type = when_type\n dispatch.when_object = when_object\n dispatch.default = func\n dispatch.has_object = lambda o: id(o) in _by_object\n dispatch.has_type = lambda t: t in _by_type\n return dispatch", "response": "Create a simple generic function that returns a new object that is a copy of the original object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef data_filename(fname, pkgdir=\"\"):\n for static_dir in STATIC_PATH:\n static_filename = os.path.join(static_dir, fname)\n if os.path.exists(static_filename):\n return static_filename\n if pkgdir:\n static_filename = os.path.join(static_dir, pkgdir, fname)\n if os.path.exists(static_filename):\n return static_filename\n raise CoverageException(\"Couldn't find static file %r\" % fname)", "response": "Return the path to a data file of ours."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef data(fname):\n data_file = open(data_filename(fname))\n try:\n return data_file.read()\n finally:\n data_file.close()", "response": "Return the contents of a data file of ours."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef report(self, morfs):\n assert self.config.html_dir, \"must give a directory for html reporting\"\n\n # Read the status data.\n self.status.read(self.config.html_dir)\n\n # Check that this run used the same settings as the last run.\n m = Hasher()\n m.update(self.config)\n these_settings = m.digest()\n if self.status.settings_hash() != these_settings:\n self.status.reset()\n self.status.set_settings_hash(these_settings)\n\n # The user may have extra CSS they want copied.\n if self.config.extra_css:\n self.extra_css = os.path.basename(self.config.extra_css)\n\n # Process all the files.\n self.report_files(self.html_file, morfs, self.config.html_dir)\n\n if not self.files:\n raise CoverageException(\"No data to report.\")\n\n # Write the index file.\n self.index_file()\n\n self.make_local_static_report_files()\n\n return self.totals.pc_covered", "response": "Generate an HTML report for a list of morfs."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef make_local_static_report_files(self):\n # The files we provide must always be copied.\n for static, pkgdir in self.STATIC_FILES:\n shutil.copyfile(\n data_filename(static, pkgdir),\n os.path.join(self.directory, static)\n )\n\n # The user may have extra CSS they want copied.\n if self.extra_css:\n shutil.copyfile(\n self.config.extra_css,\n os.path.join(self.directory, self.extra_css)\n )", "response": "Make local instances of static files for HTML report."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting html to fname properly encoded.", "response": "def write_html(self, fname, html):\n \"\"\"Write `html` to `fname`, properly encoded.\"\"\"\n fout = open(fname, \"wb\")\n try:\n fout.write(html.encode('ascii', 'xmlcharrefreplace'))\n finally:\n fout.close()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef file_hash(self, source, cu):\n m = Hasher()\n m.update(source)\n self.coverage.data.add_to_hash(cu.filename, m)\n return m.digest()", "response": "Compute a hash that changes if the file needs to be re - reported."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef html_file(self, cu, analysis):\n source_file = cu.source_file()\n try:\n source = source_file.read()\n finally:\n source_file.close()\n\n # Find out if the file on disk is already correct.\n flat_rootname = cu.flat_rootname()\n this_hash = self.file_hash(source, cu)\n that_hash = self.status.file_hash(flat_rootname)\n if this_hash == that_hash:\n # Nothing has changed to require the file to be reported again.\n self.files.append(self.status.index_info(flat_rootname))\n return\n\n self.status.set_file_hash(flat_rootname, this_hash)\n\n # If need be, determine the encoding of the source file. We use it\n # later to properly write the HTML.\n if sys.version_info < (3, 0):\n encoding = source_encoding(source)\n # Some UTF8 files have the dreaded UTF8 BOM. If so, junk it.\n if encoding.startswith(\"utf-8\") and source[:3] == \"\\xef\\xbb\\xbf\":\n source = source[3:]\n encoding = \"utf-8\"\n\n # Get the numbers for this file.\n nums = analysis.numbers\n\n if self.arcs:\n missing_branch_arcs = analysis.missing_branch_arcs()\n\n # These classes determine which lines are highlighted by default.\n c_run = \"run hide_run\"\n c_exc = \"exc\"\n c_mis = \"mis\"\n c_par = \"par \" + c_run\n\n lines = []\n\n for lineno, line in enumerate(source_token_lines(source)):\n lineno += 1 # 1-based line numbers.\n # Figure out how to mark this line.\n line_class = []\n annotate_html = \"\"\n annotate_title = \"\"\n if lineno in analysis.statements:\n line_class.append(\"stm\")\n if lineno in analysis.excluded:\n line_class.append(c_exc)\n elif lineno in analysis.missing:\n line_class.append(c_mis)\n elif self.arcs and lineno in missing_branch_arcs:\n line_class.append(c_par)\n annlines = []\n for b in missing_branch_arcs[lineno]:\n if b < 0:\n annlines.append(\"exit\")\n else:\n annlines.append(str(b))\n annotate_html = \"   \".join(annlines)\n if len(annlines) > 1:\n annotate_title = \"no jumps to these line numbers\"\n elif len(annlines) == 1:\n annotate_title = \"no jump to this line number\"\n elif lineno in analysis.statements:\n line_class.append(c_run)\n\n # Build the HTML for the line\n html = []\n for tok_type, tok_text in line:\n if tok_type == \"ws\":\n html.append(escape(tok_text))\n else:\n tok_html = escape(tok_text) or ' '\n html.append(\n \"%s\" % (tok_type, tok_html)\n )\n\n lines.append({\n 'html': ''.join(html),\n 'number': lineno,\n 'class': ' '.join(line_class) or \"pln\",\n 'annotate': annotate_html,\n 'annotate_title': annotate_title,\n })\n\n # Write the HTML page for this file.\n html = spaceless(self.source_tmpl.render({\n 'c_exc': c_exc, 'c_mis': c_mis, 'c_par': c_par, 'c_run': c_run,\n 'arcs': self.arcs, 'extra_css': self.extra_css,\n 'cu': cu, 'nums': nums, 'lines': lines,\n }))\n\n if sys.version_info < (3, 0):\n html = html.decode(encoding)\n\n html_filename = flat_rootname + \".html\"\n html_path = os.path.join(self.directory, html_filename)\n self.write_html(html_path, html)\n\n # Save this file's information for the index file.\n index_info = {\n 'nums': nums,\n 'html_filename': html_filename,\n 'name': cu.name,\n }\n self.files.append(index_info)\n self.status.set_index_info(flat_rootname, index_info)", "response": "Generate an HTML file for one source file."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites the index. html file for this report.", "response": "def index_file(self):\n \"\"\"Write the index.html file for this report.\"\"\"\n index_tmpl = Templite(\n data(\"index.html\"), self.template_globals\n )\n\n self.totals = sum([f['nums'] for f in self.files])\n\n html = index_tmpl.render({\n 'arcs': self.arcs,\n 'extra_css': self.extra_css,\n 'files': self.files,\n 'totals': self.totals,\n })\n\n if sys.version_info < (3, 0):\n html = html.decode(\"utf-8\")\n self.write_html(\n os.path.join(self.directory, \"index.html\"),\n html\n )\n\n # Write the latest hashes for next time.\n self.status.write(self.directory)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef read(self, directory):\n usable = False\n try:\n status_file = os.path.join(directory, self.STATUS_FILE)\n fstatus = open(status_file, \"rb\")\n try:\n status = pickle.load(fstatus)\n finally:\n fstatus.close()\n except (IOError, ValueError):\n usable = False\n else:\n usable = True\n if status['format'] != self.STATUS_FORMAT:\n usable = False\n elif status['version'] != coverage.__version__:\n usable = False\n\n if usable:\n self.files = status['files']\n self.settings = status['settings']\n else:\n self.reset()", "response": "Read the last status in directory."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite the current status to directory.", "response": "def write(self, directory):\n \"\"\"Write the current status to `directory`.\"\"\"\n status_file = os.path.join(directory, self.STATUS_FILE)\n status = {\n 'format': self.STATUS_FORMAT,\n 'version': coverage.__version__,\n 'settings': self.settings,\n 'files': self.files,\n }\n fout = open(status_file, \"wb\")\n try:\n pickle.dump(status, fout)\n finally:\n fout.close()"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns from an iterable a list of all the unique elements in the input but maintaining the order in which they appear.", "response": "def uniq_stable(elems):\n \"\"\"uniq_stable(elems) -> list\n\n Return from an iterable, a list of all the unique elements in the input,\n but maintaining the order in which they first appear.\n\n A naive solution to this problem which just makes a dictionary with the\n elements as keys fails to respect the stability condition, since\n dictionaries are unsorted by nature.\n\n Note: All elements in the input must be valid dictionary keys for this\n routine to work, as it internally uses a dictionary for efficiency\n reasons.\"\"\"\n\n unique = []\n unique_dict = {}\n for nn in elems:\n if nn not in unique_dict:\n unique.append(nn)\n unique_dict[nn] = None\n return unique"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sort_compare(lst1, lst2, inplace=1):\n if not inplace:\n lst1 = lst1[:]\n lst2 = lst2[:]\n lst1.sort(); lst2.sort()\n return lst1 == lst2", "response": "Sort and compare two lists."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef list2dict(lst):\n\n dic = {}\n for k,v in lst: dic[k] = v\n return dic", "response": "Takes a list of key value pairs and turns it into a dict."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntaking a list and turns it into a dict. Much slower than list2dict but more versatile.", "response": "def list2dict2(lst, default=''):\n \"\"\"Takes a list and turns it into a dict.\n Much slower than list2dict, but more versatile. This version can take\n lists with sublists of arbitrary length (including sclars).\"\"\"\n\n dic = {}\n for elem in lst:\n if type(elem) in (types.ListType,types.TupleType):\n size = len(elem)\n if size == 0:\n pass\n elif size == 1:\n dic[elem] = default\n else:\n k,v = elem[0], elem[1:]\n if len(v) == 1: v = v[0]\n dic[k] = v\n else:\n dic[elem] = default\n return dic"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_slice(seq, start=0, stop=None, step=1):\n if stop == None:\n stop = len(seq)\n item = lambda i: seq[i]\n return map(item,xrange(start,stop,step))", "response": "Get a slice of a sequence with variable step. Specify start stop step."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef chop(seq, size):\n chunk = lambda i: seq[i:i+size]\n return map(chunk,xrange(0,len(seq),size))", "response": "Chop a sequence into chunks of the given size."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread configuration from setup. cfg.", "response": "def read_config():\n \"\"\"Read configuration from setup.cfg.\"\"\"\n # XXX modifies global state, which is kind of evil\n config = ConfigParser.ConfigParser()\n config.read(['setup.cfg'])\n if not config.has_section('check-manifest'):\n return\n if (config.has_option('check-manifest', 'ignore-default-rules')\n and config.getboolean('check-manifest', 'ignore-default-rules')):\n del IGNORE[:]\n if config.has_option('check-manifest', 'ignore'):\n patterns = [p.strip() for p in config.get('check-manifest',\n 'ignore').splitlines()]\n IGNORE.extend(p for p in patterns if p)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef read_manifest():\n # XXX modifies global state, which is kind of evil\n if not os.path.isfile('MANIFEST.in'):\n return\n with open('MANIFEST.in') as manifest:\n contents = manifest.read()\n ignore, ignore_regexps = _get_ignore_from_manifest(contents)\n IGNORE.extend(ignore)\n IGNORE_REGEXPS.extend(ignore_regexps)", "response": "Read existing configuration from MANIFEST. in."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncompiling a glob pattern into a regexp.", "response": "def _glob_to_regexp(pat):\n \"\"\"Compile a glob pattern into a regexp.\n\n We need to do this because fnmatch allows * to match /, which we\n don't want. E.g. an MANIFEST.in exclude of 'dirname/*css' should\n match 'dirname/foo.css' but not 'dirname/subdir/bar.css'.\n \"\"\"\n pat = fnmatch.translate(pat)\n # Note that distutils in Python 2.6 has a buggy glob_to_re in\n # distutils.filelist -- it converts '*.cfg' to '[^/]*cfg' instead\n # of '[^\\\\]*cfg' on Windows.\n sep = r'\\\\\\\\' if os.path.sep == '\\\\' else os.path.sep\n return re.sub(r'((?\")\n assert offset > -1, 'Invalid HTML string: no tag.'\n html = ('\\n'+\n html[offset+6:])\n\n html = fix_html(html)\n f.write(IMG_RE.sub(lambda x: image_tag(x, path = None, format = \"svg\"),\n html))", "response": "Exports the contents of the Qt HTML to XHTML with inline SVGs."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwrapping for ensuring image_tag returns utf8 - encoded str on Python 2", "response": "def ensure_utf8(image_tag):\n \"\"\"wrapper for ensuring image_tag returns utf8-encoded str on Python 2\"\"\"\n if py3compat.PY3:\n # nothing to do on Python 3\n return image_tag\n \n def utf8_image_tag(*args, **kwargs):\n s = image_tag(*args, **kwargs)\n if isinstance(s, unicode):\n s = s.encode('utf8')\n return s\n return utf8_image_tag"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntransform a Qt - generated HTML string into a standards - compliant one.", "response": "def fix_html(html):\n \"\"\" Transforms a Qt-generated HTML string into a standards-compliant one.\n\n Parameters:\n -----------\n html : str,\n A utf-8 encoded Python string containing the Qt HTML.\n \"\"\"\n # A UTF-8 declaration is needed for proper rendering of some characters\n # (e.g., indented commands) when viewing exported HTML on a local system\n # (i.e., without seeing an encoding declaration in an HTTP header).\n # C.f. http://www.w3.org/International/O-charset for details.\n offset = html.find('')\n if offset > -1:\n html = (html[:offset+6]+\n '\\n\\n'+\n html[offset+6:])\n\n # Replace empty paragraphs tags with line breaks.\n html = re.sub(EMPTY_P_RE, '
', html)\n\n return html"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef export(self):\n parent = self.control.window()\n dialog = QtGui.QFileDialog(parent, 'Save as...')\n dialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)\n filters = [\n 'HTML with PNG figures (*.html *.htm)',\n 'XHTML with inline SVG figures (*.xhtml *.xml)'\n ]\n dialog.setNameFilters(filters)\n if self.filename:\n dialog.selectFile(self.filename)\n root,ext = os.path.splitext(self.filename)\n if ext.lower() in ('.xml', '.xhtml'):\n dialog.selectNameFilter(filters[-1])\n\n if dialog.exec_():\n self.filename = dialog.selectedFiles()[0]\n choice = dialog.selectedNameFilter()\n html = self.control.document().toHtml().encode('utf-8')\n\n # Configure the exporter.\n if choice.startswith('XHTML'):\n exporter = export_xhtml\n else:\n # If there are PNGs, decide how to export them.\n inline = self.inline_png\n if inline is None and IMG_RE.search(html):\n dialog = QtGui.QDialog(parent)\n dialog.setWindowTitle('Save as...')\n layout = QtGui.QVBoxLayout(dialog)\n msg = \"Exporting HTML with PNGs\"\n info = \"Would you like inline PNGs (single large html \" \\\n \"file) or external image files?\"\n checkbox = QtGui.QCheckBox(\"&Don't ask again\")\n checkbox.setShortcut('D')\n ib = QtGui.QPushButton(\"&Inline\")\n ib.setShortcut('I')\n eb = QtGui.QPushButton(\"&External\")\n eb.setShortcut('E')\n box = QtGui.QMessageBox(QtGui.QMessageBox.Question,\n dialog.windowTitle(), msg)\n box.setInformativeText(info)\n box.addButton(ib, QtGui.QMessageBox.NoRole)\n box.addButton(eb, QtGui.QMessageBox.YesRole)\n layout.setSpacing(0)\n layout.addWidget(box)\n layout.addWidget(checkbox)\n dialog.setLayout(layout)\n dialog.show()\n reply = box.exec_()\n dialog.hide()\n inline = (reply == 0)\n if checkbox.checkState():\n # Don't ask anymore; always use this choice.\n self.inline_png = inline\n exporter = lambda h, f, i: export_html(h, f, i, inline)\n\n # Perform the export!\n try:\n return exporter(html, self.filename, self.image_tag)\n except Exception, e:\n msg = \"Error exporting HTML to %s\\n\" % self.filename + str(e)\n reply = QtGui.QMessageBox.warning(parent, 'Error', msg,\n QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)\n\n return None", "response": "Displays a dialog for exporting HTML with PNGs and inline SVGs."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_unique_or_none(klass, *args, **kwargs):\n try:\n return klass.objects.get(*args, **kwargs)\n except klass.DoesNotExist:\n return None\n except klass.MultipleObjectsReturned:\n return None\n return None", "response": "Returns a unique instance of klass or None if no such instance exists."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_or_create_unique(klass, defaults, unique_fields):\n if not unique_fields or not defaults:\n return (None, False)\n\n uniqueness_query = {k: v for k, v in defaults.items() if k in unique_fields}\n\n try:\n with transaction.atomic():\n instance, created = klass.objects.get_or_create(defaults=defaults, **uniqueness_query)\n except IntegrityError:\n try:\n instance, created = klass(**defaults).save(), True\n except Exception as err:\n return (None, False)\n except Exception as err:\n return (None, False)\n\n if instance and not created:\n for attr, value in defaults.items():\n if getattr(instance, attr):\n setattr(instance, attr, value)\n instance.save()\n\n return (instance, created)", "response": "Get or create a new object of the given class based on the given dictionary of unique fields."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nbuilding a query for includes in a text search.", "response": "def get_query_includes(tokenized_terms, search_fields):\n \"\"\"\n Builds a query for included terms in a text search.\n \"\"\"\n query = None\n for term in tokenized_terms:\n or_query = None\n for field_name in search_fields:\n q = Q(**{\"%s__icontains\" % field_name: term})\n if or_query is None:\n or_query = q\n else:\n or_query = or_query | q\n if query is None:\n query = or_query\n else:\n query = query & or_query\n return query"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_text_query(query_string, search_fields):\n include_terms, exclude_terms = get_text_tokenizer(query_string)\n include_q = get_query_includes(include_terms, search_fields)\n exclude_q = get_query_excludes(exclude_terms, search_fields)\n query = None\n if include_q and exclude_q:\n query = include_q & ~exclude_q\n elif not exclude_q:\n query = include_q\n else:\n query = ~exclude_q\n return query", "response": "Builds a query for both included and excluded terms in a text search."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a Query for if date_field is within number of days ago.", "response": "def get_date_greater_query(days, date_field):\n \"\"\"\n Query for if date_field is within number of \"days\" ago.\n \"\"\"\n query = None\n days = get_integer(days)\n if days:\n past = get_days_ago(days)\n query = Q(**{\"%s__gte\" % date_field: past.isoformat()})\n return query"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a Query for if date_field is within number of days from now.", "response": "def get_date_less_query(days, date_field):\n \"\"\"\n Query for if date_field is within number of \"days\" from now.\n \"\"\"\n query = None\n days = get_integer(days)\n if days:\n future = get_days_from_now(days)\n query = Q(**{\"%s__lte\" % date_field: future.isoformat()})\n return query"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_null_or_blank_query(field=None):\n if not field:\n return field\n null_q = get_null_query(field)\n blank_q = get_blank_query(field)\n return (null_q | blank_q)", "response": "Get null or blank query for a given field."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef case_insensitive(self, fields_dict):\n if hasattr(self.model, 'CASE_INSENSITIVE_FIELDS'):\n for field in self.model.CASE_INSENSITIVE_FIELDS:\n if field in fields_dict:\n fields_dict[field + '__iexact'] = fields_dict[field]\n del fields_dict[field]", "response": "Converts queries to case insensitive for special fields."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef attr(*args, **kwargs):\n def wrap_ob(ob):\n for name in args:\n setattr(ob, name, True)\n for name, value in kwargs.iteritems():\n setattr(ob, name, value)\n return ob\n return wrap_ob", "response": "Decorator that adds attributes to classes or functions of the object passed in."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_method_attr(method, cls, attr_name, default = False):\n Missing = object()\n value = getattr(method, attr_name, Missing)\n if value is Missing and cls is not None:\n value = getattr(cls, attr_name, Missing)\n if value is Missing:\n return default\n return value", "response": "Look up an attribute on a method s object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nregistering command line options", "response": "def options(self, parser, env):\n \"\"\"Register command line options\"\"\"\n parser.add_option(\"-a\", \"--attr\",\n dest=\"attr\", action=\"append\",\n default=env.get('NOSE_ATTR'),\n metavar=\"ATTR\",\n help=\"Run only tests that have attributes \"\n \"specified by ATTR [NOSE_ATTR]\")\n # disable in < 2.4: eval can't take needed args\n if compat_24:\n parser.add_option(\"-A\", \"--eval-attr\",\n dest=\"eval_attr\", metavar=\"EXPR\", action=\"append\",\n default=env.get('NOSE_EVAL_ATTR'),\n help=\"Run only tests for whose attributes \"\n \"the Python expression EXPR evaluates \"\n \"to True [NOSE_EVAL_ATTR]\")"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconfiguring the plugin and system based on selected options.", "response": "def configure(self, options, config):\n \"\"\"Configure the plugin and system, based on selected options.\n\n attr and eval_attr may each be lists.\n\n self.attribs will be a list of lists of tuples. In that list, each\n list is a group of attributes, all of which must match for the rule to\n match.\n \"\"\"\n self.attribs = []\n\n # handle python eval-expression parameter\n if compat_24 and options.eval_attr:\n eval_attr = tolist(options.eval_attr)\n for attr in eval_attr:\n # \"\"\n # -> eval(expr) in attribute context must be True\n def eval_in_context(expr, obj, cls):\n return eval(expr, None, ContextHelper(obj, cls))\n self.attribs.append([(attr, eval_in_context)])\n\n # attribute requirements are a comma separated list of\n # 'key=value' pairs\n if options.attr:\n std_attr = tolist(options.attr)\n for attr in std_attr:\n # all attributes within an attribute group must match\n attr_group = []\n for attrib in attr.strip().split(\",\"):\n # don't die on trailing comma\n if not attrib:\n continue\n items = attrib.split(\"=\", 1)\n if len(items) > 1:\n # \"name=value\"\n # -> 'str(obj.name) == value' must be True\n key, value = items\n else:\n key = items[0]\n if key[0] == \"!\":\n # \"!name\"\n # 'bool(obj.name)' must be False\n key = key[1:]\n value = False\n else:\n # \"name\"\n # -> 'bool(obj.name)' must be True\n value = True\n attr_group.append((key, value))\n self.attribs.append(attr_group)\n if self.attribs:\n self.enabled = True"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef validateAttrib(self, method, cls = None):\n # TODO: is there a need for case-sensitive value comparison?\n any = False\n for group in self.attribs:\n match = True\n for key, value in group:\n attr = get_method_attr(method, cls, key)\n if callable(value):\n if not value(key, method, cls):\n match = False\n break\n elif value is True:\n # value must exist and be True\n if not bool(attr):\n match = False\n break\n elif value is False:\n # value must not exist or be False\n if bool(attr):\n match = False\n break\n elif type(attr) in (list, tuple):\n # value must be found in the list attribute\n if not str(value).lower() in [str(x).lower()\n for x in attr]:\n match = False\n break\n else:\n # value must match, convert to string and compare\n if (value != attr\n and str(value).lower() != str(attr).lower()):\n match = False\n break\n any = any or match\n if any:\n # not True because we don't want to FORCE the selection of the\n # item, only say that it is acceptable\n return None\n return False", "response": "Verify whether a method has the required attributes\n "} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\naccept the method if its attributes match.", "response": "def wantMethod(self, method):\n \"\"\"Accept the method if its attributes match.\n \"\"\"\n try:\n cls = method.im_class\n except AttributeError:\n return False\n return self.validateAttrib(method, cls)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrotating the kill ring then yank back the new top.", "response": "def rotate(self):\n \"\"\" Rotate the kill ring, then yank back the new top.\n \"\"\"\n if self._prev_yank:\n text = self._ring.rotate()\n if text:\n self._skip_cursor = True\n cursor = self._text_edit.textCursor()\n cursor.movePosition(QtGui.QTextCursor.Left,\n QtGui.QTextCursor.KeepAnchor,\n n = len(self._prev_yank))\n cursor.insertText(text)\n self._prev_yank = text"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef patch_pyzmq():\n \n import zmq\n \n # ioloop.install, introduced in pyzmq 2.1.7\n from zmq.eventloop import ioloop\n \n def install():\n import tornado.ioloop\n tornado.ioloop.IOLoop = ioloop.IOLoop\n \n if not hasattr(ioloop, 'install'):\n ioloop.install = install\n \n # fix missing DEALER/ROUTER aliases in pyzmq < 2.1.9\n if not hasattr(zmq, 'DEALER'):\n zmq.DEALER = zmq.XREQ\n if not hasattr(zmq, 'ROUTER'):\n zmq.ROUTER = zmq.XREP\n \n # fallback on stdlib json if jsonlib is selected, because jsonlib breaks things.\n # jsonlib support is removed from pyzmq >= 2.2.0\n\n from zmq.utils import jsonapi\n if jsonapi.jsonmod.__name__ == 'jsonlib':\n import json\n jsonapi.jsonmod = json", "response": "backport a few patches from older pyzmq versions to new version"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the API version number from the schema element.", "response": "def version_from_schema(schema_el):\n \"\"\"\n returns: API version number \n raises: \n\n NOTE:\n relies on presence of comment tags in the XSD, which are currently present\n for both ebaySvc.xsd (TradingAPI) and ShoppingService.xsd (ShoppingAPI)\n \"\"\"\n vc_el = schema_el\n while True:\n vc_el = vc_el.getprevious()\n if vc_el is None:\n break\n if vc_el.tag is etree.Comment:\n match = VERSION_COMMENT.search(vc_el.text)\n if match:\n try:\n return match.group(1)\n except IndexError:\n pass\n raise VersionNotFound('Version comment not found preceeding schema node')"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _default_ns_prefix(nsmap):\n if None in nsmap:\n default_url = nsmap[None]\n prefix = None\n for key, val in nsmap.iteritems():\n if val == default_url and key is not None:\n prefix = key\n break\n else:\n raise ValueError(\n \"Default namespace {url} not found as a prefix\".format(\n url=default_url\n )\n )\n return prefix\n raise ValueError(\"No default namespace found in map\")", "response": "This function returns the default namespace of the xml document that is used in the xml file."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef version_from_wsdl(wsdl_tree):\n prefix = _default_ns_prefix(wsdl_tree.nsmap)\n\n # XPath doesn't allow empty prefix:\n safe_map = wsdl_tree.nsmap.copy()\n try:\n del safe_map[None]\n except KeyError:\n pass\n\n try:\n # various eBay WSDLs are inconsistent - need case-insensitive matching\n version_el = wsdl_tree.xpath(\n 'wsdl:service/wsdl:documentation/'\n '*[self::{p}:version or self::{p}:Version]'.format(p=prefix),\n namespaces=safe_map\n )[0]\n except IndexError:\n raise VersionNotFound(\n 'Version not found in WSDL service documentation'\n )\n else:\n return version_el.text", "response": "Returns the API version number from the WSDL service documentation."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns an XSD - schema - enabled lxml parser from a WSDL or XSD schema_url can be local path via file:// url", "response": "def parser_from_schema(schema_url, require_version=True):\n \"\"\"\n Returns an XSD-schema-enabled lxml parser from a WSDL or XSD\n\n `schema_url` can of course be local path via file:// url\n \"\"\"\n schema_tree = etree.parse(schema_url)\n\n def get_version(element, getter):\n try:\n return getter(element)\n except VersionNotFound:\n if require_version:\n raise\n else:\n return None\n\n root = schema_tree.getroot()\n if root.tag == '{%s}definitions' % namespaces.WSDL:\n # wsdl should contain an embedded schema\n schema_el = schema_tree.find('wsdl:types/xs:schema', namespaces=NS_MAP)\n version = get_version(root, version_from_wsdl)\n else:\n schema_el = root\n version = get_version(schema_el, version_from_schema)\n\n schema = etree.XMLSchema(schema_el)\n return objectify.makeparser(schema=schema), version"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef askQuestion(question, required = True, answers = dict(), tries = 3):\n print question\n \n if not answers:\n # we get the user's answer via a named utility because direct calls to\n # raw_input() are hard to test (this way, tests can provide their own\n # implementation of ISelectedChoice without calls to raw_input())\n answer = getUtility(ISelectedChoice, 'sparc.common.cli_selected_choice').selection()\n _attempts = 0\n while True:\n if tries and _attempts > tries:\n print (u\"Too many attempts\")\n return None\n if not required or answer:\n return answer if answer else None\n print (u\"Invalid input, please try again.\")\n answer = getUtility(ISelectedChoice, 'sparc.common.cli_selected_choice').selection()\n _attempts += 1\n for selection_pair in answers:\n for key, potential_answer in selection_pair.iteritems():\n print \"(\" + key + \") \" + potential_answer\n break # only process the first dict entry for the sequence\n _attempts = 0\n while True:\n answer = getUtility(ISelectedChoice, 'sparc.common.cli_selected_choice').selection()\n if tries and _attempts > tries:\n print _(u\"Too many attempts\")\n return None\n if not answer and not required:\n return None\n for selection_pair in answers:\n if answer in selection_pair:\n return answer\n print (u\"Invalid selection: {}, please try again.\".format(answer))\n answer = getUtility(ISelectedChoice, 'sparc.common.cli_selected_choice').selection()\n _attempts += 1", "response": "Ask a question to STDIN and return the answer from STDIN."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nauthenticates this page unless readonly is active.", "response": "def authenticate_unless_readonly(f, self, *args, **kwargs):\n \"\"\"authenticate this page *unless* readonly view is active.\n \n In read-only mode, the notebook list and print view should\n be accessible without authentication.\n \"\"\"\n \n @web.authenticated\n def auth_f(self, *args, **kwargs):\n return f(self, *args, **kwargs)\n\n if self.application.read_only:\n return f(self, *args, **kwargs)\n else:\n return auth_f(self, *args, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the websocket url matching the current request", "response": "def ws_url(self):\n \"\"\"websocket url matching the current request\n\n turns http[s]://host[:port] into\n ws[s]://host[:port]\n \"\"\"\n proto = self.request.protocol.replace('http', 'ws')\n host = self.application.ipython_app.websocket_host # default to config value\n if host == '':\n host = self.request.host # get from request\n return \"%s://%s\" % (proto, host)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _inject_cookie_message(self, msg):\n if isinstance(msg, unicode):\n # Cookie can't constructor doesn't accept unicode strings for some reason\n msg = msg.encode('utf8', 'replace')\n try:\n self.request._cookies = Cookie.SimpleCookie(msg)\n except:\n logging.warn(\"couldn't parse cookie string: %s\",msg, exc_info=True)", "response": "Inject the first message which is the document cookie and\n for authentication."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef start_hb(self, callback):\n if not self._beating:\n self._kernel_alive = True\n\n def ping_or_dead():\n self.hb_stream.flush()\n if self._kernel_alive:\n self._kernel_alive = False\n self.hb_stream.send(b'ping')\n # flush stream to force immediate socket send\n self.hb_stream.flush()\n else:\n try:\n callback()\n except:\n pass\n finally:\n self.stop_hb()\n\n def beat_received(msg):\n self._kernel_alive = True\n\n self.hb_stream.on_recv(beat_received)\n loop = ioloop.IOLoop.instance()\n self._hb_periodic_callback = ioloop.PeriodicCallback(ping_or_dead, self.time_to_dead*1000, loop)\n loop.add_timeout(time.time()+self.first_beat, self._really_start_hb)\n self._beating= True", "response": "Start the heartbeating and call the callback if the kernel dies."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _really_start_hb(self):\n if self._beating and not self.hb_stream.closed():\n self._hb_periodic_callback.start()", "response": "Callback for delayed heartbeat start\n \n Only start the hb loop if we haven t already closed during the wait."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstops the heartbeating and cancel all related callbacks.", "response": "def stop_hb(self):\n \"\"\"Stop the heartbeating and cancel all related callbacks.\"\"\"\n if self._beating:\n self._beating = False\n self._hb_periodic_callback.stop()\n if not self.hb_stream.closed():\n self.hb_stream.on_recv(None)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fload(self):\n # read data and parse into blocks\n if hasattr(self, 'fobj') and self.fobj is not None:\n self.fobj.close()\n if hasattr(self.src, \"read\"):\n # It seems to be a file or a file-like object\n self.fobj = self.src\n else:\n # Assume it's a string or something that can be converted to one\n self.fobj = open(self.fname)", "response": "Load file object into blocks object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreloading the source from disk and initialize state.", "response": "def reload(self):\n \"\"\"Reload source from disk and initialize state.\"\"\"\n self.fload()\n\n self.src = self.fobj.read()\n src_b = [b.strip() for b in self.re_stop.split(self.src) if b]\n self._silent = [bool(self.re_silent.findall(b)) for b in src_b]\n self._auto = [bool(self.re_auto.findall(b)) for b in src_b]\n\n # if auto_all is not given (def. None), we read it from the file\n if self.auto_all is None:\n self.auto_all = bool(self.re_auto_all.findall(src_b[0]))\n else:\n self.auto_all = bool(self.auto_all)\n\n # Clean the sources from all markup so it doesn't get displayed when\n # running the demo\n src_blocks = []\n auto_strip = lambda s: self.re_auto.sub('',s)\n for i,b in enumerate(src_b):\n if self._auto[i]:\n src_blocks.append(auto_strip(b))\n else:\n src_blocks.append(b)\n # remove the auto_all marker\n src_blocks[0] = self.re_auto_all.sub('',src_blocks[0])\n\n self.nblocks = len(src_blocks)\n self.src_blocks = src_blocks\n\n # also build syntax-highlighted source\n self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)\n\n # ensure clean namespace and seek offset\n self.reset()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget the current block index validating and checking status.", "response": "def _get_index(self,index):\n \"\"\"Get the current block index, validating and checking status.\n\n Returns None if the demo is finished\"\"\"\n\n if index is None:\n if self.finished:\n print >>io.stdout, 'Demo finished. Use .reset() if you want to rerun it.'\n return None\n index = self.block_index\n else:\n self._validate_index(index)\n return index"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmoves the current seek pointer to the given index.", "response": "def seek(self,index):\n \"\"\"Move the current seek pointer to the given block.\n\n You can use negative indices to seek from the end, with identical\n semantics to those of Python lists.\"\"\"\n if index<0:\n index = self.nblocks + index\n self._validate_index(index)\n self.block_index = index\n self.finished = False"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nedits a block. If no number is given, use the last block executed. This edits the in-memory copy of the demo, it does NOT modify the original source file. If you want to do that, simply open the file in an editor and use reload() when you make changes to the file. This method is meant to let you change a block during a demonstration for explanatory purposes, without damaging your original script.", "response": "def edit(self,index=None):\n \"\"\"Edit a block.\n\n If no number is given, use the last block executed.\n\n This edits the in-memory copy of the demo, it does NOT modify the\n original source file. If you want to do that, simply open the file in\n an editor and use reload() when you make changes to the file. This\n method is meant to let you change a block during a demonstration for\n explanatory purposes, without damaging your original script.\"\"\"\n\n index = self._get_index(index)\n if index is None:\n return\n # decrease the index by one (unless we're at the very beginning), so\n # that the default demo.edit() call opens up the sblock we've last run\n if index>0:\n index -= 1\n\n filename = self.shell.mktempfile(self.src_blocks[index])\n self.shell.hooks.editor(filename,1)\n new_block = file_read(filename)\n # update the source and colored block\n self.src_blocks[index] = new_block\n self.src_blocks_colored[index] = self.ip_colorize(new_block)\n self.block_index = index\n # call to run with the newly edited index\n self()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef show(self,index=None):\n\n index = self._get_index(index)\n if index is None:\n return\n\n print >>io.stdout, self.marquee('<%s> block # %s (%s remaining)' %\n (self.title,index,self.nblocks-index-1))\n print >>io.stdout,(self.src_blocks_colored[index])\n sys.stdout.flush()", "response": "Show a single block on screen"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef show_all(self):\n\n fname = self.title\n title = self.title\n nblocks = self.nblocks\n silent = self._silent\n marquee = self.marquee\n for index,block in enumerate(self.src_blocks_colored):\n if silent[index]:\n print >>io.stdout, marquee('<%s> SILENT block # %s (%s remaining)' %\n (title,index,nblocks-index-1))\n else:\n print >>io.stdout, marquee('<%s> block # %s (%s remaining)' %\n (title,index,nblocks-index-1))\n print >>io.stdout, block,\n sys.stdout.flush()", "response": "Show all demo on screen block by block"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef reload(self):\n # read data and parse into blocks\n self.fload()\n lines = self.fobj.readlines()\n src_b = [l for l in lines if l.strip()]\n nblocks = len(src_b)\n self.src = ''.join(lines)\n self._silent = [False]*nblocks\n self._auto = [True]*nblocks\n self.auto_all = True\n self.nblocks = nblocks\n self.src_blocks = src_b\n\n # also build syntax-highlighted source\n self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)\n\n # ensure clean namespace and seek offset\n self.reset()", "response": "Reload source from disk and initialize state."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef series(collection, method, prints = 15, *args, **kwargs):\n '''\n Processes a collection in series \n\n Parameters\n ----------\n collection : list\n list of Record objects\n method : method to call on each Record\n prints : int\n number of timer prints to the screen\n\n Returns\n -------\n collection : list\n list of Record objects after going through method called\n \n If more than one collection is given, the function is called with an argument list \n consisting of the corresponding item of each collection, substituting None for \n missing values when not all collection have the same length. \n If the function is None, return the original collection (or a list of tuples if multiple collections).\n \n Example\n -------\n adding 2 to every number in a range\n\n >>> import turntable\n >>> collection = range(100)\n >>> method = lambda x: x + 2\n >>> collection = turntable.spin.series(collection, method)\n \n '''\n\n if 'verbose' in kwargs.keys():\n verbose = kwargs['verbose']\n else:\n verbose = True\n\n results = []\n timer = turntable.utils.Timer(nLoops=len(collection), numPrints=prints, verbose=verbose)\n for subject in collection:\n results.append(method(subject, *args, **kwargs))\n timer.loop()\n timer.fin()\n return results", "response": "This function processes a collection in series \n\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef batch(collection, method, processes=None, batch_size=None, quiet=False,\n kwargs_to_dump=None, args=None, **kwargs):\n '''Processes a collection in parallel batches, \n each batch processes in series on a single process.\n Running batches in parallel can be more effficient that splitting a list across cores as in spin.parallel \n because of parallel processing has high IO requirements.\n\n Parameters\n ----------\n collection : list\n i.e. list of Record objects\n method : method to call on each Record\n processes : int\n number of processes to run on [defaults to number of cores on machine]\n batch_size : int\n lenght of each batch [defaults to number of elements / number of processes]\n\n Returns\n -------\n collection : list\n list of Record objects after going through method called\n\n Example\n -------\n adding 2 to every number in a range\n\n >>> import turntable\n >>> collection = range(100)\n >>> def jam(record):\n >>> return record + 2\n >>> collection = turntable.spin.batch(collection, jam)\n\n Note\n ----\n\n lambda functions do not work in parallel\n\n '''\n\n if processes is None:\n # default to the number of processes, not exceeding 20 or the number of\n # subjects\n processes = min(mp.cpu_count(), 20, len(collection))\n\n if batch_size is None:\n # floor divide rounds down to nearest int\n batch_size = max(len(collection) // processes, 1)\n print 'size of each batch =', batch_size\n\n mod = len(collection) % processes\n # batch_list is a list of cars broken in to batch size chunks\n batch_list = [collection[x:x + batch_size]\n for x in xrange(0, len(collection) - mod, batch_size)]\n # remainder handling\n if mod != 0:\n batch_list[len(batch_list) - 1] += collection[-mod:]\n print 'number of batches =', len(batch_list)\n\n # New args\n if args is None:\n args = method\n else:\n if isinstance(args, tuple) == False:\n args = (args,)\n args = (method,) + args\n\n # Applying the mp method w/ or w/o dumping using the custom operator\n # method\n if kwargs_to_dump is None:\n res = parallel(\n batch_list,\n new_function_batch,\n processes=processes,\n args=args,\n **kwargs)\n else:\n res = process_dump(\n batch_list,\n new_function_batch,\n kwargs_to_dump,\n processes=processes,\n args=args,\n **kwargs)\n\n returnList = []\n for l in res:\n returnList += l\n\n # toc = time.time()\n # elapsed = toc-tic\n # if quiet is False:\n # \tif processes is None:\n # \t\tprint \"Total Elapsed time: %s :-)\" % str(elapsed)\n # \telse:\n # print \"Total Elapsed time: %s on %s processes :-)\" %\n # (str(elapsed),str(processes))\n\n return returnList", "response": "Processes a collection in parallel batches on a single machine."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef thread(function, sequence, cores=None, runSeries=False, quiet=False):\n '''sets up the threadpool with map for parallel processing'''\n\n # Make the Pool of workes\n if cores is None:\n pool = ThreadPool()\n else:\n pool = ThreadPool(cores)\n\n # Operate on the list of subjects with the requested function\n # in the split threads\n tic = time.time()\n if runSeries is False:\n try:\n results = pool.map(function, sequence)\n # close the pool and wiat for teh work to finish\n pool.close()\n pool.join()\n except:\n print 'thread Failed... running in series :-('\n results = series(sequence, function)\n else:\n results = series(sequence, function)\n toc = time.time()\n elapsed = toc - tic\n\n if quiet is False:\n if cores is None:\n print \"Elapsed time: %s :-)\\n\" % str(elapsed)\n else:\n print \"Elapsed time: %s on %s threads :-)\\n\" % (str(elapsed), str(cores))\n # Noes:\n # import functools\n # abc = map(functools.partial(sb.dist, distName = 'weibull'), wbldfList)\n\n return results", "response": "sets up the threadpool with map for parallel processing"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parallel(collection, method, processes=None, args=None, **kwargs):\n '''Processes a collection in parallel.\n\n Parameters\n ----------\n collection : list\n i.e. list of Record objects\n method : method to call on each Record\n processes : int\n number of processes to run on [defaults to number of cores on machine]\n batch_size : int\n lenght of each batch [defaults to number of elements / number of processes]\n\n Returns\n -------\n collection : list\n list of Record objects after going through method called\n\n Example\n -------\n adding 2 to every number in a range\n\n >>> import turntable\n >>> collection = range(100)\n >>> def jam(record):\n >>> return record + 2\n >>> collection = turntable.spin.parallel(collection, jam)\n\n Note\n ----\n\n lambda functions do not work in parallel\n\n '''\n\n\n if processes is None:\n # default to the number of cores, not exceeding 20\n processes = min(mp.cpu_count(), 20)\n print \"Running parallel process on \" + str(processes) + \" cores. :-)\"\n\n pool = mp.Pool(processes=processes)\n PROC = []\n tic = time.time()\n for main_arg in collection:\n if args is None:\n ARGS = (main_arg,)\n else:\n if isinstance(args, tuple) == False:\n args = (args,)\n ARGS = (main_arg,) + args\n PROC.append(pool.apply_async(method, args=ARGS, kwds=kwargs))\n #RES = [p.get() for p in PROC]\n RES = []\n for p in PROC:\n try:\n RES.append(p.get())\n except Exception as e:\n print \"shit happens...\"\n print e\n RES.append(None)\n pool.close()\n pool.join()\n\n toc = time.time()\n elapsed = toc - tic\n print \"Elapsed time: %s on %s processes :-)\\n\" % (str(elapsed), str(processes))\n\n return RES", "response": "Processes a collection in parallel."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndownloads and install MathJax for offline use.", "response": "def install_mathjax(tag='v1.1', replace=False):\n \"\"\"Download and install MathJax for offline use.\n \n This will install mathjax to the 'static' dir in the IPython notebook\n package, so it will fail if the caller does not have write access\n to that location.\n \n MathJax is a ~15MB download, and ~150MB installed.\n \n Parameters\n ----------\n \n replace : bool [False]\n Whether to remove and replace an existing install.\n tag : str ['v1.1']\n Which tag to download. Default is 'v1.1', the current stable release,\n but alternatives include 'v1.1a' and 'master'.\n \"\"\"\n mathjax_url = \"https://github.com/mathjax/MathJax/tarball/%s\"%tag\n \n nbdir = os.path.dirname(os.path.abspath(nbmod.__file__))\n static = os.path.join(nbdir, 'static')\n dest = os.path.join(static, 'mathjax')\n \n # check for existence and permissions\n if not os.access(static, os.W_OK):\n raise IOError(\"Need have write access to %s\"%static)\n if os.path.exists(dest):\n if replace:\n if not os.access(dest, os.W_OK):\n raise IOError(\"Need have write access to %s\"%dest)\n print \"removing previous MathJax install\"\n shutil.rmtree(dest)\n else:\n print \"offline MathJax apparently already installed\"\n return\n \n # download mathjax\n print \"Downloading mathjax source...\"\n response = urllib2.urlopen(mathjax_url)\n print \"done\"\n # use 'r|gz' stream mode, because socket file-like objects can't seek:\n tar = tarfile.open(fileobj=response.fp, mode='r|gz')\n topdir = tar.firstmember.path\n print \"Extracting to %s\"%dest\n tar.extractall(static)\n # it will be mathjax-MathJax-, rename to just mathjax\n os.rename(os.path.join(static, topdir), dest)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwraps `with obj` out of func. example: ``` py @with_it(Lock()) def func(): pass ```", "response": "def with_it(obj):\n '''\n wrap `with obj` out of func.\n\n example:\n\n ``` py\n @with_it(Lock())\n def func(): pass\n ```\n '''\n\n def _wrap(func):\n\n @functools.wraps(func)\n def wrapper(*args, **kwargs):\n with obj:\n return func(*args, **kwargs)\n\n return wrapper\n\n return _wrap"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrap `with getattr(self, name)` out of func. usage: ``` py class A: def __init__(self): self._lock = RLock() @with_objattr('_lock') # so easy to make a sync instance method ! def func(): pass ```", "response": "def with_objattr(name):\n '''\n wrap `with getattr(self, name)` out of func.\n\n usage:\n\n ``` py\n class A:\n def __init__(self):\n self._lock = RLock()\n\n @with_objattr('_lock') # so easy to make a sync instance method !\n def func():\n pass\n ```\n '''\n\n def _wrap(func):\n\n @functools.wraps(func)\n def wrapper(self, *args, **kwargs):\n with getattr(self, name):\n return func(self, *args, **kwargs)\n\n return wrapper\n\n return _wrap"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef with_objattrs(*names):\n '''\n like `with_objattr` but enter context one by one.\n '''\n\n def _wrap(func):\n\n @functools.wraps(func)\n def wrapper(self, *args, **kwargs):\n with contextlib.ExitStack() as stack:\n for name in names:\n stack.enter_context(getattr(self, name))\n return func(self, *args, **kwargs)\n\n return wrapper\n\n return _wrap", "response": "A decorator that can be used to wrap functions that take a list of object attributes and enter context one by one."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef inspect_traceback(tb):\n log.debug('inspect traceback %s', tb)\n\n # we only want the innermost frame, where the exception was raised\n while tb.tb_next:\n tb = tb.tb_next\n \n frame = tb.tb_frame\n lines, exc_line = tbsource(tb)\n \n # figure out the set of lines to grab.\n inspect_lines, mark_line = find_inspectable_lines(lines, exc_line)\n src = StringIO(textwrap.dedent(''.join(inspect_lines)))\n exp = Expander(frame.f_locals, frame.f_globals)\n\n while inspect_lines:\n try:\n for tok in tokenize.generate_tokens(src.readline):\n exp(*tok)\n except tokenize.TokenError, e:\n # this can happen if our inspectable region happens to butt up\n # against the end of a construct like a docstring with the closing\n # \"\"\" on separate line\n log.debug(\"Tokenizer error: %s\", e)\n inspect_lines.pop(0)\n mark_line -= 1\n src = StringIO(textwrap.dedent(''.join(inspect_lines)))\n exp = Expander(frame.f_locals, frame.f_globals)\n continue\n break\n padded = []\n if exp.expanded_source:\n exp_lines = exp.expanded_source.split('\\n')\n ep = 0\n for line in exp_lines:\n if ep == mark_line:\n padded.append('>> ' + line)\n else:\n padded.append(' ' + line)\n ep += 1\n return '\\n'.join(padded)", "response": "Inspect a traceback and its frame returning source for the expression\notope where the exception was raised and the line on which the exception was raised."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef tbsource(tb, context=6):\n \n lineno = tb.tb_lineno\n frame = tb.tb_frame\n\n if context > 0:\n start = lineno - 1 - context//2\n log.debug(\"lineno: %s start: %s\", lineno, start)\n \n try:\n lines, dummy = inspect.findsource(frame)\n except IOError:\n lines, index = [''], 0\n else:\n all_lines = lines\n start = max(start, 1)\n start = max(0, min(start, len(lines) - context))\n lines = lines[start:start+context]\n index = lineno - 1 - start\n \n # python 2.5 compat: if previous line ends in a continuation,\n # decrement start by 1 to match 2.4 behavior \n if sys.version_info >= (2, 5) and index > 0:\n while lines[index-1].strip().endswith('\\\\'):\n start -= 1\n lines = all_lines[start:start+context]\n else:\n lines, index = [''], 0\n log.debug(\"tbsource lines '''%s''' around index %s\", lines, index)\n return (lines, index)", "response": "Get source code from a traceback object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find_inspectable_lines(lines, pos):\n cnt = re.compile(r'\\\\[\\s\\n]*$')\n df = re.compile(r':[\\s\\n]*$')\n ind = re.compile(r'^(\\s*)')\n toinspect = []\n home = lines[pos]\n home_indent = ind.match(home).groups()[0]\n \n before = lines[max(pos-3, 0):pos]\n before.reverse()\n after = lines[pos+1:min(pos+4, len(lines))]\n\n for line in before:\n if ind.match(line).groups()[0] == home_indent:\n toinspect.append(line)\n else:\n break\n toinspect.reverse()\n toinspect.append(home)\n home_pos = len(toinspect)-1\n continued = cnt.search(home)\n for line in after:\n if ((continued or ind.match(line).groups()[0] == home_indent)\n and not df.search(line)):\n toinspect.append(line)\n continued = cnt.search(line)\n else:\n break\n log.debug(\"Inspecting lines '''%s''' around %s\", toinspect, home_pos)\n return toinspect, home_pos", "response": "Find lines in home that are inspectable."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cleanup(controller, engines):\n import signal, time\n \n print('Starting cleanup')\n print('Stopping engines...')\n for e in engines:\n e.send_signal(signal.SIGINT)\n print('Stopping controller...')\n # so it can shut down its queues\n controller.send_signal(signal.SIGINT)\n time.sleep(0.1)\n print('Killing controller...')\n controller.kill()\n print('Cleanup done')", "response": "This routine shuts down all the engines and the controller."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef post_call(self, ctxt, result, action, post_mod, pre_mod):\n\n # Set the ignore state\n result.ignore = self.config\n\n return result", "response": "This is the main hook function that is called after the step is performed."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef save_ids(f, self, *args, **kwargs):\n n_previous = len(self.client.history)\n try:\n ret = f(self, *args, **kwargs)\n finally:\n nmsgs = len(self.client.history) - n_previous\n msg_ids = self.client.history[-nmsgs:]\n self.history.extend(msg_ids)\n map(self.outstanding.add, msg_ids)\n return ret", "response": "Save our history and outstanding attributes up to date after a method call."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalling spin after the method.", "response": "def spin_after(f, self, *args, **kwargs):\n \"\"\"call spin after the method.\"\"\"\n ret = f(self, *args, **kwargs)\n self.spin()\n return ret"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef add_record(self, msg_id, rec):\n # print rec\n rec = self._binary_buffers(rec)\n self._records.insert(rec)", "response": "Add a new Task Record by msg_id."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget a specific Task Record by msg_id.", "response": "def get_record(self, msg_id):\n \"\"\"Get a specific Task Record, by msg_id.\"\"\"\n r = self._records.find_one({'msg_id': msg_id})\n if not r:\n # r will be '' if nothing is found\n raise KeyError(msg_id)\n return r"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating the data in an existing record.", "response": "def update_record(self, msg_id, rec):\n \"\"\"Update the data in an existing record.\"\"\"\n rec = self._binary_buffers(rec)\n\n self._records.update({'msg_id':msg_id}, {'$set': rec})"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfind records matching a query dict optionally extracting subset of keys.", "response": "def find_records(self, check, keys=None):\n \"\"\"Find records matching a query dict, optionally extracting subset of keys.\n \n Returns list of matching records.\n \n Parameters\n ----------\n \n check: dict\n mongodb-style query argument\n keys: list of strs [optional]\n if specified, the subset of keys to extract. msg_id will *always* be\n included.\n \"\"\"\n if keys and 'msg_id' not in keys:\n keys.append('msg_id')\n matches = list(self._records.find(check,keys))\n for rec in matches:\n rec.pop('_id')\n return matches"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting all msg_ids ordered by time submitted.", "response": "def get_history(self):\n \"\"\"get all msg_ids, ordered by time submitted.\"\"\"\n cursor = self._records.find({},{'msg_id':1}).sort('submitted')\n return [ rec['msg_id'] for rec in cursor ]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_msgs(self):\n msgs = []\n while True:\n try:\n msgs.append(self.get_msg(block=False))\n except Empty:\n break\n return msgs", "response": "Get all messages that are currently ready."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets a message if there is one that is ready.", "response": "def get_msg(self, block=True, timeout=None):\n \"Gets a message if there is one that is ready.\"\n return self._in_queue.get(block, timeout)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse a database URL and returns a dictionary of environment variables.", "response": "def parse(url):\n \"\"\"Parses a database URL.\"\"\"\n\n config = {}\n\n if not isinstance(url, six.string_types):\n url = ''\n\n url = urlparse.urlparse(url)\n\n # Remove query strings.\n path = url.path[1:]\n path = path.split('?', 2)[0]\n\n # Update with environment configuration.\n config.update({\n 'NAME': path,\n 'USER': url.username,\n 'PASSWORD': url.password,\n 'HOST': url.hostname,\n 'PORT': url.port,\n })\n\n if url.scheme in SCHEMES:\n config['ENGINE'] = SCHEMES[url.scheme]\n\n return config"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the list containing the names of the modules available in the given folder.", "response": "def module_list(path):\n \"\"\"\n Return the list containing the names of the modules available in the given\n folder.\n \"\"\"\n # sys.path has the cwd as an empty string, but isdir/listdir need it as '.'\n if path == '':\n path = '.'\n\n if os.path.isdir(path):\n folder_list = os.listdir(path)\n elif path.endswith('.egg'):\n try:\n folder_list = [f for f in zipimporter(path)._files]\n except:\n folder_list = []\n else:\n folder_list = []\n\n if not folder_list:\n return []\n\n # A few local constants to be used in loops below\n isfile = os.path.isfile\n pjoin = os.path.join\n basename = os.path.basename\n\n def is_importable_file(path):\n \"\"\"Returns True if the provided path is a valid importable module\"\"\"\n name, extension = os.path.splitext( path )\n return import_re.match(path) and py3compat.isidentifier(name)\n\n # Now find actual path matches for packages or modules\n folder_list = [p for p in folder_list\n if isfile(pjoin(path, p,'__init__.py'))\n or is_importable_file(p) ]\n\n return [basename(p).split('.')[0] for p in folder_list]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_root_modules():\n ip = get_ipython()\n\n if 'rootmodules' in ip.db:\n return ip.db['rootmodules']\n\n t = time()\n store = False\n modules = list(sys.builtin_module_names)\n for path in sys.path:\n modules += module_list(path)\n if time() - t >= TIMEOUT_STORAGE and not store:\n store = True\n print(\"\\nCaching the list of root modules, please wait!\")\n print(\"(This will only be done once - type '%rehashx' to \"\n \"reset cache!)\\n\")\n sys.stdout.flush()\n if time() - t > TIMEOUT_GIVEUP:\n print(\"This is taking too long, we give up.\\n\")\n ip.db['rootmodules'] = []\n return []\n\n modules = set(modules)\n if '__init__' in modules:\n modules.remove('__init__')\n modules = list(modules)\n if store:\n ip.db['rootmodules'] = modules\n return modules", "response": "Returns a list containing the names of all the modules available in the pythonpath."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef quick_completer(cmd, completions):\n\n if isinstance(completions, basestring):\n completions = completions.split()\n\n def do_complete(self, event):\n return completions\n\n get_ipython().set_hook('complete_command',do_complete, str_key = cmd)", "response": "Easily create a trivial completer for a command."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a list containing the possibilities for an import line.", "response": "def module_completion(line):\n \"\"\"\n Returns a list containing the completion possibilities for an import line.\n\n The line looks like this :\n 'import xml.d'\n 'from xml.dom import'\n \"\"\"\n\n words = line.split(' ')\n nwords = len(words)\n\n # from whatever -> 'import '\n if nwords == 3 and words[0] == 'from':\n return ['import ']\n\n # 'from xy' or 'import xy'\n if nwords < 3 and (words[0] in ['import','from']) :\n if nwords == 1:\n return get_root_modules()\n mod = words[1].split('.')\n if len(mod) < 2:\n return get_root_modules()\n completion_list = try_import('.'.join(mod[:-1]), True)\n return ['.'.join(mod[:-1] + [el]) for el in completion_list]\n\n # 'from xyz import abc'\n if nwords >= 3 and words[0] == 'from':\n mod = words[1]\n return try_import(mod)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncomplete files that end in .py or .ipy for the %run command.", "response": "def magic_run_completer(self, event):\n \"\"\"Complete files that end in .py or .ipy for the %run command.\n \"\"\"\n comps = arg_split(event.line, strict=False)\n relpath = (len(comps) > 1 and comps[-1] or '').strip(\"'\\\"\")\n\n #print(\"\\nev=\", event) # dbg\n #print(\"rp=\", relpath) # dbg\n #print('comps=', comps) # dbg\n\n lglob = glob.glob\n isdir = os.path.isdir\n relpath, tilde_expand, tilde_val = expand_user(relpath)\n\n dirs = [f.replace('\\\\','/') + \"/\" for f in lglob(relpath+'*') if isdir(f)]\n\n # Find if the user has already typed the first filename, after which we\n # should complete on all files, since after the first one other files may\n # be arguments to the input script.\n\n if filter(magic_run_re.match, comps):\n pys = [f.replace('\\\\','/') for f in lglob('*')]\n else:\n pys = [f.replace('\\\\','/')\n for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +\n lglob(relpath + '*.pyw')]\n #print('run comp:', dirs+pys) # dbg\n return [compress_user(p, tilde_expand, tilde_val) for p in dirs+pys]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef interact(self):\n jscode = self.render()\n display(Javascript(data=jscode,lib=self.jslibs))", "response": "This should call display ( Javascript code"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _quoteattr(self, attr):\n attr = xml_safe(attr)\n if isinstance(attr, unicode) and not UNICODE_STRINGS:\n attr = attr.encode(self.encoding)\n return saxutils.quoteattr(attr)", "response": "Escape an XML attribute. Value can be unicode."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconfigure the xunit plugin.", "response": "def configure(self, options, config):\n \"\"\"Configures the xunit plugin.\"\"\"\n Plugin.configure(self, options, config)\n self.config = config\n if self.enabled:\n self.stats = {'errors': 0,\n 'failures': 0,\n 'passes': 0,\n 'skipped': 0\n }\n self.errorlist = []\n self.error_report_file = codecs.open(options.xunit_file, 'w',\n self.encoding, 'replace')"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef report(self, stream):\n self.stats['encoding'] = self.encoding\n self.stats['total'] = (self.stats['errors'] + self.stats['failures']\n + self.stats['passes'] + self.stats['skipped'])\n self.error_report_file.write(\n u''\n u'' % self.stats)\n self.error_report_file.write(u''.join([self._forceUnicode(e)\n for e in self.errorlist]))\n self.error_report_file.write(u'')\n self.error_report_file.close()\n if self.config.verbosity > 1:\n stream.writeln(\"-\" * 70)\n stream.writeln(\"XML: %s\" % self.error_report_file.name)", "response": "Writes an Xunit - formatted XML file containing the test errors and failures and skipped tests."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd error output to Xunit report.", "response": "def addError(self, test, err, capt=None):\n \"\"\"Add error output to Xunit report.\n \"\"\"\n taken = self._timeTaken()\n\n if issubclass(err[0], SkipTest):\n type = 'skipped'\n self.stats['skipped'] += 1\n else:\n type = 'error'\n self.stats['errors'] += 1\n tb = ''.join(traceback.format_exception(*err))\n id = test.id()\n self.errorlist.append(\n ''\n '<%(type)s type=%(errtype)s message=%(message)s>'\n '' %\n {'cls': self._quoteattr(id_split(id)[0]),\n 'name': self._quoteattr(id_split(id)[-1]),\n 'taken': taken,\n 'type': type,\n 'errtype': self._quoteattr(nice_classname(err[0])),\n 'message': self._quoteattr(exc_message(err)),\n 'tb': escape_cdata(tb),\n })"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds failure output to Xunit report.", "response": "def addFailure(self, test, err, capt=None, tb_info=None):\n \"\"\"Add failure output to Xunit report.\n \"\"\"\n taken = self._timeTaken()\n tb = ''.join(traceback.format_exception(*err))\n self.stats['failures'] += 1\n id = test.id()\n self.errorlist.append(\n ''\n ''\n '' %\n {'cls': self._quoteattr(id_split(id)[0]),\n 'name': self._quoteattr(id_split(id)[-1]),\n 'taken': taken,\n 'errtype': self._quoteattr(nice_classname(err[0])),\n 'message': self._quoteattr(exc_message(err)),\n 'tb': escape_cdata(tb),\n })"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef addSuccess(self, test, capt=None):\n taken = self._timeTaken()\n self.stats['passes'] += 1\n id = test.id()\n self.errorlist.append(\n '' %\n {'cls': self._quoteattr(id_split(id)[0]),\n 'name': self._quoteattr(id_split(id)[-1]),\n 'taken': taken,\n })", "response": "Add success output to Xunit report."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef twobin(loads):\n n = len(loads)\n a = randint(0,n-1)\n b = randint(0,n-1)\n return min(a,b)", "response": "Pick two at random use the LRU of the two."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npick two at random using inverse load as weight. Return the less loaded of the two.", "response": "def weighted(loads):\n \"\"\"Pick two at random using inverse load as weight.\n\n Return the less loaded of the two.\n \"\"\"\n # weight 0 a million times more than 1:\n weights = 1./(1e-6+numpy.array(loads))\n sums = weights.cumsum()\n t = sums[-1]\n x = random()*t\n y = random()*t\n idx = 0\n idy = 0\n while sums[idx] < x:\n idx += 1\n while sums[idy] < y:\n idy += 1\n if weights[idy] > weights[idx]:\n return idy\n else:\n return idx"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndispatches register or unregister events.", "response": "def dispatch_notification(self, msg):\n \"\"\"dispatch register/unregister events.\"\"\"\n try:\n idents,msg = self.session.feed_identities(msg)\n except ValueError:\n self.log.warn(\"task::Invalid Message: %r\",msg)\n return\n try:\n msg = self.session.unserialize(msg)\n except ValueError:\n self.log.warn(\"task::Unauthorized message from: %r\"%idents)\n return\n\n msg_type = msg['header']['msg_type']\n\n handler = self._notification_handlers.get(msg_type, None)\n if handler is None:\n self.log.error(\"Unhandled message type: %r\"%msg_type)\n else:\n try:\n handler(cast_bytes(msg['content']['queue']))\n except Exception:\n self.log.error(\"task::Invalid notification msg: %r\", msg, exc_info=True)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nregister new engine with uid.", "response": "def _register_engine(self, uid):\n \"\"\"New engine with ident `uid` became available.\"\"\"\n # head of the line:\n self.targets.insert(0,uid)\n self.loads.insert(0,0)\n\n # initialize sets\n self.completed[uid] = set()\n self.failed[uid] = set()\n self.pending[uid] = {}\n\n # rescan the graph:\n self.update_graph(None)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _unregister_engine(self, uid):\n if len(self.targets) == 1:\n # this was our only engine\n pass\n\n # handle any potentially finished tasks:\n self.engine_stream.flush()\n\n # don't pop destinations, because they might be used later\n # map(self.destinations.pop, self.completed.pop(uid))\n # map(self.destinations.pop, self.failed.pop(uid))\n\n # prevent this engine from receiving work\n idx = self.targets.index(uid)\n self.targets.pop(idx)\n self.loads.pop(idx)\n\n # wait 5 seconds before cleaning up pending jobs, since the results might\n # still be incoming\n if self.pending[uid]:\n dc = ioloop.DelayedCallback(lambda : self.handle_stranded_tasks(uid), 5000, self.loop)\n dc.start()\n else:\n self.completed.pop(uid)\n self.failed.pop(uid)", "response": "Unregisters an existing engine with the given uid."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef handle_stranded_tasks(self, engine):\n lost = self.pending[engine]\n for msg_id in lost.keys():\n if msg_id not in self.pending[engine]:\n # prevent double-handling of messages\n continue\n\n raw_msg = lost[msg_id].raw_msg\n idents,msg = self.session.feed_identities(raw_msg, copy=False)\n parent = self.session.unpack(msg[1].bytes)\n idents = [engine, idents[0]]\n\n # build fake error reply\n try:\n raise error.EngineError(\"Engine %r died while running task %r\"%(engine, msg_id))\n except:\n content = error.wrap_exception()\n # build fake header\n header = dict(\n status='error',\n engine=engine,\n date=datetime.now(),\n )\n msg = self.session.msg('apply_reply', content, parent=parent, subheader=header)\n raw_reply = map(zmq.Message, self.session.serialize(msg, ident=idents))\n # and dispatch it\n self.dispatch_result(raw_reply)\n\n # finally scrub completed/failed lists\n self.completed.pop(engine)\n self.failed.pop(engine)", "response": "Deal with jobs resident in an engine that died."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndispatches submission to appropriate handlers.", "response": "def dispatch_submission(self, raw_msg):\n \"\"\"Dispatch job submission to appropriate handlers.\"\"\"\n # ensure targets up to date:\n self.notifier_stream.flush()\n try:\n idents, msg = self.session.feed_identities(raw_msg, copy=False)\n msg = self.session.unserialize(msg, content=False, copy=False)\n except Exception:\n self.log.error(\"task::Invaid task msg: %r\"%raw_msg, exc_info=True)\n return\n\n\n # send to monitor\n self.mon_stream.send_multipart([b'intask']+raw_msg, copy=False)\n\n header = msg['header']\n msg_id = header['msg_id']\n self.all_ids.add(msg_id)\n\n # get targets as a set of bytes objects\n # from a list of unicode objects\n targets = header.get('targets', [])\n targets = map(cast_bytes, targets)\n targets = set(targets)\n\n retries = header.get('retries', 0)\n self.retries[msg_id] = retries\n\n # time dependencies\n after = header.get('after', None)\n if after:\n after = Dependency(after)\n if after.all:\n if after.success:\n after = Dependency(after.difference(self.all_completed),\n success=after.success,\n failure=after.failure,\n all=after.all,\n )\n if after.failure:\n after = Dependency(after.difference(self.all_failed),\n success=after.success,\n failure=after.failure,\n all=after.all,\n )\n if after.check(self.all_completed, self.all_failed):\n # recast as empty set, if `after` already met,\n # to prevent unnecessary set comparisons\n after = MET\n else:\n after = MET\n\n # location dependencies\n follow = Dependency(header.get('follow', []))\n\n # turn timeouts into datetime objects:\n timeout = header.get('timeout', None)\n if timeout:\n # cast to float, because jsonlib returns floats as decimal.Decimal,\n # which timedelta does not accept\n timeout = datetime.now() + timedelta(0,float(timeout),0)\n\n job = Job(msg_id=msg_id, raw_msg=raw_msg, idents=idents, msg=msg,\n header=header, targets=targets, after=after, follow=follow,\n timeout=timeout,\n )\n\n # validate and reduce dependencies:\n for dep in after,follow:\n if not dep: # empty dependency\n continue\n # check valid:\n if msg_id in dep or dep.difference(self.all_ids):\n self.depending[msg_id] = job\n return self.fail_unreachable(msg_id, error.InvalidDependency)\n # check if unreachable:\n if dep.unreachable(self.all_completed, self.all_failed):\n self.depending[msg_id] = job\n return self.fail_unreachable(msg_id)\n\n if after.check(self.all_completed, self.all_failed):\n # time deps already met, try to run\n if not self.maybe_run(job):\n # can't run yet\n if msg_id not in self.all_failed:\n # could have failed as unreachable\n self.save_unmet(job)\n else:\n self.save_unmet(job)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef audit_timeouts(self):\n now = datetime.now()\n for msg_id in self.depending.keys():\n # must recheck, in case one failure cascaded to another:\n if msg_id in self.depending:\n job = self.depending[msg_id]\n if job.timeout and job.timeout < now:\n self.fail_unreachable(msg_id, error.TaskTimeout)", "response": "Audit all waiting tasks for expired timeouts."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsending a reply with an ImpossibleDependency exception", "response": "def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):\n \"\"\"a task has become unreachable, send a reply with an ImpossibleDependency\n error.\"\"\"\n if msg_id not in self.depending:\n self.log.error(\"msg %r already failed!\", msg_id)\n return\n job = self.depending.pop(msg_id)\n for mid in job.dependents:\n if mid in self.graph:\n self.graph[mid].remove(msg_id)\n\n try:\n raise why()\n except:\n content = error.wrap_exception()\n\n self.all_done.add(msg_id)\n self.all_failed.add(msg_id)\n\n msg = self.session.send(self.client_stream, 'apply_reply', content,\n parent=job.header, ident=job.idents)\n self.session.send(self.mon_stream, msg, ident=[b'outtask']+job.idents)\n\n self.update_graph(msg_id, success=False)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef maybe_run(self, job):\n msg_id = job.msg_id\n self.log.debug(\"Attempting to assign task %s\", msg_id)\n if not self.targets:\n # no engines, definitely can't run\n return False\n \n if job.follow or job.targets or job.blacklist or self.hwm:\n # we need a can_run filter\n def can_run(idx):\n # check hwm\n if self.hwm and self.loads[idx] == self.hwm:\n return False\n target = self.targets[idx]\n # check blacklist\n if target in job.blacklist:\n return False\n # check targets\n if job.targets and target not in job.targets:\n return False\n # check follow\n return job.follow.check(self.completed[target], self.failed[target])\n\n indices = filter(can_run, range(len(self.targets)))\n\n if not indices:\n # couldn't run\n if job.follow.all:\n # check follow for impossibility\n dests = set()\n relevant = set()\n if job.follow.success:\n relevant = self.all_completed\n if job.follow.failure:\n relevant = relevant.union(self.all_failed)\n for m in job.follow.intersection(relevant):\n dests.add(self.destinations[m])\n if len(dests) > 1:\n self.depending[msg_id] = job\n self.fail_unreachable(msg_id)\n return False\n if job.targets:\n # check blacklist+targets for impossibility\n job.targets.difference_update(job.blacklist)\n if not job.targets or not job.targets.intersection(self.targets):\n self.depending[msg_id] = job\n self.fail_unreachable(msg_id)\n return False\n return False\n else:\n indices = None\n\n self.submit_task(job, indices)\n return True", "response": "check location dependencies and run if they are met."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef save_unmet(self, job):\n msg_id = job.msg_id\n self.depending[msg_id] = job\n # track the ids in follow or after, but not those already finished\n for dep_id in job.after.union(job.follow).difference(self.all_done):\n if dep_id not in self.graph:\n self.graph[dep_id] = set()\n self.graph[dep_id].add(msg_id)", "response": "Save a message for later submission when its dependencies are met."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef submit_task(self, job, indices=None):\n if indices:\n loads = [self.loads[i] for i in indices]\n else:\n loads = self.loads\n idx = self.scheme(loads)\n if indices:\n idx = indices[idx]\n target = self.targets[idx]\n # print (target, map(str, msg[:3]))\n # send job to the engine\n self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)\n self.engine_stream.send_multipart(job.raw_msg, copy=False)\n # update load\n self.add_job(idx)\n self.pending[target][job.msg_id] = job\n # notify Hub\n content = dict(msg_id=job.msg_id, engine_id=target.decode('ascii'))\n self.session.send(self.mon_stream, 'task_destination', content=content,\n ident=[b'tracktask',self.ident])", "response": "Submit a task to any of a subset of our targets."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dispatch_result(self, raw_msg):\n try:\n idents,msg = self.session.feed_identities(raw_msg, copy=False)\n msg = self.session.unserialize(msg, content=False, copy=False)\n engine = idents[0]\n try:\n idx = self.targets.index(engine)\n except ValueError:\n pass # skip load-update for dead engines\n else:\n self.finish_job(idx)\n except Exception:\n self.log.error(\"task::Invaid result: %r\", raw_msg, exc_info=True)\n return\n\n header = msg['header']\n parent = msg['parent_header']\n if header.get('dependencies_met', True):\n success = (header['status'] == 'ok')\n msg_id = parent['msg_id']\n retries = self.retries[msg_id]\n if not success and retries > 0:\n # failed\n self.retries[msg_id] = retries - 1\n self.handle_unmet_dependency(idents, parent)\n else:\n del self.retries[msg_id]\n # relay to client and update graph\n self.handle_result(idents, parent, raw_msg, success)\n # send to Hub monitor\n self.mon_stream.send_multipart([b'outtask']+raw_msg, copy=False)\n else:\n self.handle_unmet_dependency(idents, parent)", "response": "dispatch method for result replies"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef handle_result(self, idents, parent, raw_msg, success=True):\n # first, relay result to client\n engine = idents[0]\n client = idents[1]\n # swap_ids for ROUTER-ROUTER mirror\n raw_msg[:2] = [client,engine]\n # print (map(str, raw_msg[:4]))\n self.client_stream.send_multipart(raw_msg, copy=False)\n # now, update our data structures\n msg_id = parent['msg_id']\n self.pending[engine].pop(msg_id)\n if success:\n self.completed[engine].add(msg_id)\n self.all_completed.add(msg_id)\n else:\n self.failed[engine].add(msg_id)\n self.all_failed.add(msg_id)\n self.all_done.add(msg_id)\n self.destinations[msg_id] = engine\n\n self.update_graph(msg_id, success)", "response": "handle a real task result either success or failure"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhandling an unmet dependency", "response": "def handle_unmet_dependency(self, idents, parent):\n \"\"\"handle an unmet dependency\"\"\"\n engine = idents[0]\n msg_id = parent['msg_id']\n\n job = self.pending[engine].pop(msg_id)\n job.blacklist.add(engine)\n\n if job.blacklist == job.targets:\n self.depending[msg_id] = job\n self.fail_unreachable(msg_id)\n elif not self.maybe_run(job):\n # resubmit failed\n if msg_id not in self.all_failed:\n # put it back in our dependency tree\n self.save_unmet(job)\n\n if self.hwm:\n try:\n idx = self.targets.index(engine)\n except ValueError:\n pass # skip load-update for dead engines\n else:\n if self.loads[idx] == self.hwm-1:\n self.update_graph(None)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update_graph(self, dep_id=None, success=True):\n # print (\"\\n\\n***********\")\n # pprint (dep_id)\n # pprint (self.graph)\n # pprint (self.depending)\n # pprint (self.all_completed)\n # pprint (self.all_failed)\n # print (\"\\n\\n***********\\n\\n\")\n # update any jobs that depended on the dependency\n jobs = self.graph.pop(dep_id, [])\n\n # recheck *all* jobs if\n # a) we have HWM and an engine just become no longer full\n # or b) dep_id was given as None\n \n if dep_id is None or self.hwm and any( [ load==self.hwm-1 for load in self.loads ]):\n jobs = self.depending.keys()\n \n for msg_id in sorted(jobs, key=lambda msg_id: self.depending[msg_id].timestamp):\n job = self.depending[msg_id]\n\n if job.after.unreachable(self.all_completed, self.all_failed)\\\n or job.follow.unreachable(self.all_completed, self.all_failed):\n self.fail_unreachable(msg_id)\n\n elif job.after.check(self.all_completed, self.all_failed): # time deps met, maybe run\n if self.maybe_run(job):\n\n self.depending.pop(msg_id)\n for mid in job.dependents:\n if mid in self.graph:\n self.graph[mid].remove(msg_id)", "response": "Update the dependency graph and submit any jobs that just became runable."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalls after self. targets [ idx ] just got the job with header. Override with subclasses.", "response": "def add_job(self, idx):\n \"\"\"Called after self.targets[idx] just got the job with header.\n Override with subclasses. The default ordering is simple LRU.\n The default loads are the number of outstanding jobs.\"\"\"\n self.loads[idx] += 1\n for lis in (self.targets, self.loads):\n lis.append(lis.pop(idx))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef logstart(self, logfname=None, loghead=None, logmode=None,\n log_output=False, timestamp=False, log_raw_input=False):\n \"\"\"Generate a new log-file with a default header.\n\n Raises RuntimeError if the log has already been started\"\"\"\n\n if self.logfile is not None:\n raise RuntimeError('Log file is already active: %s' %\n self.logfname)\n\n # The parameters can override constructor defaults\n if logfname is not None: self.logfname = logfname\n if loghead is not None: self.loghead = loghead\n if logmode is not None: self.logmode = logmode\n\n # Parameters not part of the constructor\n self.timestamp = timestamp\n self.log_output = log_output\n self.log_raw_input = log_raw_input\n\n # init depending on the log mode requested\n isfile = os.path.isfile\n logmode = self.logmode\n\n if logmode == 'append':\n self.logfile = io.open(self.logfname, 'a', encoding='utf-8')\n\n elif logmode == 'backup':\n if isfile(self.logfname):\n backup_logname = self.logfname+'~'\n # Manually remove any old backup, since os.rename may fail\n # under Windows.\n if isfile(backup_logname):\n os.remove(backup_logname)\n os.rename(self.logfname,backup_logname)\n self.logfile = io.open(self.logfname, 'w', encoding='utf-8')\n\n elif logmode == 'global':\n self.logfname = os.path.join(self.home_dir,self.logfname)\n self.logfile = io.open(self.logfname, 'a', encoding='utf-8')\n\n elif logmode == 'over':\n if isfile(self.logfname):\n os.remove(self.logfname)\n self.logfile = io.open(self.logfname,'w', encoding='utf-8')\n\n elif logmode == 'rotate':\n if isfile(self.logfname):\n if isfile(self.logfname+'.001~'):\n old = glob.glob(self.logfname+'.*~')\n old.sort()\n old.reverse()\n for f in old:\n root, ext = os.path.splitext(f)\n num = int(ext[1:-1])+1\n os.rename(f, root+'.'+`num`.zfill(3)+'~')\n os.rename(self.logfname, self.logfname+'.001~')\n self.logfile = io.open(self.logfname, 'w', encoding='utf-8')\n\n if logmode != 'append':\n self.logfile.write(self.loghead)\n\n self.logfile.flush()\n self.log_active = True", "response": "This function creates a new log file with the specified header and logmode."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef switch_log(self,val):\n\n if val not in [False,True,0,1]:\n raise ValueError, \\\n 'Call switch_log ONLY with a boolean argument, not with:',val\n\n label = {0:'OFF',1:'ON',False:'OFF',True:'ON'}\n\n if self.logfile is None:\n print \"\"\"\nLogging hasn't been started yet (use logstart for that).\n\n%logon/%logoff are for temporarily starting and stopping logging for a logfile\nwhich already exists. But you must first start the logging process with\n%logstart (optionally giving a logfile name).\"\"\"\n\n else:\n if self.log_active == val:\n print 'Logging is already',label[val]\n else:\n print 'Switching logging',label[val]\n self.log_active = not self.log_active\n self.log_active_out = self.log_active", "response": "Switch logging on or off. val should be ONLY a boolean argument."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef logstate(self):\n if self.logfile is None:\n print 'Logging has not been activated.'\n else:\n state = self.log_active and 'active' or 'temporarily suspended'\n print 'Filename :',self.logfname\n print 'Mode :',self.logmode\n print 'Output logging :',self.log_output\n print 'Raw input log :',self.log_raw_input\n print 'Timestamping :',self.timestamp\n print 'State :',state", "response": "Print a status message about the logger."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef log(self, line_mod, line_ori):\n\n # Write the log line, but decide which one according to the\n # log_raw_input flag, set when the log is started.\n if self.log_raw_input:\n self.log_write(line_ori)\n else:\n self.log_write(line_mod)", "response": "Write the sources to a log."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef log_write(self, data, kind='input'):\n\n #print 'data: %r' % data # dbg\n if self.log_active and data:\n write = self.logfile.write\n if kind=='input':\n if self.timestamp:\n write(str_to_unicode(time.strftime('# %a, %d %b %Y %H:%M:%S\\n',\n time.localtime())))\n write(data)\n elif kind=='output' and self.log_output:\n odata = u'\\n'.join([u'#[Out]# %s' % s\n for s in data.splitlines()])\n write(u'%s\\n' % odata)\n self.logfile.flush()", "response": "Write data to the log file if active"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef logstop(self):\n\n if self.logfile is not None:\n self.logfile.close()\n self.logfile = None\n else:\n print \"Logging hadn't been started.\"\n self.log_active = False", "response": "Fully stop logging and close log file."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef new_worksheet(name=None, cells=None):\n ws = NotebookNode()\n if name is not None:\n ws.name = unicode(name)\n if cells is None:\n ws.cells = []\n else:\n ws.cells = list(cells)\n return ws", "response": "Create a new worksheet by name with a list of cells."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef new_notebook(metadata=None, worksheets=None):\n nb = NotebookNode()\n nb.nbformat = 2\n if worksheets is None:\n nb.worksheets = []\n else:\n nb.worksheets = list(worksheets)\n if metadata is None:\n nb.metadata = new_metadata()\n else:\n nb.metadata = NotebookNode(metadata)\n return nb", "response": "Create a notebook by name id and a list of worksheets."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd a target string for dispatching.", "response": "def add_s(self, s, obj, priority= 0 ):\n \"\"\" Adds a target 'string' for dispatching \"\"\"\n\n chain = self.strs.get(s, CommandChainDispatcher())\n chain.add(obj,priority)\n self.strs[s] = chain"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a target regexp for dispatching.", "response": "def add_re(self, regex, obj, priority= 0 ):\n \"\"\" Adds a target regexp for dispatching \"\"\"\n\n chain = self.regexs.get(regex, CommandChainDispatcher())\n chain.add(obj,priority)\n self.regexs[regex] = chain"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets a seq of Commandchain objects that match the given key", "response": "def dispatch(self, key):\n \"\"\" Get a seq of Commandchain objects that match key \"\"\"\n if key in self.strs:\n yield self.strs[key]\n\n for r, obj in self.regexs.items():\n if re.match(r, key):\n yield obj\n else:\n #print \"nomatch\",key # dbg\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nyields all value targets with priority", "response": "def flat_matches(self, key):\n \"\"\" Yield all 'value' targets, without priority \"\"\"\n for val in self.dispatch(key):\n for el in val:\n yield el[1] # only value, no priority\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _notebook_dir_changed(self, name, old, new):\n if os.path.exists(new) and not os.path.isdir(new):\n raise TraitError(\"notebook dir %r is not a directory\" % new)\n if not os.path.exists(new):\n self.log.info(\"Creating notebook dir %s\", new)\n try:\n os.mkdir(new)\n except:\n raise TraitError(\"Couldn't create notebook dir %r\" % new)", "response": "check if the notebook dir is a directory and create it if it doesn t exist"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef list_notebooks(self):\n names = glob.glob(os.path.join(self.notebook_dir,\n '*' + self.filename_ext))\n names = [os.path.splitext(os.path.basename(name))[0]\n for name in names]\n\n data = []\n for name in names:\n if name not in self.rev_mapping:\n notebook_id = self.new_notebook_id(name)\n else:\n notebook_id = self.rev_mapping[name]\n data.append(dict(notebook_id=notebook_id,name=name))\n data = sorted(data, key=lambda item: item['name'])\n return data", "response": "List all notebooks in the notebook dir. This returns a list of dicts of the form ::\n\n dict ( notebook_id = notebook name"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef new_notebook_id(self, name):\n # TODO: the following will give stable urls for notebooks, but unless\n # the notebooks are immediately redirected to their new urls when their\n # filemname changes, nasty inconsistencies result. So for now it's\n # disabled and instead we use a random uuid4() call. But we leave the\n # logic here so that we can later reactivate it, whhen the necessary\n # url redirection code is written.\n #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL,\n # 'file://'+self.get_path_by_name(name).encode('utf-8')))\n \n notebook_id = unicode(uuid.uuid4())\n \n self.mapping[notebook_id] = name\n self.rev_mapping[name] = notebook_id\n return notebook_id", "response": "Generate a new notebook_id for a name and store its mappings."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndeletes a notebook s id only. This doesn t delete the actual notebook.", "response": "def delete_notebook_id(self, notebook_id):\n \"\"\"Delete a notebook's id only. This doesn't delete the actual notebook.\"\"\"\n name = self.mapping[notebook_id]\n del self.mapping[notebook_id]\n del self.rev_mapping[name]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndoing a notebook exist?", "response": "def notebook_exists(self, notebook_id):\n \"\"\"Does a notebook exist?\"\"\"\n if notebook_id not in self.mapping:\n return False\n path = self.get_path_by_name(self.mapping[notebook_id])\n return os.path.isfile(path)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a full path to a notebook given its notebook_id.", "response": "def find_path(self, notebook_id):\n \"\"\"Return a full path to a notebook given its notebook_id.\"\"\"\n try:\n name = self.mapping[notebook_id]\n except KeyError:\n raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)\n return self.get_path_by_name(name)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a full path to a notebook given its name.", "response": "def get_path_by_name(self, name):\n \"\"\"Return a full path to a notebook given its name.\"\"\"\n filename = name + self.filename_ext\n path = os.path.join(self.notebook_dir, filename)\n return path"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_notebook(self, notebook_id, format=u'json'):\n format = unicode(format)\n if format not in self.allowed_formats:\n raise web.HTTPError(415, u'Invalid notebook format: %s' % format)\n last_modified, nb = self.get_notebook_object(notebook_id)\n kwargs = {}\n if format == 'json':\n # don't split lines for sending over the wire, because it\n # should match the Python in-memory format.\n kwargs['split_lines'] = False\n data = current.writes(nb, format, **kwargs)\n name = nb.metadata.get('name','notebook')\n return last_modified, name, data", "response": "Get the representation of a notebook in format by notebook_id."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the NotebookNode representation of a notebook by notebook_id.", "response": "def get_notebook_object(self, notebook_id):\n \"\"\"Get the NotebookNode representation of a notebook by notebook_id.\"\"\"\n path = self.find_path(notebook_id)\n if not os.path.isfile(path):\n raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)\n info = os.stat(path)\n last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime)\n with open(path,'r') as f:\n s = f.read()\n try:\n # v1 and v2 and json in the .ipynb files.\n nb = current.reads(s, u'json')\n except:\n raise web.HTTPError(500, u'Unreadable JSON notebook.')\n # Always use the filename as the notebook name.\n nb.metadata.name = os.path.splitext(os.path.basename(path))[0]\n return last_modified, nb"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves a new notebook and return its notebook_id.", "response": "def save_new_notebook(self, data, name=None, format=u'json'):\n \"\"\"Save a new notebook and return its notebook_id.\n\n If a name is passed in, it overrides any values in the notebook data\n and the value in the data is updated to use that value.\n \"\"\"\n if format not in self.allowed_formats:\n raise web.HTTPError(415, u'Invalid notebook format: %s' % format)\n\n try:\n nb = current.reads(data.decode('utf-8'), format)\n except:\n raise web.HTTPError(400, u'Invalid JSON data')\n\n if name is None:\n try:\n name = nb.metadata.name\n except AttributeError:\n raise web.HTTPError(400, u'Missing notebook name')\n nb.metadata.name = name\n\n notebook_id = self.new_notebook_id(name)\n self.save_notebook_object(notebook_id, nb)\n return notebook_id"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsave an existing notebook by notebook_id.", "response": "def save_notebook(self, notebook_id, data, name=None, format=u'json'):\n \"\"\"Save an existing notebook by notebook_id.\"\"\"\n if format not in self.allowed_formats:\n raise web.HTTPError(415, u'Invalid notebook format: %s' % format)\n\n try:\n nb = current.reads(data.decode('utf-8'), format)\n except:\n raise web.HTTPError(400, u'Invalid JSON data')\n\n if name is not None:\n nb.metadata.name = name\n self.save_notebook_object(notebook_id, nb)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsave an existing notebook object by notebook_id.", "response": "def save_notebook_object(self, notebook_id, nb):\n \"\"\"Save an existing notebook object by notebook_id.\"\"\"\n if notebook_id not in self.mapping:\n raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)\n old_name = self.mapping[notebook_id]\n try:\n new_name = nb.metadata.name\n except AttributeError:\n raise web.HTTPError(400, u'Missing notebook name')\n path = self.get_path_by_name(new_name)\n try:\n with open(path,'w') as f:\n current.write(nb, f, u'json')\n except Exception as e:\n raise web.HTTPError(400, u'Unexpected error while saving notebook: %s' % e)\n # save .py script as well\n if self.save_script:\n pypath = os.path.splitext(path)[0] + '.py'\n try:\n with io.open(pypath,'w', encoding='utf-8') as f:\n current.write(nb, f, u'py')\n except Exception as e:\n raise web.HTTPError(400, u'Unexpected error while saving notebook as script: %s' % e)\n \n if old_name != new_name:\n old_path = self.get_path_by_name(old_name)\n if os.path.isfile(old_path):\n os.unlink(old_path)\n if self.save_script:\n old_pypath = os.path.splitext(old_path)[0] + '.py'\n if os.path.isfile(old_pypath):\n os.unlink(old_pypath)\n self.mapping[notebook_id] = new_name\n self.rev_mapping[new_name] = notebook_id\n del self.rev_mapping[old_name]"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndeletes notebook by notebook_id.", "response": "def delete_notebook(self, notebook_id):\n \"\"\"Delete notebook by notebook_id.\"\"\"\n path = self.find_path(notebook_id)\n if not os.path.isfile(path):\n raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)\n os.unlink(path)\n self.delete_notebook_id(notebook_id)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a non - used filename of the form basename.", "response": "def increment_filename(self, basename):\n \"\"\"Return a non-used filename of the form basename.\n \n This searches through the filenames (basename0, basename1, ...)\n until is find one that is not already being used. It is used to\n create Untitled and Copy names that are unique.\n \"\"\"\n i = 0\n while True:\n name = u'%s%i' % (basename,i)\n path = self.get_path_by_name(name)\n if not os.path.isfile(path):\n break\n else:\n i = i+1\n return path, name"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef new_notebook(self):\n path, name = self.increment_filename('Untitled')\n notebook_id = self.new_notebook_id(name)\n metadata = current.new_metadata(name=name)\n nb = current.new_notebook(metadata=metadata)\n with open(path,'w') as f:\n current.write(nb, f, u'json')\n return notebook_id", "response": "Create a new notebook and return its notebook_id."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncopy an existing notebook and return its notebook_id.", "response": "def copy_notebook(self, notebook_id):\n \"\"\"Copy an existing notebook and return its notebook_id.\"\"\"\n last_mod, nb = self.get_notebook_object(notebook_id)\n name = nb.metadata.name + '-Copy'\n path, name = self.increment_filename(name)\n nb.metadata.name = name\n notebook_id = self.new_notebook_id(name)\n self.save_notebook_object(notebook_id, nb)\n return notebook_id"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nyields all tokens that are part of the given line.", "response": "def phys_tokens(toks):\n \"\"\"Return all physical tokens, even line continuations.\n\n tokenize.generate_tokens() doesn't return a token for the backslash that\n continues lines. This wrapper provides those tokens so that we can\n re-create a faithful representation of the original source.\n\n Returns the same values as generate_tokens()\n\n \"\"\"\n last_line = None\n last_lineno = -1\n last_ttype = None\n for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks:\n if last_lineno != elineno:\n if last_line and last_line.endswith(\"\\\\\\n\"):\n # We are at the beginning of a new line, and the last line\n # ended with a backslash. We probably have to inject a\n # backslash token into the stream. Unfortunately, there's more\n # to figure out. This code::\n #\n # usage = \"\"\"\\\n # HEY THERE\n # \"\"\"\n #\n # triggers this condition, but the token text is::\n #\n # '\"\"\"\\\\\\nHEY THERE\\n\"\"\"'\n #\n # so we need to figure out if the backslash is already in the\n # string token or not.\n inject_backslash = True\n if last_ttype == tokenize.COMMENT:\n # Comments like this \\\n # should never result in a new token.\n inject_backslash = False\n elif ttype == token.STRING:\n if \"\\n\" in ttext and ttext.split('\\n', 1)[0][-1] == '\\\\':\n # It's a multiline string and the first line ends with\n # a backslash, so we don't need to inject another.\n inject_backslash = False\n if inject_backslash:\n # Figure out what column the backslash is in.\n ccol = len(last_line.split(\"\\n\")[-2]) - 1\n # Yield the token, with a fake token type.\n yield (\n 99999, \"\\\\\\n\",\n (slineno, ccol), (slineno, ccol+2),\n last_line\n )\n last_line = ltext\n last_ttype = ttype\n yield ttype, ttext, (slineno, scol), (elineno, ecol), ltext\n last_lineno = elineno"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef source_token_lines(source):\n ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])\n line = []\n col = 0\n source = source.expandtabs(8).replace('\\r\\n', '\\n')\n tokgen = generate_tokens(source)\n for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):\n mark_start = True\n for part in re.split('(\\n)', ttext):\n if part == '\\n':\n yield line\n line = []\n col = 0\n mark_end = False\n elif part == '':\n mark_end = False\n elif ttype in ws_tokens:\n mark_end = False\n else:\n if mark_start and scol > col:\n line.append((\"ws\", \" \" * (scol - col)))\n mark_start = False\n tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]\n if ttype == token.NAME and keyword.iskeyword(ttext):\n tok_class = \"key\"\n line.append((tok_class, part))\n mark_end = True\n scol = 0\n if mark_end:\n col = ecol\n\n if line:\n yield line", "response": "Yields a series of lines one for each line in source."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef source_encoding(source):\n # Note: this function should never be called on Python 3, since py3 has\n # built-in tools to do this.\n assert sys.version_info < (3, 0)\n\n # This is mostly code adapted from Py3.2's tokenize module.\n\n cookie_re = re.compile(r\"coding[:=]\\s*([-\\w.]+)\")\n\n # Do this so the detect_encode code we copied will work.\n readline = iter(source.splitlines(True)).next\n\n def _get_normal_name(orig_enc):\n \"\"\"Imitates get_normal_name in tokenizer.c.\"\"\"\n # Only care about the first 12 characters.\n enc = orig_enc[:12].lower().replace(\"_\", \"-\")\n if re.match(r\"^utf-8($|-)\", enc):\n return \"utf-8\"\n if re.match(r\"^(latin-1|iso-8859-1|iso-latin-1)($|-)\", enc):\n return \"iso-8859-1\"\n return orig_enc\n\n # From detect_encode():\n # It detects the encoding from the presence of a utf-8 bom or an encoding\n # cookie as specified in pep-0263. If both a bom and a cookie are present,\n # but disagree, a SyntaxError will be raised. If the encoding cookie is an\n # invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,\n # 'utf-8-sig' is returned.\n\n # If no encoding is specified, then the default will be returned. The\n # default varied with version.\n\n if sys.version_info <= (2, 4):\n default = 'iso-8859-1'\n else:\n default = 'ascii'\n\n bom_found = False\n encoding = None\n\n def read_or_stop():\n \"\"\"Get the next source line, or ''.\"\"\"\n try:\n return readline()\n except StopIteration:\n return ''\n\n def find_cookie(line):\n \"\"\"Find an encoding cookie in `line`.\"\"\"\n try:\n line_string = line.decode('ascii')\n except UnicodeDecodeError:\n return None\n\n matches = cookie_re.findall(line_string)\n if not matches:\n return None\n encoding = _get_normal_name(matches[0])\n try:\n codec = codecs.lookup(encoding)\n except LookupError:\n # This behaviour mimics the Python interpreter\n raise SyntaxError(\"unknown encoding: \" + encoding)\n\n if bom_found:\n # codecs in 2.3 were raw tuples of functions, assume the best.\n codec_name = getattr(codec, 'name', encoding)\n if codec_name != 'utf-8':\n # This behaviour mimics the Python interpreter\n raise SyntaxError('encoding problem: utf-8')\n encoding += '-sig'\n return encoding\n\n first = read_or_stop()\n if first.startswith(codecs.BOM_UTF8):\n bom_found = True\n first = first[3:]\n default = 'utf-8-sig'\n if not first:\n return default\n\n encoding = find_cookie(first)\n if encoding:\n return encoding\n\n second = read_or_stop()\n if not second:\n return default\n\n encoding = find_cookie(second)\n if encoding:\n return encoding\n\n return default", "response": "Determine the encoding for source."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads the default config file from the default ipython_dir.", "response": "def load_default_config(ipython_dir=None):\n \"\"\"Load the default config file from the default ipython_dir.\n\n This is useful for embedded shells.\n \"\"\"\n if ipython_dir is None:\n ipython_dir = get_ipython_dir()\n profile_dir = os.path.join(ipython_dir, 'profile_default')\n cl = PyFileConfigLoader(default_config_file_name, profile_dir)\n try:\n config = cl.load_config()\n except ConfigFileNotFound:\n # no config found\n config = Config()\n return config"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef make_report(self,traceback):\n\n sec_sep = self.section_sep\n # Start with parent report\n report = [super(IPAppCrashHandler, self).make_report(traceback)]\n # Add interactive-specific info we may have\n rpt_add = report.append\n try:\n rpt_add(sec_sep+\"History of session input:\")\n for line in self.app.shell.user_ns['_ih']:\n rpt_add(line)\n rpt_add('\\n*** Last line of input (may not be in above history):\\n')\n rpt_add(self.app.shell._last_input_line+'\\n')\n except:\n pass\n\n return ''.join(report)", "response": "Return a string containing a crash report."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _classes_default(self):\n return [\n InteractiveShellApp, # ShellApp comes before TerminalApp, because\n self.__class__, # it will also affect subclasses (e.g. QtConsole)\n TerminalInteractiveShell,\n PromptManager,\n HistoryManager,\n ProfileDir,\n PlainTextFormatter,\n IPCompleter,\n ScriptMagics,\n ]", "response": "Returns a list of classes that can be used for this instance of TerminalIPythonApp."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse_command_line(self, argv=None):\n\n argv = sys.argv[1:] if argv is None else argv\n\n if '-pylab' in argv:\n # deprecated `-pylab` given,\n # warn and transform into current syntax\n argv = argv[:] # copy, don't clobber\n idx = argv.index('-pylab')\n warn.warn(\"`-pylab` flag has been deprecated.\\n\"\n \" Use `--pylab` instead, or `--pylab=foo` to specify a backend.\")\n sub = '--pylab'\n if len(argv) > idx+1:\n # check for gui arg, as in '-pylab qt'\n gui = argv[idx+1]\n if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):\n sub = '--pylab='+gui\n argv.pop(idx+1)\n argv[idx] = sub\n\n return super(TerminalIPythonApp, self).parse_command_line(argv)", "response": "override to allow old - pylab flag with deprecation warning"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef initialize(self, argv=None):\n super(TerminalIPythonApp, self).initialize(argv)\n if self.subapp is not None:\n # don't bother initializing further, starting subapp\n return\n if not self.ignore_old_config:\n check_for_old_config(self.ipython_dir)\n # print self.extra_args\n if self.extra_args and not self.something_to_run:\n self.file_to_run = self.extra_args[0]\n self.init_path()\n # create the shell\n self.init_shell()\n # and draw the banner\n self.init_banner()\n # Now a variety of things that happen after the banner is printed.\n self.init_gui_pylab()\n self.init_extensions()\n self.init_code()", "response": "Do actions after construct but before starting the app."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef init_shell(self):\n # Create an InteractiveShell instance.\n # shell.display_banner should always be False for the terminal\n # based app, because we call shell.show_banner() by hand below\n # so the banner shows *before* all extension loading stuff.\n self.shell = TerminalInteractiveShell.instance(config=self.config,\n display_banner=False, profile_dir=self.profile_dir,\n ipython_dir=self.ipython_dir)\n self.shell.configurables.append(self)", "response": "initialize the InteractiveShell instance"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncall when the pylab attribute is changed.", "response": "def _pylab_changed(self, name, old, new):\n \"\"\"Replace --pylab='inline' with --pylab='auto'\"\"\"\n if new == 'inline':\n warn.warn(\"'inline' not available as pylab backend, \"\n \"using 'auto' instead.\\n\")\n self.pylab = 'auto'"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the class name of an object.", "response": "def class_of ( object ):\n \"\"\" Returns a string containing the class name of an object with the\n correct indefinite article ('a' or 'an') preceding it (e.g., 'an Image',\n 'a PlotValue').\n \"\"\"\n if isinstance( object, basestring ):\n return add_article( object )\n\n return add_article( object.__class__.__name__ )"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a string representation of a value and its type for readable error messages.", "response": "def repr_type(obj):\n \"\"\" Return a string representation of a value and its type for readable\n error messages.\n \"\"\"\n the_type = type(obj)\n if (not py3compat.PY3) and the_type is InstanceType:\n # Old-style class.\n the_type = obj.__class__\n msg = '%r %r' % (obj, the_type)\n return msg"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert the name argument to a list of names.", "response": "def parse_notifier_name(name):\n \"\"\"Convert the name argument to a list of names.\n\n Examples\n --------\n\n >>> parse_notifier_name('a')\n ['a']\n >>> parse_notifier_name(['a','b'])\n ['a', 'b']\n >>> parse_notifier_name(None)\n ['anytrait']\n \"\"\"\n if isinstance(name, str):\n return [name]\n elif name is None:\n return ['anytrait']\n elif isinstance(name, (list, tuple)):\n for n in name:\n assert isinstance(n, str), \"names must be strings\"\n return name"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the default value on a per instance basis.", "response": "def set_default_value(self, obj):\n \"\"\"Set the default value on a per instance basis.\n\n This method is called by :meth:`instance_init` to create and\n validate the default value. The creation and validation of\n default values must be delayed until the parent :class:`HasTraits`\n class has been instantiated.\n \"\"\"\n # Check for a deferred initializer defined in the same class as the\n # trait declaration or above.\n mro = type(obj).mro()\n meth_name = '_%s_default' % self.name\n for cls in mro[:mro.index(self.this_class)+1]:\n if meth_name in cls.__dict__:\n break\n else:\n # We didn't find one. Do static initialization.\n dv = self.get_default_value()\n newdv = self._validate(obj, dv)\n obj._trait_values[self.name] = newdv\n return\n # Complete the dynamic initialization.\n obj._trait_dyn_inits[self.name] = cls.__dict__[meth_name]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting a list of all the traits of this class.", "response": "def class_traits(cls, **metadata):\n \"\"\"Get a list of all the traits of this class.\n\n This method is just like the :meth:`traits` method, but is unbound.\n\n The TraitTypes returned don't know anything about the values\n that the various HasTrait's instances are holding.\n\n This follows the same algorithm as traits does and does not allow\n for any simple way of specifying merely that a metadata name\n exists, but has any value. This is because get_metadata returns\n None if a metadata key doesn't exist.\n \"\"\"\n traits = dict([memb for memb in getmembers(cls) if \\\n isinstance(memb[1], TraitType)])\n\n if len(metadata) == 0:\n return traits\n\n for meta_name, meta_eval in metadata.items():\n if type(meta_eval) is not FunctionType:\n metadata[meta_name] = _SimpleTest(meta_eval)\n\n result = {}\n for name, trait in traits.items():\n for meta_name, meta_eval in metadata.items():\n if not meta_eval(trait.get_metadata(meta_name)):\n break\n else:\n result[name] = trait\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets metadata values for trait by key.", "response": "def trait_metadata(self, traitname, key):\n \"\"\"Get metadata values for trait by key.\"\"\"\n try:\n trait = getattr(self.__class__, traitname)\n except AttributeError:\n raise TraitError(\"Class %s does not have a trait named %s\" %\n (self.__class__.__name__, traitname))\n else:\n return trait.get_metadata(key)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nvalidating that the value is a valid object instance.", "response": "def validate(self, obj, value):\n \"\"\"Validates that the value is a valid object instance.\"\"\"\n try:\n if issubclass(value, self.klass):\n return value\n except:\n if (value is None) and (self._allow_none):\n return value\n\n self.error(obj, value)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a description of the trait.", "response": "def info(self):\n \"\"\" Returns a description of the trait.\"\"\"\n if isinstance(self.klass, basestring):\n klass = self.klass\n else:\n klass = self.klass.__name__\n result = 'a subclass of ' + klass\n if self._allow_none:\n return result + ' or None'\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef info(self):\n result = 'any of ' + repr(self.values)\n if self._allow_none:\n return result + ' or None'\n return result", "response": "Returns a description of the trait."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef require(*mods):\n names = []\n for mod in mods:\n if isinstance(mod, ModuleType):\n mod = mod.__name__\n \n if isinstance(mod, basestring):\n names.append(mod)\n else:\n raise TypeError(\"names must be modules or module names, not %s\"%type(mod))\n \n return depend(_require, *names)", "response": "Simple decorator for requiring names to be importable."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef check(self, completed, failed=None):\n if len(self) == 0:\n return True\n against = set()\n if self.success:\n against = completed\n if failed is not None and self.failure:\n against = against.union(failed)\n if self.all:\n return self.issubset(against)\n else:\n return not self.isdisjoint(against)", "response": "check whether our dependencies have been met."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn whether this dependency has become impossible.", "response": "def unreachable(self, completed, failed=None):\n \"\"\"return whether this dependency has become impossible.\"\"\"\n if len(self) == 0:\n return False\n against = set()\n if not self.success:\n against = completed\n if failed is not None and not self.failure:\n against = against.union(failed)\n if self.all:\n return not self.isdisjoint(against)\n else:\n return self.issubset(against)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef as_dict(self):\n return dict(\n dependencies=list(self),\n all=self.all,\n success=self.success,\n failure=self.failure\n )", "response": "Represent this dependency as a dict. For json compatibility."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a Solver instance", "response": "def Ainv(self):\n 'Returns a Solver instance'\n\n if not hasattr(self, '_Ainv'):\n self._Ainv = self.Solver(self.A)\n return self._Ainv"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a Solver instance", "response": "def Ainv(self):\n 'Returns a Solver instance'\n\n if getattr(self, '_Ainv', None) is None:\n self._Ainv = self.Solver(self.A, 13)\n self._Ainv.run_pardiso(12)\n return self._Ainv"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconstruct a dict representation of a binary tree", "response": "def bintree(ids, parent=None):\n \"\"\"construct {child:parent} dict representation of a binary tree\n \n keys are the nodes in the tree, and values are the parent of each node.\n \n The root node has parent `parent`, default: None.\n \n >>> tree = bintree(range(7))\n >>> tree\n {0: None, 1: 0, 2: 1, 3: 1, 4: 0, 5: 4, 6: 4}\n >>> print_bintree(tree)\n 0\n 1\n 2\n 3\n 4\n 5\n 6\n \"\"\"\n parents = {}\n n = len(ids)\n if n == 0:\n return parents\n root = ids[0]\n parents[root] = parent\n if len(ids) == 1:\n return parents\n else:\n ids = ids[1:]\n n = len(ids)\n left = bintree(ids[:n/2], parent=root)\n right = bintree(ids[n/2:], parent=root)\n parents.update(left)\n parents.update(right)\n return parents"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef reverse_bintree(parents):\n children = {}\n for child,parent in parents.iteritems():\n if parent is None:\n children[None] = child\n continue\n elif parent not in children:\n children[parent] = []\n children[parent].append(child)\n \n return children", "response": "reverse bintree - > tree"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the depth of an element in the tree", "response": "def depth(n, tree):\n \"\"\"get depth of an element in the tree\"\"\"\n d = 0\n parent = tree[n]\n while parent is not None:\n d += 1\n parent = tree[parent]\n return d"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprint a binary tree", "response": "def print_bintree(tree, indent=' '):\n \"\"\"print a binary tree\"\"\"\n for n in sorted(tree.keys()):\n print \"%s%s\" % (indent * depth(n,tree), n)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\naccepting either IP address or dns name and return IP", "response": "def disambiguate_dns_url(url, location):\n \"\"\"accept either IP address or dns name, and return IP\"\"\"\n if not ip_pat.match(location):\n location = socket.gethostbyname(location)\n return disambiguate_url(url, location)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef connect(self, peers, btree, pub_url, root_id=0):\n \n # count the number of children we have\n self.nchildren = btree.values().count(self.id)\n \n if self.root:\n return # root only binds\n \n root_location = peers[root_id][-1]\n self.sub.connect(disambiguate_dns_url(pub_url, root_location))\n \n parent = btree[self.id]\n \n tree_url, location = peers[parent]\n self.upstream.connect(disambiguate_dns_url(tree_url, location))", "response": "connect to peers. `peers` will be a dict of 4-tuples, keyed by name.\n {peer : (ident, addr, pub_addr, location)}\n where peer is the name, ident is the XREP identity, addr,pub_addr are the"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparallel reduce on binary tree", "response": "def reduce(self, f, value, flat=True, all=False):\n \"\"\"parallel reduce on binary tree\n \n if flat:\n value is an entry in the sequence\n else:\n value is a list of entries in the sequence\n \n if all:\n broadcast final result to all nodes\n else:\n only root gets final result\n \"\"\"\n if not flat:\n value = reduce(f, value)\n \n for i in range(self.nchildren):\n value = f(value, self.recv_downstream())\n \n if not self.root:\n self.send_upstream(value)\n \n if all:\n if self.root:\n self.publish(value)\n else:\n value = self.consume()\n return value"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef allreduce(self, f, value, flat=True):\n return self.reduce(f, value, flat=flat, all=True)", "response": "parallel reduce followed by broadcast of the result"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef init_hub(self):\n client_iface = \"%s://%s:\" % (self.client_transport, self.client_ip) + \"%i\"\n engine_iface = \"%s://%s:\" % (self.engine_transport, self.engine_ip) + \"%i\"\n\n ctx = self.context\n loop = self.loop\n\n # Registrar socket\n q = ZMQStream(ctx.socket(zmq.ROUTER), loop)\n q.bind(client_iface % self.regport)\n self.log.info(\"Hub listening on %s for registration.\", client_iface % self.regport)\n if self.client_ip != self.engine_ip:\n q.bind(engine_iface % self.regport)\n self.log.info(\"Hub listening on %s for registration.\", engine_iface % self.regport)\n\n ### Engine connections ###\n\n # heartbeat\n hpub = ctx.socket(zmq.PUB)\n hpub.bind(engine_iface % self.hb[0])\n hrep = ctx.socket(zmq.ROUTER)\n hrep.bind(engine_iface % self.hb[1])\n self.heartmonitor = HeartMonitor(loop=loop, config=self.config, log=self.log,\n pingstream=ZMQStream(hpub,loop),\n pongstream=ZMQStream(hrep,loop)\n )\n\n ### Client connections ###\n # Notifier socket\n n = ZMQStream(ctx.socket(zmq.PUB), loop)\n n.bind(client_iface%self.notifier_port)\n\n ### build and launch the queues ###\n\n # monitor socket\n sub = ctx.socket(zmq.SUB)\n sub.setsockopt(zmq.SUBSCRIBE, b\"\")\n sub.bind(self.monitor_url)\n sub.bind('inproc://monitor')\n sub = ZMQStream(sub, loop)\n\n # connect the db\n db_class = _db_shortcuts.get(self.db_class.lower(), self.db_class)\n self.log.info('Hub using DB backend: %r', (db_class.split('.')[-1]))\n self.db = import_item(str(db_class))(session=self.session.session,\n config=self.config, log=self.log)\n time.sleep(.25)\n try:\n scheme = self.config.TaskScheduler.scheme_name\n except AttributeError:\n from .scheduler import TaskScheduler\n scheme = TaskScheduler.scheme_name.get_default_value()\n # build connection dicts\n self.engine_info = {\n 'control' : engine_iface%self.control[1],\n 'mux': engine_iface%self.mux[1],\n 'heartbeat': (engine_iface%self.hb[0], engine_iface%self.hb[1]),\n 'task' : engine_iface%self.task[1],\n 'iopub' : engine_iface%self.iopub[1],\n # 'monitor' : engine_iface%self.mon_port,\n }\n\n self.client_info = {\n 'control' : client_iface%self.control[0],\n 'mux': client_iface%self.mux[0],\n 'task' : (scheme, client_iface%self.task[0]),\n 'iopub' : client_iface%self.iopub[0],\n 'notification': client_iface%self.notifier_port\n }\n self.log.debug(\"Hub engine addrs: %s\", self.engine_info)\n self.log.debug(\"Hub client addrs: %s\", self.client_info)\n\n # resubmit stream\n r = ZMQStream(ctx.socket(zmq.DEALER), loop)\n url = util.disambiguate_url(self.client_info['task'][-1])\n r.setsockopt(zmq.IDENTITY, self.session.bsession)\n r.connect(url)\n\n self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,\n query=q, notifier=n, resubmit=r, db=self.db,\n engine_info=self.engine_info, client_info=self.client_info,\n log=self.log)", "response": "Initialize the hub with the same client and engine interfaces."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nturns any valid targets argument into a list of integer ids", "response": "def _validate_targets(self, targets):\n \"\"\"turn any valid targets argument into a list of integer ids\"\"\"\n if targets is None:\n # default to all\n return self.ids\n\n if isinstance(targets, (int,str,unicode)):\n # only one target specified\n targets = [targets]\n _targets = []\n for t in targets:\n # map raw identities to ids\n if isinstance(t, (str,unicode)):\n t = self.by_ident.get(cast_bytes(t), t)\n _targets.append(t)\n targets = _targets\n bad_targets = [ t for t in targets if t not in self.ids ]\n if bad_targets:\n raise IndexError(\"No Such Engine: %r\" % bad_targets)\n if not targets:\n raise IndexError(\"No Engines Registered\")\n return targets"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nroutes registration requests and queries from clients.", "response": "def dispatch_query(self, msg):\n \"\"\"Route registration requests and queries from clients.\"\"\"\n try:\n idents, msg = self.session.feed_identities(msg)\n except ValueError:\n idents = []\n if not idents:\n self.log.error(\"Bad Query Message: %r\", msg)\n return\n client_id = idents[0]\n try:\n msg = self.session.unserialize(msg, content=True)\n except Exception:\n content = error.wrap_exception()\n self.log.error(\"Bad Query Message: %r\", msg, exc_info=True)\n self.session.send(self.query, \"hub_error\", ident=client_id,\n content=content)\n return\n # print client_id, header, parent, content\n #switch on message type:\n msg_type = msg['header']['msg_type']\n self.log.info(\"client::client %r requested %r\", client_id, msg_type)\n handler = self.query_handlers.get(msg_type, None)\n try:\n assert handler is not None, \"Bad Message Type: %r\" % msg_type\n except:\n content = error.wrap_exception()\n self.log.error(\"Bad Message Type: %r\", msg_type, exc_info=True)\n self.session.send(self.query, \"hub_error\", ident=client_id,\n content=content)\n return\n\n else:\n handler(idents, msg)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef handle_new_heart(self, heart):\n self.log.debug(\"heartbeat::handle_new_heart(%r)\", heart)\n if heart not in self.incoming_registrations:\n self.log.info(\"heartbeat::ignoring new heart: %r\", heart)\n else:\n self.finish_registration(heart)", "response": "handler to attach to heartbeater.\n Called when a new heart starts to be beat.\n Called when a new heart starts to be beat.\n Triggers completion of registration."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef handle_heart_failure(self, heart):\n self.log.debug(\"heartbeat::handle_heart_failure(%r)\", heart)\n eid = self.hearts.get(heart, None)\n queue = self.engines[eid].queue\n if eid is None or self.keytable[eid] in self.dead_engines:\n self.log.info(\"heartbeat::ignoring heart failure %r (not an engine or already dead)\", heart)\n else:\n self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))", "response": "handle_heart_failure - handle a heartbeat failure."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef save_task_request(self, idents, msg):\n client_id = idents[0]\n\n try:\n msg = self.session.unserialize(msg)\n except Exception:\n self.log.error(\"task::client %r sent invalid task message: %r\",\n client_id, msg, exc_info=True)\n return\n record = init_record(msg)\n\n record['client_uuid'] = client_id.decode('ascii')\n record['queue'] = 'task'\n header = msg['header']\n msg_id = header['msg_id']\n self.pending.add(msg_id)\n self.unassigned.add(msg_id)\n try:\n # it's posible iopub arrived first:\n existing = self.db.get_record(msg_id)\n if existing['resubmitted']:\n for key in ('submitted', 'client_uuid', 'buffers'):\n # don't clobber these keys on resubmit\n # submitted and client_uuid should be different\n # and buffers might be big, and shouldn't have changed\n record.pop(key)\n # still check content,header which should not change\n # but are not expensive to compare as buffers\n\n for key,evalue in existing.iteritems():\n if key.endswith('buffers'):\n # don't compare buffers\n continue\n rvalue = record.get(key, None)\n if evalue and rvalue and evalue != rvalue:\n self.log.warn(\"conflicting initial state for record: %r:%r <%r> %r\", msg_id, rvalue, key, evalue)\n elif evalue and not rvalue:\n record[key] = evalue\n try:\n self.db.update_record(msg_id, record)\n except Exception:\n self.log.error(\"DB Error updating record %r\", msg_id, exc_info=True)\n except KeyError:\n try:\n self.db.add_record(msg_id, record)\n except Exception:\n self.log.error(\"DB Error adding record %r\", msg_id, exc_info=True)\n except Exception:\n self.log.error(\"DB Error saving task request %r\", msg_id, exc_info=True)", "response": "Save the submission of a task."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsaving the result of a completed task.", "response": "def save_task_result(self, idents, msg):\n \"\"\"save the result of a completed task.\"\"\"\n client_id = idents[0]\n try:\n msg = self.session.unserialize(msg)\n except Exception:\n self.log.error(\"task::invalid task result message send to %r: %r\",\n client_id, msg, exc_info=True)\n return\n\n parent = msg['parent_header']\n if not parent:\n # print msg\n self.log.warn(\"Task %r had no parent!\", msg)\n return\n msg_id = parent['msg_id']\n if msg_id in self.unassigned:\n self.unassigned.remove(msg_id)\n\n header = msg['header']\n engine_uuid = header.get('engine', u'')\n eid = self.by_ident.get(cast_bytes(engine_uuid), None)\n \n status = header.get('status', None)\n\n if msg_id in self.pending:\n self.log.info(\"task::task %r finished on %s\", msg_id, eid)\n self.pending.remove(msg_id)\n self.all_completed.add(msg_id)\n if eid is not None:\n if status != 'aborted':\n self.completed[eid].append(msg_id)\n if msg_id in self.tasks[eid]:\n self.tasks[eid].remove(msg_id)\n completed = header['date']\n started = header.get('started', None)\n result = {\n 'result_header' : header,\n 'result_content': msg['content'],\n 'started' : started,\n 'completed' : completed,\n 'received' : datetime.now(),\n 'engine_uuid': engine_uuid,\n }\n\n result['result_buffers'] = msg['buffers']\n try:\n self.db.update_record(msg_id, result)\n except Exception:\n self.log.error(\"DB Error saving task request %r\", msg_id, exc_info=True)\n\n else:\n self.log.debug(\"task::unknown task %r finished\", msg_id)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves an iopub message into the database", "response": "def save_iopub_message(self, topics, msg):\n \"\"\"save an iopub message into the db\"\"\"\n # print (topics)\n try:\n msg = self.session.unserialize(msg, content=True)\n except Exception:\n self.log.error(\"iopub::invalid IOPub message\", exc_info=True)\n return\n\n parent = msg['parent_header']\n if not parent:\n self.log.warn(\"iopub::IOPub message lacks parent: %r\", msg)\n return\n msg_id = parent['msg_id']\n msg_type = msg['header']['msg_type']\n content = msg['content']\n\n # ensure msg_id is in db\n try:\n rec = self.db.get_record(msg_id)\n except KeyError:\n rec = empty_record()\n rec['msg_id'] = msg_id\n self.db.add_record(msg_id, rec)\n # stream\n d = {}\n if msg_type == 'stream':\n name = content['name']\n s = rec[name] or ''\n d[name] = s + content['data']\n\n elif msg_type == 'pyerr':\n d['pyerr'] = content\n elif msg_type == 'pyin':\n d['pyin'] = content['code']\n elif msg_type in ('display_data', 'pyout'):\n d[msg_type] = content\n elif msg_type == 'status':\n pass\n else:\n self.log.warn(\"unhandled iopub msg_type: %r\", msg_type)\n\n if not d:\n return\n\n try:\n self.db.update_record(msg_id, d)\n except Exception:\n self.log.error(\"DB Error saving iopub message %r\", msg_id, exc_info=True)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreply with connection addresses for clients.", "response": "def connection_request(self, client_id, msg):\n \"\"\"Reply with connection addresses for clients.\"\"\"\n self.log.info(\"client::client %r connected\", client_id)\n content = dict(status='ok')\n content.update(self.client_info)\n jsonable = {}\n for k,v in self.keytable.iteritems():\n if v not in self.dead_engines:\n jsonable[str(k)] = v.decode('ascii')\n content['engines'] = jsonable\n self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nregisters a new engine and create the sockets necessary.", "response": "def register_engine(self, reg, msg):\n \"\"\"Register a new engine.\"\"\"\n content = msg['content']\n try:\n queue = cast_bytes(content['queue'])\n except KeyError:\n self.log.error(\"registration::queue not specified\", exc_info=True)\n return\n heart = content.get('heartbeat', None)\n if heart:\n heart = cast_bytes(heart)\n \"\"\"register a new engine, and create the socket(s) necessary\"\"\"\n eid = self._next_id\n # print (eid, queue, reg, heart)\n\n self.log.debug(\"registration::register_engine(%i, %r, %r, %r)\", eid, queue, reg, heart)\n\n content = dict(id=eid,status='ok')\n content.update(self.engine_info)\n # check if requesting available IDs:\n if queue in self.by_ident:\n try:\n raise KeyError(\"queue_id %r in use\" % queue)\n except:\n content = error.wrap_exception()\n self.log.error(\"queue_id %r in use\", queue, exc_info=True)\n elif heart in self.hearts: # need to check unique hearts?\n try:\n raise KeyError(\"heart_id %r in use\" % heart)\n except:\n self.log.error(\"heart_id %r in use\", heart, exc_info=True)\n content = error.wrap_exception()\n else:\n for h, pack in self.incoming_registrations.iteritems():\n if heart == h:\n try:\n raise KeyError(\"heart_id %r in use\" % heart)\n except:\n self.log.error(\"heart_id %r in use\", heart, exc_info=True)\n content = error.wrap_exception()\n break\n elif queue == pack[1]:\n try:\n raise KeyError(\"queue_id %r in use\" % queue)\n except:\n self.log.error(\"queue_id %r in use\", queue, exc_info=True)\n content = error.wrap_exception()\n break\n\n msg = self.session.send(self.query, \"registration_reply\",\n content=content,\n ident=reg)\n\n if content['status'] == 'ok':\n if heart in self.heartmonitor.hearts:\n # already beating\n self.incoming_registrations[heart] = (eid,queue,reg[0],None)\n self.finish_registration(heart)\n else:\n purge = lambda : self._purge_stalled_registration(heart)\n dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)\n dc.start()\n self.incoming_registrations[heart] = (eid,queue,reg[0],dc)\n else:\n self.log.error(\"registration::registration %i failed: %r\", eid, content['evalue'])\n return eid"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nhandle stranded messages from the queue.", "response": "def _handle_stranded_msgs(self, eid, uuid):\n \"\"\"Handle messages known to be on an engine when the engine unregisters.\n\n It is possible that this will fire prematurely - that is, an engine will\n go down after completing a result, and the client will be notified\n that the result failed and later receive the actual result.\n \"\"\"\n\n outstanding = self.queues[eid]\n\n for msg_id in outstanding:\n self.pending.remove(msg_id)\n self.all_completed.add(msg_id)\n try:\n raise error.EngineError(\"Engine %r died while running task %r\" % (eid, msg_id))\n except:\n content = error.wrap_exception()\n # build a fake header:\n header = {}\n header['engine'] = uuid\n header['date'] = datetime.now()\n rec = dict(result_content=content, result_header=header, result_buffers=[])\n rec['completed'] = header['date']\n rec['engine_uuid'] = uuid\n try:\n self.db.update_record(msg_id, rec)\n except Exception:\n self.log.error(\"DB Error handling stranded msg %r\", msg_id, exc_info=True)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef finish_registration(self, heart):\n try:\n (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)\n except KeyError:\n self.log.error(\"registration::tried to finish nonexistant registration\", exc_info=True)\n return\n self.log.info(\"registration::finished registering engine %i:%r\", eid, queue)\n if purge is not None:\n purge.stop()\n control = queue\n self.ids.add(eid)\n self.keytable[eid] = queue\n self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg,\n control=control, heartbeat=heart)\n self.by_ident[queue] = eid\n self.queues[eid] = list()\n self.tasks[eid] = list()\n self.completed[eid] = list()\n self.hearts[heart] = eid\n content = dict(id=eid, queue=self.engines[eid].queue.decode('ascii'))\n if self.notifier:\n self.session.send(self.notifier, \"registration_notification\", content=content)\n self.log.info(\"engine::Engine Connected: %i\", eid)", "response": "Finishes the engine registration."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef queue_status(self, client_id, msg):\n content = msg['content']\n targets = content['targets']\n try:\n targets = self._validate_targets(targets)\n except:\n content = error.wrap_exception()\n self.session.send(self.query, \"hub_error\",\n content=content, ident=client_id)\n return\n verbose = content.get('verbose', False)\n content = dict(status='ok')\n for t in targets:\n queue = self.queues[t]\n completed = self.completed[t]\n tasks = self.tasks[t]\n if not verbose:\n queue = len(queue)\n completed = len(completed)\n tasks = len(tasks)\n content[str(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks}\n content['unassigned'] = list(self.unassigned) if verbose else len(self.unassigned)\n # print (content)\n self.session.send(self.query, \"queue_reply\", content=content, ident=client_id)", "response": "Return the status of one or more targets."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef purge_results(self, client_id, msg):\n content = msg['content']\n self.log.info(\"Dropping records with %s\", content)\n msg_ids = content.get('msg_ids', [])\n reply = dict(status='ok')\n if msg_ids == 'all':\n try:\n self.db.drop_matching_records(dict(completed={'$ne':None}))\n except Exception:\n reply = error.wrap_exception()\n else:\n pending = filter(lambda m: m in self.pending, msg_ids)\n if pending:\n try:\n raise IndexError(\"msg pending: %r\" % pending[0])\n except:\n reply = error.wrap_exception()\n else:\n try:\n self.db.drop_matching_records(dict(msg_id={'$in':msg_ids}))\n except Exception:\n reply = error.wrap_exception()\n\n if reply['status'] == 'ok':\n eids = content.get('engine_ids', [])\n for eid in eids:\n if eid not in self.engines:\n try:\n raise IndexError(\"No such engine: %i\" % eid)\n except:\n reply = error.wrap_exception()\n break\n uid = self.engines[eid].queue\n try:\n self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))\n except Exception:\n reply = error.wrap_exception()\n break\n\n self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)", "response": "Purge results from memory. This method is more valuable before we move\n to a DB based message storage mechanism. This method is more valuable before we move\n to a DB based message storage mechanism. This method is more valuable before we move\n to a DB based message storage mechanism."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef resubmit_task(self, client_id, msg):\n def finish(reply):\n self.session.send(self.query, 'resubmit_reply', content=reply, ident=client_id)\n\n content = msg['content']\n msg_ids = content['msg_ids']\n reply = dict(status='ok')\n try:\n records = self.db.find_records({'msg_id' : {'$in' : msg_ids}}, keys=[\n 'header', 'content', 'buffers'])\n except Exception:\n self.log.error('db::db error finding tasks to resubmit', exc_info=True)\n return finish(error.wrap_exception())\n\n # validate msg_ids\n found_ids = [ rec['msg_id'] for rec in records ]\n pending_ids = [ msg_id for msg_id in found_ids if msg_id in self.pending ]\n if len(records) > len(msg_ids):\n try:\n raise RuntimeError(\"DB appears to be in an inconsistent state.\"\n \"More matching records were found than should exist\")\n except Exception:\n return finish(error.wrap_exception())\n elif len(records) < len(msg_ids):\n missing = [ m for m in msg_ids if m not in found_ids ]\n try:\n raise KeyError(\"No such msg(s): %r\" % missing)\n except KeyError:\n return finish(error.wrap_exception())\n elif pending_ids:\n pass\n # no need to raise on resubmit of pending task, now that we\n # resubmit under new ID, but do we want to raise anyway?\n # msg_id = invalid_ids[0]\n # try:\n # raise ValueError(\"Task(s) %r appears to be inflight\" % )\n # except Exception:\n # return finish(error.wrap_exception())\n\n # mapping of original IDs to resubmitted IDs\n resubmitted = {}\n\n # send the messages\n for rec in records:\n header = rec['header']\n msg = self.session.msg(header['msg_type'], parent=header)\n msg_id = msg['msg_id']\n msg['content'] = rec['content']\n \n # use the old header, but update msg_id and timestamp\n fresh = msg['header']\n header['msg_id'] = fresh['msg_id']\n header['date'] = fresh['date']\n msg['header'] = header\n\n self.session.send(self.resubmit, msg, buffers=rec['buffers'])\n\n resubmitted[rec['msg_id']] = msg_id\n self.pending.add(msg_id)\n msg['buffers'] = rec['buffers']\n try:\n self.db.add_record(msg_id, init_record(msg))\n except Exception:\n self.log.error(\"db::DB Error updating record: %s\", msg_id, exc_info=True)\n\n finish(dict(status='ok', resubmitted=resubmitted))\n \n # store the new IDs in the Task DB\n for msg_id, resubmit_id in resubmitted.iteritems():\n try:\n self.db.update_record(msg_id, {'resubmitted' : resubmit_id})\n except Exception:\n self.log.error(\"db::DB Error updating record: %s\", msg_id, exc_info=True)", "response": "Resubmit one or more tasks."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _extract_record(self, rec):\n io_dict = {}\n for key in ('pyin', 'pyout', 'pyerr', 'stdout', 'stderr'):\n io_dict[key] = rec[key]\n content = { 'result_content': rec['result_content'],\n 'header': rec['header'],\n 'result_header' : rec['result_header'],\n 'received' : rec['received'],\n 'io' : io_dict,\n }\n if rec['result_buffers']:\n buffers = map(bytes, rec['result_buffers'])\n else:\n buffers = []\n\n return content, buffers", "response": "decompose a TaskRecord dict into subsection of reply for get_result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_results(self, client_id, msg):\n content = msg['content']\n msg_ids = sorted(set(content['msg_ids']))\n statusonly = content.get('status_only', False)\n pending = []\n completed = []\n content = dict(status='ok')\n content['pending'] = pending\n content['completed'] = completed\n buffers = []\n if not statusonly:\n try:\n matches = self.db.find_records(dict(msg_id={'$in':msg_ids}))\n # turn match list into dict, for faster lookup\n records = {}\n for rec in matches:\n records[rec['msg_id']] = rec\n except Exception:\n content = error.wrap_exception()\n self.session.send(self.query, \"result_reply\", content=content,\n parent=msg, ident=client_id)\n return\n else:\n records = {}\n for msg_id in msg_ids:\n if msg_id in self.pending:\n pending.append(msg_id)\n elif msg_id in self.all_completed:\n completed.append(msg_id)\n if not statusonly:\n c,bufs = self._extract_record(records[msg_id])\n content[msg_id] = c\n buffers.extend(bufs)\n elif msg_id in records:\n if rec['completed']:\n completed.append(msg_id)\n c,bufs = self._extract_record(records[msg_id])\n content[msg_id] = c\n buffers.extend(bufs)\n else:\n pending.append(msg_id)\n else:\n try:\n raise KeyError('No such message: '+msg_id)\n except:\n content = error.wrap_exception()\n break\n self.session.send(self.query, \"result_reply\", content=content,\n parent=msg, ident=client_id,\n buffers=buffers)", "response": "Get the result of 1 or more messages."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget a list of all msg_ids in our DB records", "response": "def get_history(self, client_id, msg):\n \"\"\"Get a list of all msg_ids in our DB records\"\"\"\n try:\n msg_ids = self.db.get_history()\n except Exception as e:\n content = error.wrap_exception()\n else:\n content = dict(status='ok', history=msg_ids)\n\n self.session.send(self.query, \"history_reply\", content=content,\n parent=msg, ident=client_id)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nperforms a raw query on the task record database.", "response": "def db_query(self, client_id, msg):\n \"\"\"Perform a raw query on the task record database.\"\"\"\n content = msg['content']\n query = content.get('query', {})\n keys = content.get('keys', None)\n buffers = []\n empty = list()\n try:\n records = self.db.find_records(query, keys)\n except Exception as e:\n content = error.wrap_exception()\n else:\n # extract buffers from reply content:\n if keys is not None:\n buffer_lens = [] if 'buffers' in keys else None\n result_buffer_lens = [] if 'result_buffers' in keys else None\n else:\n buffer_lens = None\n result_buffer_lens = None\n\n for rec in records:\n # buffers may be None, so double check\n b = rec.pop('buffers', empty) or empty\n if buffer_lens is not None:\n buffer_lens.append(len(b))\n buffers.extend(b)\n rb = rec.pop('result_buffers', empty) or empty\n if result_buffer_lens is not None:\n result_buffer_lens.append(len(rb))\n buffers.extend(rb)\n content = dict(status='ok', records=records, buffer_lens=buffer_lens,\n result_buffer_lens=result_buffer_lens)\n # self.log.debug (content)\n self.session.send(self.query, \"db_reply\", content=content,\n parent=msg, ident=client_id,\n buffers=buffers)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchanges the path of the current directory to newdir", "response": "def cd(self, newdir):\n \"\"\"\n go to the path\n \"\"\"\n prevdir = os.getcwd()\n os.chdir(newdir)\n try:\n yield\n finally:\n os.chdir(prevdir)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndecode a command output into a ParsedCompletedCommand object.", "response": "def decode_cmd_out(self, completed_cmd):\n \"\"\"\n return a standard message\n \"\"\"\n try:\n stdout = completed_cmd.stdout.encode('utf-8').decode()\n except AttributeError:\n try:\n stdout = str(bytes(completed_cmd.stdout), 'big5').strip()\n except AttributeError:\n stdout = str(bytes(completed_cmd.stdout).decode('utf-8')).strip()\n try:\n stderr = completed_cmd.stderr.encode('utf-8').decode()\n except AttributeError:\n try:\n stderr = str(bytes(completed_cmd.stderr), 'big5').strip()\n except AttributeError:\n stderr = str(bytes(completed_cmd.stderr).decode('utf-8')).strip()\n return ParsedCompletedCommand(\n completed_cmd.returncode,\n completed_cmd.args,\n stdout,\n stderr\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nruns command under the r - root directory", "response": "def run_command_under_r_root(self, cmd, catched=True):\n \"\"\"\n subprocess run on here\n \"\"\"\n RPATH = self.path\n with self.cd(newdir=RPATH):\n if catched:\n process = sp.run(cmd, stdout=sp.PIPE, stderr=sp.PIPE)\n else:\n process = sp.run(cmd)\n return process"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef execute(self):\n rprocess = OrderedDict()\n commands = OrderedDict([\n (self.file, ['Rscript', self.file] + self.cmd),\n ])\n for cmd_name, cmd in commands.items():\n rprocess[cmd_name] = self.run_command_under_r_root(cmd)\n \n return self.decode_cmd_out(completed_cmd=rprocess[self.file])", "response": "Execute R script and return a list of results."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _set_kernel_manager(self, kernel_manager):\n # Disconnect the old kernel manager, if necessary.\n old_manager = self._kernel_manager\n if old_manager is not None:\n old_manager.started_kernel.disconnect(self._started_kernel)\n old_manager.started_channels.disconnect(self._started_channels)\n old_manager.stopped_channels.disconnect(self._stopped_channels)\n\n # Disconnect the old kernel manager's channels.\n old_manager.sub_channel.message_received.disconnect(self._dispatch)\n old_manager.shell_channel.message_received.disconnect(self._dispatch)\n old_manager.stdin_channel.message_received.disconnect(self._dispatch)\n old_manager.hb_channel.kernel_died.disconnect(\n self._handle_kernel_died)\n\n # Handle the case where the old kernel manager is still listening.\n if old_manager.channels_running:\n self._stopped_channels()\n\n # Set the new kernel manager.\n self._kernel_manager = kernel_manager\n if kernel_manager is None:\n return\n\n # Connect the new kernel manager.\n kernel_manager.started_kernel.connect(self._started_kernel)\n kernel_manager.started_channels.connect(self._started_channels)\n kernel_manager.stopped_channels.connect(self._stopped_channels)\n\n # Connect the new kernel manager's channels.\n kernel_manager.sub_channel.message_received.connect(self._dispatch)\n kernel_manager.shell_channel.message_received.connect(self._dispatch)\n kernel_manager.stdin_channel.message_received.connect(self._dispatch)\n kernel_manager.hb_channel.kernel_died.connect(self._handle_kernel_died)\n\n # Handle the case where the kernel manager started channels before\n # we connected.\n if kernel_manager.channels_running:\n self._started_channels()", "response": "Connect to the current kernel manager and set a new kernel manager."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _dispatch(self, msg):\n msg_type = msg['header']['msg_type']\n handler = getattr(self, '_handle_' + msg_type, None)\n if handler:\n handler(msg)", "response": "Dispatches the message to the frontend handler associated with the message type of the\n given message."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning whether a reply from the kernel originated from a request from this frontend.", "response": "def _is_from_this_session(self, msg):\n \"\"\" Returns whether a reply from the kernel originated from a request\n from this frontend.\n \"\"\"\n session = self._kernel_manager.session.session\n parent = msg['parent_header']\n if not parent:\n # if the message has no parent, assume it is meant for all frontends\n return True\n else:\n return parent.get('session') == session"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef report(self, morfs, directory=None):\n self.report_files(self.annotate_file, morfs, directory)", "response": "Run the report.\n\n See `coverage.report()` for arguments."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef annotate_file(self, cu, analysis):\n if not cu.relative:\n return\n\n filename = cu.filename\n source = cu.source_file()\n if self.directory:\n dest_file = os.path.join(self.directory, cu.flat_rootname())\n dest_file += \".py,cover\"\n else:\n dest_file = filename + \",cover\"\n dest = open(dest_file, 'w')\n\n statements = sorted(analysis.statements)\n missing = sorted(analysis.missing)\n excluded = sorted(analysis.excluded)\n\n lineno = 0\n i = 0\n j = 0\n covered = True\n while True:\n line = source.readline()\n if line == '':\n break\n lineno += 1\n while i < len(statements) and statements[i] < lineno:\n i += 1\n while j < len(missing) and missing[j] < lineno:\n j += 1\n if i < len(statements) and statements[i] == lineno:\n covered = j >= len(missing) or missing[j] > lineno\n if self.blank_re.match(line):\n dest.write(' ')\n elif self.else_re.match(line):\n # Special logic for lines containing only 'else:'.\n if i >= len(statements) and j >= len(missing):\n dest.write('! ')\n elif i >= len(statements) or j >= len(missing):\n dest.write('> ')\n elif statements[i] == missing[j]:\n dest.write('! ')\n else:\n dest.write('> ')\n elif lineno in excluded:\n dest.write('- ')\n elif covered:\n dest.write('> ')\n else:\n dest.write('! ')\n dest.write(line)\n source.close()\n dest.close()", "response": "Annotate a single file with the information from the analysis."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find(name):\n '''\n returns a list of tuples (package_name, description) for apt-cache search results\n '''\n cmd = 'apt-cache search %s' % name\n args = shlex.split(cmd)\n try:\n output = subprocess.check_output(args)\n except CalledProcessError:\n return []\n lines = output.splitlines()\n packages = []\n for line in lines:\n package, _, desc = line.partition(' - ')\n packages.append((package, desc)) \n return packages", "response": "returns a list of tuples ( package_name description ) for apt - cache search results\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_installed_version(name):\n '''\n returns installed package version and None if package is not installed\n '''\n pattern = re.compile(r'''Installed:\\s+(?P.*)''')\n cmd = 'apt-cache policy %s' % name\n args = shlex.split(cmd)\n try:\n output = subprocess.check_output(args)\n if not output:\n return None\n except CalledProcessError:\n return None\n # check output\n match = pattern.search(output)\n if match:\n version = match.groupdict()['version']\n if version == '(none)':\n return None\n else:\n return version", "response": "Returns installed version and None if package is not installed"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef squash_unicode(obj):\n if isinstance(obj,dict):\n for key in obj.keys():\n obj[key] = squash_unicode(obj[key])\n if isinstance(key, unicode):\n obj[squash_unicode(key)] = obj.pop(key)\n elif isinstance(obj, list):\n for i,v in enumerate(obj):\n obj[i] = squash_unicode(v)\n elif isinstance(obj, unicode):\n obj = obj.encode('utf8')\n return obj", "response": "coerce unicode back to bytestrings."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the default behavior for a config environment to be secure.", "response": "def default_secure(cfg):\n \"\"\"Set the default behavior for a config environment to be secure.\n \n If Session.key/keyfile have not been set, set Session.key to\n a new random UUID.\n \"\"\"\n \n if 'Session' in cfg:\n if 'key' in cfg.Session or 'keyfile' in cfg.Session:\n return\n # key/keyfile not specified, generate new UUID:\n cfg.Session.key = str_to_bytes(str(uuid.uuid4()))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngiving a message or header return the header.", "response": "def extract_header(msg_or_header):\n \"\"\"Given a message or header, return the header.\"\"\"\n if not msg_or_header:\n return {}\n try:\n # See if msg_or_header is the entire message.\n h = msg_or_header['header']\n except KeyError:\n try:\n # See if msg_or_header is just the header\n h = msg_or_header['msg_id']\n except KeyError:\n raise\n else:\n h = msg_or_header\n if not isinstance(h, dict):\n h = dict(h)\n return h"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck packers for binary data and datetime support.", "response": "def _check_packers(self):\n \"\"\"check packers for binary data and datetime support.\"\"\"\n pack = self.pack\n unpack = self.unpack\n\n # check simple serialization\n msg = dict(a=[1,'hi'])\n try:\n packed = pack(msg)\n except Exception:\n raise ValueError(\"packer could not serialize a simple message\")\n\n # ensure packed message is bytes\n if not isinstance(packed, bytes):\n raise ValueError(\"message packed to %r, but bytes are required\"%type(packed))\n\n # check that unpack is pack's inverse\n try:\n unpacked = unpack(packed)\n except Exception:\n raise ValueError(\"unpacker could not handle the packer's output\")\n\n # check datetime support\n msg = dict(t=datetime.now())\n try:\n unpacked = unpack(pack(msg))\n except Exception:\n self.pack = lambda o: pack(squash_dates(o))\n self.unpack = lambda s: extract_dates(unpack(s))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sign(self, msg_list):\n if self.auth is None:\n return b''\n h = self.auth.copy()\n for m in msg_list:\n h.update(m)\n return str_to_bytes(h.hexdigest())", "response": "Sign a message with HMAC digest."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef serialize(self, msg, ident=None):\n content = msg.get('content', {})\n if content is None:\n content = self.none\n elif isinstance(content, dict):\n content = self.pack(content)\n elif isinstance(content, bytes):\n # content is already packed, as in a relayed message\n pass\n elif isinstance(content, unicode):\n # should be bytes, but JSON often spits out unicode\n content = content.encode('utf8')\n else:\n raise TypeError(\"Content incorrect type: %s\"%type(content))\n\n real_message = [self.pack(msg['header']),\n self.pack(msg['parent_header']),\n content\n ]\n\n to_send = []\n\n if isinstance(ident, list):\n # accept list of idents\n to_send.extend(ident)\n elif ident is not None:\n to_send.append(ident)\n to_send.append(DELIM)\n\n signature = self.sign(real_message)\n to_send.append(signature)\n\n to_send.extend(real_message)\n\n return to_send", "response": "Serialize the message components to bytes."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef send(self, stream, msg_or_type, content=None, parent=None, ident=None,\n buffers=None, subheader=None, track=False, header=None):\n \"\"\"Build and send a message via stream or socket.\n\n The message format used by this function internally is as follows:\n\n [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,\n buffer1,buffer2,...]\n\n The serialize/unserialize methods convert the nested message dict into this\n format.\n\n Parameters\n ----------\n\n stream : zmq.Socket or ZMQStream\n The socket-like object used to send the data.\n msg_or_type : str or Message/dict\n Normally, msg_or_type will be a msg_type unless a message is being\n sent more than once. If a header is supplied, this can be set to\n None and the msg_type will be pulled from the header.\n\n content : dict or None\n The content of the message (ignored if msg_or_type is a message).\n header : dict or None\n The header dict for the message (ignores if msg_to_type is a message).\n parent : Message or dict or None\n The parent or parent header describing the parent of this message\n (ignored if msg_or_type is a message).\n ident : bytes or list of bytes\n The zmq.IDENTITY routing path.\n subheader : dict or None\n Extra header keys for this message's header (ignored if msg_or_type\n is a message).\n buffers : list or None\n The already-serialized buffers to be appended to the message.\n track : bool\n Whether to track. Only for use with Sockets, because ZMQStream\n objects cannot track messages.\n\n Returns\n -------\n msg : dict\n The constructed message.\n (msg,tracker) : (dict, MessageTracker)\n if track=True, then a 2-tuple will be returned,\n the first element being the constructed\n message, and the second being the MessageTracker\n\n \"\"\"\n\n if not isinstance(stream, (zmq.Socket, ZMQStream)):\n raise TypeError(\"stream must be Socket or ZMQStream, not %r\"%type(stream))\n elif track and isinstance(stream, ZMQStream):\n raise TypeError(\"ZMQStream cannot track messages\")\n\n if isinstance(msg_or_type, (Message, dict)):\n # We got a Message or message dict, not a msg_type so don't\n # build a new Message.\n msg = msg_or_type\n else:\n msg = self.msg(msg_or_type, content=content, parent=parent,\n subheader=subheader, header=header)\n\n buffers = [] if buffers is None else buffers\n to_send = self.serialize(msg, ident)\n flag = 0\n if buffers:\n flag = zmq.SNDMORE\n _track = False\n else:\n _track=track\n if track:\n tracker = stream.send_multipart(to_send, flag, copy=False, track=_track)\n else:\n tracker = stream.send_multipart(to_send, flag, copy=False)\n for b in buffers[:-1]:\n stream.send(b, flag, copy=False)\n if buffers:\n if track:\n tracker = stream.send(buffers[-1], copy=False, track=track)\n else:\n tracker = stream.send(buffers[-1], copy=False)\n\n # omsg = Message(msg)\n if self.debug:\n pprint.pprint(msg)\n pprint.pprint(to_send)\n pprint.pprint(buffers)\n\n msg['tracker'] = tracker\n\n return msg", "response": "Builds and sends a message via a socket or a message."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):\n to_send = []\n if isinstance(ident, bytes):\n ident = [ident]\n if ident is not None:\n to_send.extend(ident)\n\n to_send.append(DELIM)\n to_send.append(self.sign(msg_list))\n to_send.extend(msg_list)\n stream.send_multipart(msg_list, flags, copy=copy)", "response": "Send a message via the raw socket."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):\n if isinstance(socket, ZMQStream):\n socket = socket.socket\n try:\n msg_list = socket.recv_multipart(mode, copy=copy)\n except zmq.ZMQError as e:\n if e.errno == zmq.EAGAIN:\n # We can convert EAGAIN to None as we know in this case\n # recv_multipart won't return None.\n return None,None\n else:\n raise\n # split multipart message into identity list and message dict\n # invalid large messages can cause very expensive string comparisons\n idents, msg_list = self.feed_identities(msg_list, copy)\n try:\n return idents, self.unserialize(msg_list, content=content, copy=copy)\n except Exception as e:\n # TODO: handle it\n raise e", "response": "Receive and unpack a message from a socket."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfeeding the identities from the beginning of a message into two lists of bytes.", "response": "def feed_identities(self, msg_list, copy=True):\n \"\"\"Split the identities from the rest of the message.\n\n Feed until DELIM is reached, then return the prefix as idents and\n remainder as msg_list. This is easily broken by setting an IDENT to DELIM,\n but that would be silly.\n\n Parameters\n ----------\n msg_list : a list of Message or bytes objects\n The message to be split.\n copy : bool\n flag determining whether the arguments are bytes or Messages\n\n Returns\n -------\n (idents, msg_list) : two lists\n idents will always be a list of bytes, each of which is a ZMQ\n identity. msg_list will be a list of bytes or zmq.Messages of the\n form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and\n should be unpackable/unserializable via self.unserialize at this\n point.\n \"\"\"\n if copy:\n idx = msg_list.index(DELIM)\n return msg_list[:idx], msg_list[idx+1:]\n else:\n failed = True\n for idx,m in enumerate(msg_list):\n if m.bytes == DELIM:\n failed = False\n break\n if failed:\n raise ValueError(\"DELIM not in msg_list\")\n idents, msg_list = msg_list[:idx], msg_list[idx+1:]\n return [m.bytes for m in idents], msg_list"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_svg(string, parent=None):\n if isinstance(string, unicode):\n string = string.encode('utf-8')\n\n dialog = QtGui.QFileDialog(parent, 'Save SVG Document')\n dialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)\n dialog.setDefaultSuffix('svg')\n dialog.setNameFilter('SVG document (*.svg)')\n if dialog.exec_():\n filename = dialog.selectedFiles()[0]\n f = open(filename, 'w')\n try:\n f.write(string)\n finally:\n f.close()\n return filename\n return None", "response": "Prompts the user to save an SVG document to disk."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef svg_to_clipboard(string):\n if isinstance(string, unicode):\n string = string.encode('utf-8')\n\n mime_data = QtCore.QMimeData()\n mime_data.setData('image/svg+xml', string)\n QtGui.QApplication.clipboard().setMimeData(mime_data)", "response": "Copy a SVG document to the clipboard."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts a SVG document to an image.", "response": "def svg_to_image(string, size=None):\n \"\"\" Convert a SVG document to a QImage.\n\n Parameters:\n -----------\n string : basestring\n A Python string containing a SVG document.\n\n size : QSize, optional\n The size of the image that is produced. If not specified, the SVG\n document's default size is used.\n \n Raises:\n -------\n ValueError\n If an invalid SVG string is provided.\n\n Returns:\n --------\n A QImage of format QImage.Format_ARGB32.\n \"\"\"\n if isinstance(string, unicode):\n string = string.encode('utf-8')\n\n renderer = QtSvg.QSvgRenderer(QtCore.QByteArray(string))\n if not renderer.isValid():\n raise ValueError('Invalid SVG data.')\n\n if size is None:\n size = renderer.defaultSize()\n image = QtGui.QImage(size, QtGui.QImage.Format_ARGB32)\n painter = QtGui.QPainter(image)\n renderer.render(painter)\n return image"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nmaking an object info dict with all fields present.", "response": "def object_info(**kw):\n \"\"\"Make an object info dict with all fields present.\"\"\"\n infodict = dict(izip_longest(info_fields, [None]))\n infodict.update(kw)\n return infodict"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getdoc(obj):\n # Allow objects to offer customized documentation via a getdoc method:\n try:\n ds = obj.getdoc()\n except Exception:\n pass\n else:\n # if we get extra info, we add it to the normal docstring.\n if isinstance(ds, basestring):\n return inspect.cleandoc(ds)\n \n try:\n return inspect.getdoc(obj)\n except Exception:\n # Harden against an inspect failure, which can occur with\n # SWIG-wrapped extensions.\n return None", "response": "Stable wrapper around inspect. getdoc."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwraps around inspect. getsource.", "response": "def getsource(obj,is_binary=False):\n \"\"\"Wrapper around inspect.getsource.\n\n This can be modified by other projects to provide customized source\n extraction.\n\n Inputs:\n\n - obj: an object whose source code we will attempt to extract.\n\n Optional inputs:\n\n - is_binary: whether the object is known to come from a binary source.\n This implementation will skip returning any output for binary objects, but\n custom extractors may know how to meaningfully process them.\"\"\"\n\n if is_binary:\n return None\n else:\n # get source if obj was decorated with @decorator\n if hasattr(obj,\"__wrapped__\"):\n obj = obj.__wrapped__\n try:\n src = inspect.getsource(obj)\n except TypeError:\n if hasattr(obj,'__class__'):\n src = inspect.getsource(obj.__class__)\n return src"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getargspec(obj):\n\n if inspect.isfunction(obj):\n func_obj = obj\n elif inspect.ismethod(obj):\n func_obj = obj.im_func\n elif hasattr(obj, '__call__'):\n func_obj = obj.__call__\n else:\n raise TypeError('arg is not a Python function')\n args, varargs, varkw = inspect.getargs(func_obj.func_code)\n return args, varargs, varkw, func_obj.func_defaults", "response": "Get the names and default values of a Python function."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nextracts call tip data from an object info dict.", "response": "def call_tip(oinfo, format_call=True):\n \"\"\"Extract call tip data from an oinfo dict.\n\n Parameters\n ----------\n oinfo : dict\n\n format_call : bool, optional\n If True, the call line is formatted and returned as a string. If not, a\n tuple of (name, argspec) is returned.\n\n Returns\n -------\n call_info : None, str or (str, dict) tuple.\n When format_call is True, the whole call information is formattted as a\n single string. Otherwise, the object's name and its argspec dict are\n returned. If no call information is available, None is returned.\n\n docstring : str or None\n The most relevant docstring for calling purposes is returned, if\n available. The priority is: call docstring for callable instances, then\n constructor docstring for classes, then main object's docstring otherwise\n (regular functions).\n \"\"\"\n # Get call definition\n argspec = oinfo.get('argspec')\n if argspec is None:\n call_line = None\n else:\n # Callable objects will have 'self' as their first argument, prune\n # it out if it's there for clarity (since users do *not* pass an\n # extra first argument explicitly).\n try:\n has_self = argspec['args'][0] == 'self'\n except (KeyError, IndexError):\n pass\n else:\n if has_self:\n argspec['args'] = argspec['args'][1:]\n\n call_line = oinfo['name']+format_argspec(argspec)\n\n # Now get docstring.\n # The priority is: call docstring, constructor docstring, main one.\n doc = oinfo.get('call_docstring')\n if doc is None:\n doc = oinfo.get('init_docstring')\n if doc is None:\n doc = oinfo.get('docstring','')\n\n return call_line, doc"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfinds the absolute path to the file where an object was defined.", "response": "def find_file(obj):\n \"\"\"Find the absolute path to the file where an object was defined.\n\n This is essentially a robust wrapper around `inspect.getabsfile`.\n\n Returns None if no file can be found.\n\n Parameters\n ----------\n obj : any Python object\n\n Returns\n -------\n fname : str\n The absolute path to the file where the object was defined.\n \"\"\"\n # get source if obj was decorated with @decorator\n if hasattr(obj, '__wrapped__'):\n obj = obj.__wrapped__\n\n fname = None\n try:\n fname = inspect.getabsfile(obj)\n except TypeError:\n # For an instance, the file that matters is where its class was\n # declared.\n if hasattr(obj, '__class__'):\n try:\n fname = inspect.getabsfile(obj.__class__)\n except TypeError:\n # Can happen for builtins\n pass\n except:\n pass\n return fname"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef find_source_lines(obj):\n # get source if obj was decorated with @decorator\n if hasattr(obj, '__wrapped__'):\n obj = obj.__wrapped__\n \n try:\n try:\n lineno = inspect.getsourcelines(obj)[1]\n except TypeError:\n # For instances, try the class object like getsource() does\n if hasattr(obj, '__class__'):\n lineno = inspect.getsourcelines(obj.__class__)[1]\n except:\n return None\n\n return lineno", "response": "Find the line number in a file where an object was defined."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the definition header for any callable object.", "response": "def _getdef(self,obj,oname=''):\n \"\"\"Return the definition header for any callable object.\n\n If any exception is generated, None is returned instead and the\n exception is suppressed.\"\"\"\n\n try:\n # We need a plain string here, NOT unicode!\n hdef = oname + inspect.formatargspec(*getargspec(obj))\n return py3compat.unicode_to_str(hdef, 'ascii')\n except:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __head(self,h):\n return '%s%s%s' % (self.color_table.active_colors.header,h,\n self.color_table.active_colors.normal)", "response": "Return a header string with proper colors."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef noinfo(self, msg, oname):\n print 'No %s found' % msg,\n if oname:\n print 'for %s' % oname\n else:\n print", "response": "Generic message when no information is found."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pdef(self, obj, oname=''):\n\n if not callable(obj):\n print 'Object is not callable.'\n return\n\n header = ''\n\n if inspect.isclass(obj):\n header = self.__head('Class constructor information:\\n')\n obj = obj.__init__\n elif (not py3compat.PY3) and type(obj) is types.InstanceType:\n obj = obj.__call__\n\n output = self._getdef(obj,oname)\n if output is None:\n self.noinfo('definition header',oname)\n else:\n print >>io.stdout, header,self.format(output),", "response": "Print the definition header for any callable object."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprint the docstring for any object.", "response": "def pdoc(self,obj,oname='',formatter = None):\n \"\"\"Print the docstring for any object.\n\n Optional:\n -formatter: a function to run the docstring through for specially\n formatted docstrings.\n\n Examples\n --------\n\n In [1]: class NoInit:\n ...: pass\n\n In [2]: class NoDoc:\n ...: def __init__(self):\n ...: pass\n\n In [3]: %pdoc NoDoc\n No documentation found for NoDoc\n\n In [4]: %pdoc NoInit\n No documentation found for NoInit\n\n In [5]: obj = NoInit()\n\n In [6]: %pdoc obj\n No documentation found for obj\n\n In [5]: obj2 = NoDoc()\n\n In [6]: %pdoc obj2\n No documentation found for obj2\n \"\"\"\n\n head = self.__head # For convenience\n lines = []\n ds = getdoc(obj)\n if formatter:\n ds = formatter(ds)\n if ds:\n lines.append(head(\"Class Docstring:\"))\n lines.append(indent(ds))\n if inspect.isclass(obj) and hasattr(obj, '__init__'):\n init_ds = getdoc(obj.__init__)\n if init_ds is not None:\n lines.append(head(\"Constructor Docstring:\"))\n lines.append(indent(init_ds))\n elif hasattr(obj,'__call__'):\n call_ds = getdoc(obj.__call__)\n if call_ds:\n lines.append(head(\"Calling Docstring:\"))\n lines.append(indent(call_ds))\n\n if not lines:\n self.noinfo('documentation',oname)\n else:\n page.page('\\n'.join(lines))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef psource(self,obj,oname=''):\n\n # Flush the source cache because inspect can return out-of-date source\n linecache.checkcache()\n try:\n src = getsource(obj)\n except:\n self.noinfo('source',oname)\n else:\n page.page(self.format(py3compat.unicode_to_str(src)))", "response": "Print the source code for an object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pfile(self, obj, oname=''):\n \n lineno = find_source_lines(obj)\n if lineno is None:\n self.noinfo('file', oname)\n return\n\n ofile = find_file(obj)\n # run contents of file through pager starting at line where the object\n # is defined, as long as the file isn't binary and is actually on the\n # filesystem.\n if ofile.endswith(('.so', '.dll', '.pyd')):\n print 'File %r is binary, not printing.' % ofile\n elif not os.path.isfile(ofile):\n print 'File %r does not exist, not printing.' % ofile\n else:\n # Print only text files, not extension binaries. Note that\n # getsourcelines returns lineno with 1-offset and page() uses\n # 0-offset, so we must adjust.\n page.page(self.format(open(ofile).read()), lineno-1)", "response": "Print the whole file where an object was defined."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _format_fields(self, fields, title_width=12):\n out = []\n header = self.__head\n for title, content in fields:\n if len(content.splitlines()) > 1:\n title = header(title + \":\") + \"\\n\"\n else:\n title = header((title+\":\").ljust(title_width))\n out.append(title + content)\n return \"\\n\".join(out)", "response": "Formats a list of fields for display."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pinfo(self,obj,oname='',formatter=None,info=None,detail_level=0):\n info = self.info(obj, oname=oname, formatter=formatter,\n info=info, detail_level=detail_level)\n displayfields = []\n def add_fields(fields):\n for title, key in fields:\n field = info[key]\n if field is not None:\n displayfields.append((title, field.rstrip()))\n \n add_fields(self.pinfo_fields1)\n \n # Base class for old-style instances\n if (not py3compat.PY3) and isinstance(obj, types.InstanceType) and info['base_class']:\n displayfields.append((\"Base Class\", info['base_class'].rstrip()))\n \n add_fields(self.pinfo_fields2)\n \n # Namespace\n if info['namespace'] != 'Interactive':\n displayfields.append((\"Namespace\", info['namespace'].rstrip()))\n\n add_fields(self.pinfo_fields3)\n \n # Source or docstring, depending on detail level and whether\n # source found.\n if detail_level > 0 and info['source'] is not None:\n displayfields.append((\"Source\", self.format(py3compat.cast_bytes_py2(info['source']))))\n elif info['docstring'] is not None:\n displayfields.append((\"Docstring\", info[\"docstring\"]))\n\n # Constructor info for classes\n if info['isclass']:\n if info['init_definition'] or info['init_docstring']:\n displayfields.append((\"Constructor information\", \"\"))\n if info['init_definition'] is not None:\n displayfields.append((\" Definition\",\n info['init_definition'].rstrip()))\n if info['init_docstring'] is not None:\n displayfields.append((\" Docstring\",\n indent(info['init_docstring'])))\n\n # Info for objects:\n else:\n add_fields(self.pinfo_fields_obj)\n\n # Finally send to printer/pager:\n if displayfields:\n page.page(self._format_fields(displayfields))", "response": "Show detailed information about an object."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute a dict with detailed information about an object.", "response": "def info(self, obj, oname='', formatter=None, info=None, detail_level=0):\n \"\"\"Compute a dict with detailed information about an object.\n\n Optional arguments:\n\n - oname: name of the variable pointing to the object.\n\n - formatter: special formatter for docstrings (see pdoc)\n\n - info: a structure with some information fields which may have been\n precomputed already.\n\n - detail_level: if set to 1, more information is given.\n \"\"\"\n\n obj_type = type(obj)\n\n header = self.__head\n if info is None:\n ismagic = 0\n isalias = 0\n ospace = ''\n else:\n ismagic = info.ismagic\n isalias = info.isalias\n ospace = info.namespace\n\n # Get docstring, special-casing aliases:\n if isalias:\n if not callable(obj):\n try:\n ds = \"Alias to the system command:\\n %s\" % obj[1]\n except:\n ds = \"Alias: \" + str(obj)\n else:\n ds = \"Alias to \" + str(obj)\n if obj.__doc__:\n ds += \"\\nDocstring:\\n\" + obj.__doc__\n else:\n ds = getdoc(obj)\n if ds is None:\n ds = ''\n if formatter is not None:\n ds = formatter(ds)\n\n # store output in a dict, we initialize it here and fill it as we go\n out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic)\n\n string_max = 200 # max size of strings to show (snipped if longer)\n shalf = int((string_max -5)/2)\n\n if ismagic:\n obj_type_name = 'Magic function'\n elif isalias:\n obj_type_name = 'System alias'\n else:\n obj_type_name = obj_type.__name__\n out['type_name'] = obj_type_name\n\n try:\n bclass = obj.__class__\n out['base_class'] = str(bclass)\n except: pass\n\n # String form, but snip if too long in ? form (full in ??)\n if detail_level >= self.str_detail_level:\n try:\n ostr = str(obj)\n str_head = 'string_form'\n if not detail_level and len(ostr)>string_max:\n ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]\n ostr = (\"\\n\" + \" \" * len(str_head.expandtabs())).\\\n join(q.strip() for q in ostr.split(\"\\n\"))\n out[str_head] = ostr\n except:\n pass\n\n if ospace:\n out['namespace'] = ospace\n\n # Length (for strings and lists)\n try:\n out['length'] = str(len(obj))\n except: pass\n\n # Filename where object was defined\n binary_file = False\n fname = find_file(obj)\n if fname is None:\n # if anything goes wrong, we don't want to show source, so it's as\n # if the file was binary\n binary_file = True\n else:\n if fname.endswith(('.so', '.dll', '.pyd')):\n binary_file = True\n elif fname.endswith(''):\n fname = 'Dynamically generated function. No source code available.'\n out['file'] = fname\n \n # reconstruct the function definition and print it:\n defln = self._getdef(obj, oname)\n if defln:\n out['definition'] = self.format(defln)\n\n # Docstrings only in detail 0 mode, since source contains them (we\n # avoid repetitions). If source fails, we add them back, see below.\n if ds and detail_level == 0:\n out['docstring'] = ds\n\n # Original source code for any callable\n if detail_level:\n # Flush the source cache because inspect can return out-of-date\n # source\n linecache.checkcache()\n source = None\n try:\n try:\n source = getsource(obj, binary_file)\n except TypeError:\n if hasattr(obj, '__class__'):\n source = getsource(obj.__class__, binary_file)\n if source is not None:\n out['source'] = source.rstrip()\n except Exception:\n pass\n\n if ds and source is None:\n out['docstring'] = ds\n\n\n # Constructor docstring for classes\n if inspect.isclass(obj):\n out['isclass'] = True\n # reconstruct the function definition and print it:\n try:\n obj_init = obj.__init__\n except AttributeError:\n init_def = init_ds = None\n else:\n init_def = self._getdef(obj_init,oname)\n init_ds = getdoc(obj_init)\n # Skip Python's auto-generated docstrings\n if init_ds and \\\n init_ds.startswith('x.__init__(...) initializes'):\n init_ds = None\n\n if init_def or init_ds:\n if init_def:\n out['init_definition'] = self.format(init_def)\n if init_ds:\n out['init_docstring'] = init_ds\n\n # and class docstring for instances:\n else:\n # First, check whether the instance docstring is identical to the\n # class one, and print it separately if they don't coincide. In\n # most cases they will, but it's nice to print all the info for\n # objects which use instance-customized docstrings.\n if ds:\n try:\n cls = getattr(obj,'__class__')\n except:\n class_ds = None\n else:\n class_ds = getdoc(cls)\n # Skip Python's auto-generated docstrings\n if class_ds and \\\n (class_ds.startswith('function(code, globals[,') or \\\n class_ds.startswith('instancemethod(function, instance,') or \\\n class_ds.startswith('module(name[,') ):\n class_ds = None\n if class_ds and ds != class_ds:\n out['class_docstring'] = class_ds\n\n # Next, try to show constructor docstrings\n try:\n init_ds = getdoc(obj.__init__)\n # Skip Python's auto-generated docstrings\n if init_ds and \\\n init_ds.startswith('x.__init__(...) initializes'):\n init_ds = None\n except AttributeError:\n init_ds = None\n if init_ds:\n out['init_docstring'] = init_ds\n\n # Call form docstring for callable instances\n if hasattr(obj, '__call__'):\n call_def = self._getdef(obj.__call__, oname)\n if call_def is not None:\n out['call_def'] = self.format(call_def)\n call_ds = getdoc(obj.__call__)\n # Skip Python's auto-generated docstrings\n if call_ds and call_ds.startswith('x.__call__(...) <==> x(...)'):\n call_ds = None\n if call_ds:\n out['call_docstring'] = call_ds\n\n # Compute the object's argspec as a callable. The key is to decide\n # whether to pull it from the object itself, from its __init__ or\n # from its __call__ method.\n\n if inspect.isclass(obj):\n # Old-style classes need not have an __init__\n callable_obj = getattr(obj, \"__init__\", None)\n elif callable(obj):\n callable_obj = obj\n else:\n callable_obj = None\n\n if callable_obj:\n try:\n args, varargs, varkw, defaults = getargspec(callable_obj)\n except (TypeError, AttributeError):\n # For extensions/builtins we can't retrieve the argspec\n pass\n else:\n out['argspec'] = dict(args=args, varargs=varargs,\n varkw=varkw, defaults=defaults)\n\n return object_info(**out)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef psearch(self,pattern,ns_table,ns_search=[],\n ignore_case=False,show_all=False):\n \"\"\"Search namespaces with wildcards for objects.\n\n Arguments:\n\n - pattern: string containing shell-like wildcards to use in namespace\n searches and optionally a type specification to narrow the search to\n objects of that type.\n\n - ns_table: dict of name->namespaces for search.\n\n Optional arguments:\n\n - ns_search: list of namespace names to include in search.\n\n - ignore_case(False): make the search case-insensitive.\n\n - show_all(False): show all names, including those starting with\n underscores.\n \"\"\"\n #print 'ps pattern:<%r>' % pattern # dbg\n\n # defaults\n type_pattern = 'all'\n filter = ''\n\n cmds = pattern.split()\n len_cmds = len(cmds)\n if len_cmds == 1:\n # Only filter pattern given\n filter = cmds[0]\n elif len_cmds == 2:\n # Both filter and type specified\n filter,type_pattern = cmds\n else:\n raise ValueError('invalid argument string for psearch: <%s>' %\n pattern)\n\n # filter search namespaces\n for name in ns_search:\n if name not in ns_table:\n raise ValueError('invalid namespace <%s>. Valid names: %s' %\n (name,ns_table.keys()))\n\n #print 'type_pattern:',type_pattern # dbg\n search_result, namespaces_seen = set(), set()\n for ns_name in ns_search:\n ns = ns_table[ns_name]\n # Normally, locals and globals are the same, so we just check one.\n if id(ns) in namespaces_seen:\n continue\n namespaces_seen.add(id(ns))\n tmp_res = list_namespace(ns, type_pattern, filter,\n ignore_case=ignore_case, show_all=show_all)\n search_result.update(tmp_res)\n\n page.page('\\n'.join(sorted(search_result)))", "response": "Search for objects of type pattern in namespace with wildcards."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef threaded_reactor():\n global _twisted_thread\n try:\n from twisted.internet import reactor\n except ImportError:\n return None, None\n if not _twisted_thread:\n from twisted.python import threadable\n from threading import Thread\n _twisted_thread = Thread(target=lambda: reactor.run( \\\n installSignalHandlers=False))\n _twisted_thread.setDaemon(True)\n _twisted_thread.start()\n return reactor, _twisted_thread", "response": "Start the Twisted reactor in a separate thread if not already done."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nstops the reactor and join the reactor thread until it stops.", "response": "def stop_reactor():\n \"\"\"Stop the reactor and join the reactor thread until it stops.\n Call this function in teardown at the module or package level to\n reset the twisted system after your tests. You *must* do this if\n you mix tests using these tools and tests using twisted.trial.\n \"\"\"\n global _twisted_thread\n\n def stop_reactor():\n '''Helper for calling stop from withing the thread.'''\n reactor.stop()\n\n reactor.callFromThread(stop_reactor)\n reactor_thread.join()\n for p in reactor.getDelayedCalls():\n if p.active():\n p.cancel()\n _twisted_thread = None"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfinding the best matching string in a corpus.", "response": "def find_best_string(query,\n corpus,\n step=4,\n flex=3,\n case_sensitive=False):\n \"\"\"Return best matching substring of corpus.\n\n Parameters\n ----------\n query : str\n corpus : str\n step : int\n Step size of first match-value scan through corpus. Can be thought of\n as a sort of \"scan resolution\". Should not exceed length of query.\n flex : int\n Max. left/right substring position adjustment value. Should not\n exceed length of query / 2.\n\n Outputs\n -------\n output0 : str\n Best matching substring.\n output1 : float\n Match ratio of best matching substring. 1 is perfect match.\n\n \"\"\"\n\n def ratio(a, b):\n \"\"\"Compact alias for SequenceMatcher.\"\"\"\n return SequenceMatcher(None, a, b).ratio()\n\n def scan_corpus(step):\n \"\"\"Return list of match values from corpus-wide scan.\"\"\"\n match_values = []\n m = 0\n while m + qlen - step <= len(corpus):\n match_values.append(ratio(query, corpus[m : m-1+qlen]))\n m += step\n return match_values\n\n def index_max(v):\n \"\"\"Return index of max value.\"\"\"\n return max(range(len(v)), key=v.__getitem__)\n\n def adjust_left_right_positions():\n \"\"\"Return left/right positions for best string match.\"\"\"\n # bp_* is synonym for 'Best Position Left/Right' and are adjusted\n # to optimize bmv_*\n p_l, bp_l = [pos] * 2\n p_r, bp_r = [pos + qlen] * 2\n\n # bmv_* are declared here in case they are untouched in optimization\n bmv_l = match_values[round_decimal(p_l / step)]\n bmv_r = match_values[round_decimal(p_r / step)]\n\n for f in range(flex):\n ll = ratio(query, corpus[p_l - f: p_r])\n if ll > bmv_l:\n bmv_l = ll\n bp_l = p_l - f\n\n lr = ratio(query, corpus[p_l + f: p_r])\n if lr > bmv_l:\n bmv_l = lr\n bp_l = p_l + f\n\n rl = ratio(query, corpus[p_l: p_r - f])\n if rl > bmv_r:\n bmv_r = rl\n bp_r = p_r - f\n\n rr = ratio(query, corpus[p_l: p_r + f])\n if rr > bmv_r:\n bmv_r = rr\n bp_r = p_r + f\n\n return bp_l, bp_r, ratio(query, corpus[bp_l : bp_r])\n\n if not case_sensitive:\n query = query.lower()\n corpus = corpus.lower()\n\n qlen = len(query)\n\n if flex >= qlen/2:\n print(\"Warning: flex exceeds length of query / 2. Setting to default.\")\n flex = 3\n\n match_values = scan_corpus(step)\n pos = index_max(match_values) * step\n\n pos_left, pos_right, match_value = adjust_left_right_positions()\n\n return corpus[pos_left: pos_right].strip(), match_value"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _singleton_method(name):\n # Disable pylint msg W0612, because a bunch of variables look unused, but\n # they're accessed via locals().\n # pylint: disable=W0612\n\n def wrapper(*args, **kwargs):\n \"\"\"Singleton wrapper around a coverage method.\"\"\"\n global _the_coverage\n if not _the_coverage:\n _the_coverage = coverage(auto_data=True)\n return getattr(_the_coverage, name)(*args, **kwargs)\n\n import inspect\n meth = getattr(coverage, name)\n args, varargs, kw, defaults = inspect.getargspec(meth)\n argspec = inspect.formatargspec(args[1:], varargs, kw, defaults)\n docstring = meth.__doc__\n wrapper.__doc__ = (\"\"\"\\\n A first-use-singleton wrapper around coverage.%(name)s.\n\n This wrapper is provided for backward compatibility with legacy code.\n New code should use coverage.%(name)s directly.\n\n %(name)s%(argspec)s:\n\n %(docstring)s\n \"\"\" % locals()\n )\n\n return wrapper", "response": "Return a function to the name method on a singleton coverage object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef to_string(self, indent=True, declaration=True):\n return etree.tostring(self.to_xml(),\n encoding=self.encoding,\n xml_declaration=declaration,\n pretty_print=indent\n )", "response": "Encodes the stored data to XML and returns a string."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef to_xml(self):\n if self.data:\n self.document = self._update_document(self.document, self.data)\n\n return self.document", "response": "Encodes the stored data to XML and returns\n an lxml. etree. _Element"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_all_modules_in_packages(package_or_set_of_packages):\n if isinstance(package_or_set_of_packages, types.ModuleType):\n packages = [package_or_set_of_packages]\n elif isinstance(package_or_set_of_packages, Iterable) and not isinstance(package_or_set_of_packages, (dict, str)):\n packages = package_or_set_of_packages\n else:\n raise Exception(\"This function only accepts a module reference, or an iterable of said objects\")\n\n imported = packages.copy()\n\n for package in packages:\n if not hasattr(package, '__path__'):\n raise Exception(\n 'Package object passed in has no __path__ attribute. '\n 'Make sure to pass in imported references to the packages in question.'\n )\n\n for module_finder, name, ispkg in pkgutil.walk_packages(package.__path__):\n module_name = '{}.{}'.format(package.__name__, name)\n current_module = importlib.import_module(module_name)\n imported.append(current_module)\n if ispkg:\n imported += load_all_modules_in_packages(current_module)\n\n for module in imported:\n # This is to cover cases where simply importing a module doesn't execute all the code/definitions within\n # I don't totally understand the reasons for this, but I do know enumerating a module's context (like with dir)\n # seems to solve things\n dir(module)\n\n return list(\n {\n module.__name__: module\n\n for module in imported\n }.values()\n )", "response": "Recursively loads all modules in a set of packages and returns a list of all unique modules contained within the set of packages."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __dict_invert(self, data):\n outdict = {}\n for k,lst in data.items():\n if isinstance(lst, str):\n lst = lst.split()\n for entry in lst:\n outdict[entry] = k\n return outdict", "response": "Helper function for merge.\n Takes a dictionary whose values are lists and returns a dict with the elements of each list as keys and the original keys as values."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef merge(self, __loc_data__=None, __conflict_solve=None, **kw):\n\n data_dict = dict(__loc_data__,**kw)\n\n # policies for conflict resolution: two argument functions which return\n # the value that will go in the new struct\n preserve = lambda old,new: old\n update = lambda old,new: new\n add = lambda old,new: old + new\n add_flip = lambda old,new: new + old # note change of order!\n add_s = lambda old,new: old + ' ' + new\n\n # default policy is to keep current keys when there's a conflict\n conflict_solve = list2dict2(self.keys(), default = preserve)\n\n # the conflict_solve dictionary is given by the user 'inverted': we\n # need a name-function mapping, it comes as a function -> names\n # dict. Make a local copy (b/c we'll make changes), replace user\n # strings for the three builtin policies and invert it.\n if __conflict_solve:\n inv_conflict_solve_user = __conflict_solve.copy()\n for name, func in [('preserve',preserve), ('update',update),\n ('add',add), ('add_flip',add_flip),\n ('add_s',add_s)]:\n if name in inv_conflict_solve_user.keys():\n inv_conflict_solve_user[func] = inv_conflict_solve_user[name]\n del inv_conflict_solve_user[name]\n conflict_solve.update(self.__dict_invert(inv_conflict_solve_user))\n for key in data_dict:\n if key not in self:\n self[key] = data_dict[key]\n else:\n self[key] = conflict_solve[key](self[key],data_dict[key])", "response": "This method merges two Structs with customizable conflict resolution."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef fullvars(obj):\n '''\n like `vars()` but support `__slots__`.\n '''\n\n try:\n return vars(obj)\n except TypeError:\n pass\n\n # __slots__\n slotsnames = set()\n for cls in type(obj).__mro__:\n __slots__ = getattr(cls, '__slots__', None)\n if __slots__:\n if isinstance(__slots__, str):\n slotsnames.add(__slots__)\n else:\n slotsnames.update(__slots__)\n return _SlotsProxy(obj, slotsnames)", "response": "Like vars but support __slots__."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef object_to_primitive(obj):\n '''\n convert object to primitive type so we can serialize it to data format like python.\n\n all primitive types: dict, list, int, float, bool, str, None\n '''\n if obj is None:\n return obj\n\n if isinstance(obj, (int, float, bool, str)):\n return obj\n\n if isinstance(obj, (list, frozenset, set)):\n return [object_to_primitive(x) for x in obj]\n\n if isinstance(obj, dict):\n return dict([(object_to_primitive(k), object_to_primitive(v)) for k, v in obj.items()])\n\n data = vars(obj)\n assert isinstance(data, dict)\n return object_to_primitive(data)", "response": "converts object to primitive type so we can serialize it to data format like python."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef main(argv=None):\n\n usage_msg = \"\"\"%prog [options] [filename]\n\nColorize a python file or stdin using ANSI color escapes and print to stdout.\nIf no filename is given, or if filename is -, read standard input.\"\"\"\n\n parser = optparse.OptionParser(usage=usage_msg)\n newopt = parser.add_option\n newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',\n choices=['Linux','LightBG','NoColor'],default=_scheme_default,\n help=\"give the color scheme to use. Currently only 'Linux'\\\n (default) and 'LightBG' and 'NoColor' are implemented (give without\\\n quotes)\")\n\n opts,args = parser.parse_args(argv)\n\n if len(args) > 1:\n parser.error(\"you must give at most one filename.\")\n\n if len(args) == 0:\n fname = '-' # no filename given; setup to read from stdin\n else:\n fname = args[0]\n\n if fname == '-':\n stream = sys.stdin\n else:\n try:\n stream = open(fname)\n except IOError,msg:\n print >> sys.stderr, msg\n sys.exit(1)\n\n parser = Parser()\n\n # we need nested try blocks because pre-2.5 python doesn't support unified\n # try-except-finally\n try:\n try:\n # write colorized version to stdout\n parser.format(stream.read(),scheme=opts.scheme_name)\n except IOError,msg:\n # if user reads through a pager and quits, don't print traceback\n if msg.args != (32,'Broken pipe'):\n raise\n finally:\n if stream is not sys.stdin:\n stream.close()", "response": "This function is a command - line script that will parse the command - line arguments and then parse the output."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse and send the colored source.", "response": "def format2(self, raw, out = None, scheme = ''):\n \"\"\" Parse and send the colored source.\n\n If out and scheme are not specified, the defaults (given to\n constructor) are used.\n\n out should be a file-type object. Optionally, out can be given as the\n string 'str' and the parser will automatically return the output in a\n string.\"\"\"\n\n string_output = 0\n if out == 'str' or self.out == 'str' or \\\n isinstance(self.out,StringIO.StringIO):\n # XXX - I don't really like this state handling logic, but at this\n # point I don't want to make major changes, so adding the\n # isinstance() check is the simplest I can do to ensure correct\n # behavior.\n out_old = self.out\n self.out = StringIO.StringIO()\n string_output = 1\n elif out is not None:\n self.out = out\n\n # Fast return of the unmodified input for NoColor scheme\n if scheme == 'NoColor':\n error = False\n self.out.write(raw)\n if string_output:\n return raw,error\n else:\n return None,error\n\n # local shorthands\n colors = self.color_table[scheme].colors\n self.colors = colors # put in object so __call__ sees it\n\n # Remove trailing whitespace and normalize tabs\n self.raw = raw.expandtabs().rstrip()\n\n # store line offsets in self.lines\n self.lines = [0, 0]\n pos = 0\n raw_find = self.raw.find\n lines_append = self.lines.append\n while 1:\n pos = raw_find('\\n', pos) + 1\n if not pos: break\n lines_append(pos)\n lines_append(len(self.raw))\n\n # parse the source and write it\n self.pos = 0\n text = StringIO.StringIO(self.raw)\n\n error = False\n try:\n for atoken in generate_tokens(text.readline):\n self(*atoken)\n except tokenize.TokenError as ex:\n msg = ex.args[0]\n line = ex.args[1][0]\n self.out.write(\"%s\\n\\n*** ERROR: %s%s%s\\n\" %\n (colors[token.ERRORTOKEN],\n msg, self.raw[self.lines[line]:],\n colors.normal)\n )\n error = True\n self.out.write(colors.normal+'\\n')\n if string_output:\n output = self.out.getvalue()\n self.out = out_old\n return (output, error)\n return (None, error)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getfigs(*fig_nums):\n from matplotlib._pylab_helpers import Gcf\n if not fig_nums:\n fig_managers = Gcf.get_all_fig_managers()\n return [fm.canvas.figure for fm in fig_managers]\n else:\n figs = []\n for num in fig_nums:\n f = Gcf.figs.get(num)\n if f is None:\n print('Warning: figure %s not available.' % num)\n else:\n figs.append(f.canvas.figure)\n return figs", "response": "Get a list of matplotlib figures by figure numbers."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef print_figure(fig, fmt='png'):\n # When there's an empty figure, we shouldn't return anything, otherwise we\n # get big blank areas in the qt console.\n if not fig.axes and not fig.lines:\n return\n\n fc = fig.get_facecolor()\n ec = fig.get_edgecolor()\n fig.set_facecolor('white')\n fig.set_edgecolor('white')\n try:\n bytes_io = BytesIO()\n fig.canvas.print_figure(bytes_io, format=fmt, bbox_inches='tight')\n data = bytes_io.getvalue()\n finally:\n fig.set_facecolor(fc)\n fig.set_edgecolor(ec)\n return data", "response": "Convert a figure to svg or png for inline display."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a function suitable for use as a matplotlib - enabled runner for the magic %run.", "response": "def mpl_runner(safe_execfile):\n \"\"\"Factory to return a matplotlib-enabled runner for %run.\n\n Parameters\n ----------\n safe_execfile : function\n This must be a function with the same interface as the\n :meth:`safe_execfile` method of IPython.\n\n Returns\n -------\n A function suitable for use as the ``runner`` argument of the %run magic\n function.\n \"\"\"\n \n def mpl_execfile(fname,*where,**kw):\n \"\"\"matplotlib-aware wrapper around safe_execfile.\n\n Its interface is identical to that of the :func:`execfile` builtin.\n\n This is ultimately a call to execfile(), but wrapped in safeties to\n properly handle interactive rendering.\"\"\"\n\n import matplotlib\n import matplotlib.pylab as pylab\n\n #print '*** Matplotlib runner ***' # dbg\n # turn off rendering until end of script\n is_interactive = matplotlib.rcParams['interactive']\n matplotlib.interactive(False)\n safe_execfile(fname,*where,**kw)\n matplotlib.interactive(is_interactive)\n # make rendering call now, if the user tried to do it\n if pylab.draw_if_interactive.called:\n pylab.draw()\n pylab.draw_if_interactive.called = False\n\n return mpl_execfile"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef select_figure_format(shell, fmt):\n from matplotlib.figure import Figure\n from IPython.zmq.pylab import backend_inline\n\n svg_formatter = shell.display_formatter.formatters['image/svg+xml']\n png_formatter = shell.display_formatter.formatters['image/png']\n\n if fmt=='png':\n svg_formatter.type_printers.pop(Figure, None)\n png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png'))\n elif fmt=='svg':\n png_formatter.type_printers.pop(Figure, None)\n svg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'svg'))\n else:\n raise ValueError(\"supported formats are: 'png', 'svg', not %r\"%fmt)\n\n # set the format to be used in the backend()\n backend_inline._figure_format = fmt", "response": "Select figure format for inline backend either png or svg."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngive a gui string return the gui and mpl backend.", "response": "def find_gui_and_backend(gui=None):\n \"\"\"Given a gui string return the gui and mpl backend.\n\n Parameters\n ----------\n gui : str\n Can be one of ('tk','gtk','wx','qt','qt4','inline').\n\n Returns\n -------\n A tuple of (gui, backend) where backend is one of ('TkAgg','GTKAgg',\n 'WXAgg','Qt4Agg','module://IPython.zmq.pylab.backend_inline').\n \"\"\"\n\n import matplotlib\n\n if gui and gui != 'auto':\n # select backend based on requested gui\n backend = backends[gui]\n else:\n backend = matplotlib.rcParams['backend']\n # In this case, we need to find what the appropriate gui selection call\n # should be for IPython, so we can activate inputhook accordingly\n gui = backend2gui.get(backend, None)\n return gui, backend"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nactivating the given backend and set interactive to True.", "response": "def activate_matplotlib(backend):\n \"\"\"Activate the given backend and set interactive to True.\"\"\"\n\n import matplotlib\n if backend.startswith('module://'):\n # Work around bug in matplotlib: matplotlib.use converts the\n # backend_id to lowercase even if a module name is specified!\n matplotlib.rcParams['backend'] = backend\n else:\n matplotlib.use(backend)\n matplotlib.interactive(True)\n\n # This must be imported last in the matplotlib series, after\n # backend/interactivity choices have been made\n import matplotlib.pylab as pylab\n\n # XXX For now leave this commented out, but depending on discussions with\n # mpl-dev, we may be able to allow interactive switching...\n #import matplotlib.pyplot\n #matplotlib.pyplot.switch_backend(backend)\n\n pylab.show._needmain = False\n # We need to detect at runtime whether show() is called by the user.\n # For this, we wrap it into a decorator which adds a 'called' flag.\n pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef import_pylab(user_ns, import_all=True):\n\n # Import numpy as np/pyplot as plt are conventions we're trying to\n # somewhat standardize on. Making them available to users by default\n # will greatly help this.\n s = (\"import numpy\\n\"\n \"import matplotlib\\n\"\n \"from matplotlib import pylab, mlab, pyplot\\n\"\n \"np = numpy\\n\"\n \"plt = pyplot\\n\"\n )\n exec s in user_ns\n\n if import_all:\n s = (\"from matplotlib.pylab import *\\n\"\n \"from numpy import *\\n\")\n exec s in user_ns", "response": "Import the standard pylab symbols into user_ns."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconfiguring an IPython shell object for matplotlib use.", "response": "def configure_inline_support(shell, backend, user_ns=None):\n \"\"\"Configure an IPython shell object for matplotlib use.\n\n Parameters\n ----------\n shell : InteractiveShell instance\n\n backend : matplotlib backend\n\n user_ns : dict\n A namespace where all configured variables will be placed. If not given,\n the `user_ns` attribute of the shell object is used.\n \"\"\"\n # If using our svg payload backend, register the post-execution\n # function that will pick up the results for display. This can only be\n # done with access to the real shell object.\n\n # Note: if we can't load the inline backend, then there's no point\n # continuing (such as in terminal-only shells in environments without\n # zeromq available).\n try:\n from IPython.zmq.pylab.backend_inline import InlineBackend\n except ImportError:\n return\n\n user_ns = shell.user_ns if user_ns is None else user_ns\n \n cfg = InlineBackend.instance(config=shell.config)\n cfg.shell = shell\n if cfg not in shell.configurables:\n shell.configurables.append(cfg)\n\n if backend == backends['inline']:\n from IPython.zmq.pylab.backend_inline import flush_figures\n from matplotlib import pyplot\n shell.register_post_execute(flush_figures)\n # load inline_rc\n pyplot.rcParams.update(cfg.rc)\n # Add 'figsize' to pyplot and to the user's namespace\n user_ns['figsize'] = pyplot.figsize = figsize\n\n # Setup the default figure format\n fmt = cfg.figure_format\n select_figure_format(shell, fmt)\n\n # The old pastefig function has been replaced by display\n from IPython.core.display import display\n # Add display and getfigs to the user's namespace\n user_ns['display'] = display\n user_ns['getfigs'] = getfigs"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nactivate pylab mode in the user s namespace.", "response": "def pylab_activate(user_ns, gui=None, import_all=True, shell=None):\n \"\"\"Activate pylab mode in the user's namespace.\n\n Loads and initializes numpy, matplotlib and friends for interactive use.\n\n Parameters\n ----------\n user_ns : dict\n Namespace where the imports will occur.\n\n gui : optional, string\n A valid gui name following the conventions of the %gui magic.\n\n import_all : optional, boolean\n If true, an 'import *' is done from numpy and pylab.\n\n Returns\n -------\n The actual gui used (if not given as input, it was obtained from matplotlib\n itself, and will be needed next to configure IPython's gui integration.\n \"\"\"\n gui, backend = find_gui_and_backend(gui)\n activate_matplotlib(backend)\n import_pylab(user_ns, import_all)\n if shell is not None:\n configure_inline_support(shell, backend, user_ns)\n \n print \"\"\"\nWelcome to pylab, a matplotlib-based Python environment [backend: %s].\nFor more information, type 'help(pylab)'.\"\"\" % backend\n # flush stdout, just to be safe\n sys.stdout.flush()\n \n return gui"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _trace(self, frame, event, arg_unused):\n\n if self.stopped:\n return\n\n if 0:\n sys.stderr.write(\"trace event: %s %r @%d\\n\" % (\n event, frame.f_code.co_filename, frame.f_lineno\n ))\n\n if self.last_exc_back:\n if frame == self.last_exc_back:\n # Someone forgot a return event.\n if self.arcs and self.cur_file_data:\n pair = (self.last_line, -self.last_exc_firstlineno)\n self.cur_file_data[pair] = None\n self.cur_file_data, self.last_line = self.data_stack.pop()\n self.last_exc_back = None\n\n if event == 'call':\n # Entering a new function context. Decide if we should trace\n # in this file.\n self.data_stack.append((self.cur_file_data, self.last_line))\n filename = frame.f_code.co_filename\n if filename not in self.should_trace_cache:\n tracename = self.should_trace(filename, frame)\n self.should_trace_cache[filename] = tracename\n else:\n tracename = self.should_trace_cache[filename]\n #print(\"called, stack is %d deep, tracename is %r\" % (\n # len(self.data_stack), tracename))\n if tracename:\n if tracename not in self.data:\n self.data[tracename] = {}\n self.cur_file_data = self.data[tracename]\n else:\n self.cur_file_data = None\n # Set the last_line to -1 because the next arc will be entering a\n # code block, indicated by (-1, n).\n self.last_line = -1\n elif event == 'line':\n # Record an executed line.\n if self.cur_file_data is not None:\n if self.arcs:\n #print(\"lin\", self.last_line, frame.f_lineno)\n self.cur_file_data[(self.last_line, frame.f_lineno)] = None\n else:\n #print(\"lin\", frame.f_lineno)\n self.cur_file_data[frame.f_lineno] = None\n self.last_line = frame.f_lineno\n elif event == 'return':\n if self.arcs and self.cur_file_data:\n first = frame.f_code.co_firstlineno\n self.cur_file_data[(self.last_line, -first)] = None\n # Leaving this function, pop the filename stack.\n self.cur_file_data, self.last_line = self.data_stack.pop()\n #print(\"returned, stack is %d deep\" % (len(self.data_stack)))\n elif event == 'exception':\n #print(\"exc\", self.last_line, frame.f_lineno)\n self.last_exc_back = frame.f_back\n self.last_exc_firstlineno = frame.f_code.co_firstlineno\n return self._trace", "response": "Trace the current line of the code block."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstarting this Tracer. Return a Python function suitable for use with sys. settrace.", "response": "def start(self):\n \"\"\"Start this Tracer.\n\n Return a Python function suitable for use with sys.settrace().\n\n \"\"\"\n self.thread = threading.currentThread()\n sys.settrace(self._trace)\n return self._trace"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _start_tracer(self):\n tracer = self._trace_class()\n tracer.data = self.data\n tracer.arcs = self.branch\n tracer.should_trace = self.should_trace\n tracer.should_trace_cache = self.should_trace_cache\n tracer.warn = self.warn\n fn = tracer.start()\n self.tracers.append(tracer)\n return fn", "response": "Start a new Tracer object and store it in self. tracers."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalls on new threads installs the real tracer.", "response": "def _installation_trace(self, frame_unused, event_unused, arg_unused):\n \"\"\"Called on new threads, installs the real tracer.\"\"\"\n # Remove ourselves as the trace function\n sys.settrace(None)\n # Install the real tracer.\n fn = self._start_tracer()\n # Invoke the real trace function with the current event, to be sure\n # not to lose an event.\n if fn:\n fn = fn(frame_unused, event_unused, arg_unused)\n # Return the new trace function to continue tracing in this scope.\n return fn"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef start(self):\n if self._collectors:\n self._collectors[-1].pause()\n self._collectors.append(self)\n #print(\"Started: %r\" % self._collectors, file=sys.stderr)\n\n # Check to see whether we had a fullcoverage tracer installed.\n traces0 = []\n if hasattr(sys, \"gettrace\"):\n fn0 = sys.gettrace()\n if fn0:\n tracer0 = getattr(fn0, '__self__', None)\n if tracer0:\n traces0 = getattr(tracer0, 'traces', [])\n\n # Install the tracer on this thread.\n fn = self._start_tracer()\n\n for args in traces0:\n (frame, event, arg), lineno = args\n try:\n fn(frame, event, arg, lineno=lineno)\n except TypeError:\n raise Exception(\n \"fullcoverage must be run with the C trace function.\"\n )\n\n # Install our installation tracer in threading, to jump start other\n # threads.\n threading.settrace(self._installation_trace)", "response": "Start collecting trace information."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef stop(self):\n #print >>sys.stderr, \"Stopping: %r\" % self._collectors\n assert self._collectors\n assert self._collectors[-1] is self\n\n self.pause()\n self.tracers = []\n\n # Remove this Collector from the stack, and resume the one underneath\n # (if any).\n self._collectors.pop()\n if self._collectors:\n self._collectors[-1].resume()", "response": "Stop collecting trace information."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pause(self):\n for tracer in self.tracers:\n tracer.stop()\n stats = tracer.get_stats()\n if stats:\n print(\"\\nCoverage.py tracer stats:\")\n for k in sorted(stats.keys()):\n print(\"%16s: %s\" % (k, stats[k]))\n threading.settrace(None)", "response": "Pause tracing but be prepared to resume."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef resume(self):\n for tracer in self.tracers:\n tracer.start()\n threading.settrace(self._installation_trace)", "response": "Resume tracing after a pause."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_line_data(self):\n if self.branch:\n # If we were measuring branches, then we have to re-build the dict\n # to show line data.\n line_data = {}\n for f, arcs in self.data.items():\n line_data[f] = ldf = {}\n for l1, _ in list(arcs.keys()):\n if l1:\n ldf[l1] = None\n return line_data\n else:\n return self.data", "response": "Return the line data collected."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a new code cell with input and output", "response": "def new_code_cell(code=None, prompt_number=None):\n \"\"\"Create a new code cell with input and output\"\"\"\n cell = NotebookNode()\n cell.cell_type = u'code'\n if code is not None:\n cell.code = unicode(code)\n if prompt_number is not None:\n cell.prompt_number = int(prompt_number)\n return cell"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef new_text_cell(text=None):\n cell = NotebookNode()\n if text is not None:\n cell.text = unicode(text)\n cell.cell_type = u'text'\n return cell", "response": "Create a new text cell."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a notebook by name id and a list of worksheets.", "response": "def new_notebook(cells=None):\n \"\"\"Create a notebook by name, id and a list of worksheets.\"\"\"\n nb = NotebookNode()\n if cells is not None:\n nb.cells = cells\n else:\n nb.cells = []\n return nb"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef eq_(a, b, msg=None):\n if not a == b:\n raise AssertionError(msg or \"%r != %r\" % (a, b))", "response": "Assert that a and b are equal."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef collect_exceptions(rdict_or_list, method='unspecified'):\n elist = []\n if isinstance(rdict_or_list, dict):\n rlist = rdict_or_list.values()\n else:\n rlist = rdict_or_list\n for r in rlist:\n if isinstance(r, RemoteError):\n en, ev, etb, ei = r.ename, r.evalue, r.traceback, r.engine_info\n # Sometimes we could have CompositeError in our list. Just take\n # the errors out of them and put them in our new list. This\n # has the effect of flattening lists of CompositeErrors into one\n # CompositeError\n if en=='CompositeError':\n for e in ev.elist:\n elist.append(e)\n else:\n elist.append((en, ev, etb, ei))\n if len(elist)==0:\n return rdict_or_list\n else:\n msg = \"one or more exceptions from call to method: %s\" % (method)\n # This silliness is needed so the debugger has access to the exception\n # instance (e in this case)\n try:\n raise CompositeError(msg, elist)\n except CompositeError as e:\n raise e", "response": "check a result dict for errors and raise CompositeError if any exist."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrender one or all of my tracebacks to a list of lines", "response": "def render_traceback(self, excid=None):\n \"\"\"render one or all of my tracebacks to a list of lines\"\"\"\n lines = []\n if excid is None:\n for (en,ev,etb,ei) in self.elist:\n lines.append(self._get_engine_str(ei))\n lines.extend((etb or 'No traceback available').splitlines())\n lines.append('')\n else:\n try:\n en,ev,etb,ei = self.elist[excid]\n except:\n raise IndexError(\"an exception with index %i does not exist\"%excid)\n else:\n lines.append(self._get_engine_str(ei))\n lines.extend((etb or 'No traceback available').splitlines())\n \n return lines"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalling this at Python startup to perhaps measure coverage.", "response": "def process_startup():\n \"\"\"Call this at Python startup to perhaps measure coverage.\n\n If the environment variable COVERAGE_PROCESS_START is defined, coverage\n measurement is started. The value of the variable is the config file\n to use.\n\n There are two ways to configure your Python installation to invoke this\n function when Python starts:\n\n #. Create or append to sitecustomize.py to add these lines::\n\n import coverage\n coverage.process_startup()\n\n #. Create a .pth file in your Python installation containing::\n\n import coverage; coverage.process_startup()\n\n \"\"\"\n cps = os.environ.get(\"COVERAGE_PROCESS_START\")\n if cps:\n cov = coverage(config_file=cps, auto_data=True)\n cov.start()\n cov._warn_no_data = False\n cov._warn_unimported_source = False"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _canonical_dir(self, morf):\n return os.path.split(CodeUnit(morf, self.file_locator).filename)[0]", "response": "Return the canonical directory of the module or file morf."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _source_for_file(self, filename):\n if not filename.endswith(\".py\"):\n if filename[-4:-1] == \".py\":\n filename = filename[:-1]\n elif filename.endswith(\"$py.class\"): # jython\n filename = filename[:-9] + \".py\"\n return filename", "response": "Return the source file for filename."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndeciding whether the execution of filename should be traced or not.", "response": "def _should_trace_with_reason(self, filename, frame):\n \"\"\"Decide whether to trace execution in `filename`, with a reason.\n\n This function is called from the trace function. As each new file name\n is encountered, this function determines whether it is traced or not.\n\n Returns a pair of values: the first indicates whether the file should\n be traced: it's a canonicalized filename if it should be traced, None\n if it should not. The second value is a string, the resason for the\n decision.\n\n \"\"\"\n if not filename:\n # Empty string is pretty useless\n return None, \"empty string isn't a filename\"\n\n if filename.startswith('<'):\n # Lots of non-file execution is represented with artificial\n # filenames like \"\", \"\", or\n # \"\". Don't ever trace these executions, since we\n # can't do anything with the data later anyway.\n return None, \"not a real filename\"\n\n self._check_for_packages()\n\n # Compiled Python files have two filenames: frame.f_code.co_filename is\n # the filename at the time the .pyc was compiled. The second name is\n # __file__, which is where the .pyc was actually loaded from. Since\n # .pyc files can be moved after compilation (for example, by being\n # installed), we look for __file__ in the frame and prefer it to the\n # co_filename value.\n dunder_file = frame.f_globals.get('__file__')\n if dunder_file:\n filename = self._source_for_file(dunder_file)\n\n # Jython reports the .class file to the tracer, use the source file.\n if filename.endswith(\"$py.class\"):\n filename = filename[:-9] + \".py\"\n\n canonical = self.file_locator.canonical_filename(filename)\n\n # If the user specified source or include, then that's authoritative\n # about the outer bound of what to measure and we don't have to apply\n # any canned exclusions. If they didn't, then we have to exclude the\n # stdlib and coverage.py directories.\n if self.source_match:\n if not self.source_match.match(canonical):\n return None, \"falls outside the --source trees\"\n elif self.include_match:\n if not self.include_match.match(canonical):\n return None, \"falls outside the --include trees\"\n else:\n # If we aren't supposed to trace installed code, then check if this\n # is near the Python standard library and skip it if so.\n if self.pylib_match and self.pylib_match.match(canonical):\n return None, \"is in the stdlib\"\n\n # We exclude the coverage code itself, since a little of it will be\n # measured otherwise.\n if self.cover_match and self.cover_match.match(canonical):\n return None, \"is part of coverage.py\"\n\n # Check the file against the omit pattern.\n if self.omit_match and self.omit_match.match(canonical):\n return None, \"is inside an --omit pattern\"\n\n return canonical, \"because we love you\""} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _should_trace(self, filename, frame):\n canonical, reason = self._should_trace_with_reason(filename, frame)\n if self.debug.should('trace'):\n if not canonical:\n msg = \"Not tracing %r: %s\" % (filename, reason)\n else:\n msg = \"Tracing %r\" % (filename,)\n self.debug.write(msg)\n return canonical", "response": "Decide whether to trace execution in filename."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _warn(self, msg):\n self._warnings.append(msg)\n sys.stderr.write(\"Coverage.py warning: %s\\n\" % msg)", "response": "Use msg as a warning."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _check_for_packages(self):\n # Our self.source_pkgs attribute is a list of package names we want to\n # measure. Each time through here, we see if we've imported any of\n # them yet. If so, we add its file to source_match, and we don't have\n # to look for that package any more.\n if self.source_pkgs:\n found = []\n for pkg in self.source_pkgs:\n try:\n mod = sys.modules[pkg]\n except KeyError:\n continue\n\n found.append(pkg)\n\n try:\n pkg_file = mod.__file__\n except AttributeError:\n pkg_file = None\n else:\n d, f = os.path.split(pkg_file)\n if f.startswith('__init__'):\n # This is actually a package, return the directory.\n pkg_file = d\n else:\n pkg_file = self._source_for_file(pkg_file)\n pkg_file = self.file_locator.canonical_filename(pkg_file)\n if not os.path.exists(pkg_file):\n pkg_file = None\n\n if pkg_file:\n self.source.append(pkg_file)\n self.source_match.add(pkg_file)\n else:\n self._warn(\"Module %s has no Python source.\" % pkg)\n\n for pkg in found:\n self.source_pkgs.remove(pkg)", "response": "Update the source_match matcher with latest imported packages."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nstart measuring code coverage.", "response": "def start(self):\n \"\"\"Start measuring code coverage.\n\n Coverage measurement actually occurs in functions called after `start`\n is invoked. Statements in the same scope as `start` won't be measured.\n\n Once you invoke `start`, you must also call `stop` eventually, or your\n process might not shut down cleanly.\n\n \"\"\"\n if self.run_suffix:\n # Calling start() means we're running code, so use the run_suffix\n # as the data_suffix when we eventually save the data.\n self.data_suffix = self.run_suffix\n if self.auto_data:\n self.load()\n\n # Create the matchers we need for _should_trace\n if self.source or self.source_pkgs:\n self.source_match = TreeMatcher(self.source)\n else:\n if self.cover_dir:\n self.cover_match = TreeMatcher([self.cover_dir])\n if self.pylib_dirs:\n self.pylib_match = TreeMatcher(self.pylib_dirs)\n if self.include:\n self.include_match = FnmatchMatcher(self.include)\n if self.omit:\n self.omit_match = FnmatchMatcher(self.omit)\n\n # The user may want to debug things, show info if desired.\n if self.debug.should('config'):\n self.debug.write(\"Configuration values:\")\n config_info = sorted(self.config.__dict__.items())\n self.debug.write_formatted_info(config_info)\n\n if self.debug.should('sys'):\n self.debug.write(\"Debugging info:\")\n self.debug.write_formatted_info(self.sysinfo())\n\n self.collector.start()\n self._started = True\n self._measured = True"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _atexit(self):\n if self._started:\n self.stop()\n if self.auto_data:\n self.save()", "response": "Clean up on process shutdown."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nexclude source lines from execution consideration.", "response": "def exclude(self, regex, which='exclude'):\n \"\"\"Exclude source lines from execution consideration.\n\n A number of lists of regular expressions are maintained. Each list\n selects lines that are treated differently during reporting.\n\n `which` determines which list is modified. The \"exclude\" list selects\n lines that are not considered executable at all. The \"partial\" list\n indicates lines with branches that are not taken.\n\n `regex` is a regular expression. The regex is added to the specified\n list. If any of the regexes in the list is found in a line, the line\n is marked for special treatment during reporting.\n\n \"\"\"\n excl_list = getattr(self.config, which + \"_list\")\n excl_list.append(regex)\n self._exclude_regex_stale()"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _exclude_regex(self, which):\n if which not in self._exclude_re:\n excl_list = getattr(self.config, which + \"_list\")\n self._exclude_re[which] = join_regex(excl_list)\n return self._exclude_re[which]", "response": "Return a compiled regex for the given exclusion list."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef save(self):\n data_suffix = self.data_suffix\n if data_suffix is True:\n # If data_suffix was a simple true value, then make a suffix with\n # plenty of distinguishing information. We do this here in\n # `save()` at the last minute so that the pid will be correct even\n # if the process forks.\n extra = \"\"\n if _TEST_NAME_FILE:\n f = open(_TEST_NAME_FILE)\n test_name = f.read()\n f.close()\n extra = \".\" + test_name\n data_suffix = \"%s%s.%s.%06d\" % (\n socket.gethostname(), extra, os.getpid(),\n random.randint(0, 999999)\n )\n\n self._harvest_data()\n self.data.write(suffix=data_suffix)", "response": "Save the collected coverage data to the data file."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncombines together a number of similarly - named coverage data files.", "response": "def combine(self):\n \"\"\"Combine together a number of similarly-named coverage data files.\n\n All coverage data files whose name starts with `data_file` (from the\n coverage() constructor) will be read, and combined together into the\n current measurements.\n\n \"\"\"\n aliases = None\n if self.config.paths:\n aliases = PathAliases(self.file_locator)\n for paths in self.config.paths.values():\n result = paths[0]\n for pattern in paths[1:]:\n aliases.add(pattern, result)\n self.data.combine_parallel_data(aliases=aliases)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the collected data and reset the collector.", "response": "def _harvest_data(self):\n \"\"\"Get the collected data and reset the collector.\n\n Also warn about various problems collecting data.\n\n \"\"\"\n if not self._measured:\n return\n\n self.data.add_line_data(self.collector.get_line_data())\n self.data.add_arc_data(self.collector.get_arc_data())\n self.collector.reset()\n\n # If there are still entries in the source_pkgs list, then we never\n # encountered those packages.\n if self._warn_unimported_source:\n for pkg in self.source_pkgs:\n self._warn(\"Module %s was never imported.\" % pkg)\n\n # Find out if we got any data.\n summary = self.data.summary()\n if not summary and self._warn_no_data:\n self._warn(\"No data was collected.\")\n\n # Find files that were never executed at all.\n for src in self.source:\n for py_file in find_python_files(src):\n py_file = self.file_locator.canonical_filename(py_file)\n\n if self.omit_match and self.omit_match.match(py_file):\n # Turns out this file was omitted, so don't pull it back\n # in as unexecuted.\n continue\n\n self.data.touch_file(py_file)\n\n self._measured = False"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef analysis(self, morf):\n f, s, _, m, mf = self.analysis2(morf)\n return f, s, m, mf", "response": "Like analysis2 but doesn t return excluded line numbers."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef analysis2(self, morf):\n analysis = self._analyze(morf)\n return (\n analysis.filename,\n sorted(analysis.statements),\n sorted(analysis.excluded),\n sorted(analysis.missing),\n analysis.missing_formatted(),\n )", "response": "Analyze a module.\n ArcGIS file and return a 5 - tuple containing the filename statements excluded missing and readable formatted string of the missing line numbers."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _analyze(self, it):\n self._harvest_data()\n if not isinstance(it, CodeUnit):\n it = code_unit_factory(it, self.file_locator)[0]\n\n return Analysis(self, it)", "response": "Analyze a single morf or code unit."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef report(self, morfs=None, show_missing=True, ignore_errors=None,\n file=None, # pylint: disable=W0622\n omit=None, include=None\n ):\n \"\"\"Write a summary report to `file`.\n\n Each module in `morfs` is listed, with counts of statements, executed\n statements, missing statements, and a list of lines missed.\n\n `include` is a list of filename patterns. Modules whose filenames\n match those patterns will be included in the report. Modules matching\n `omit` will not be included in the report.\n\n Returns a float, the total percentage covered.\n\n \"\"\"\n self._harvest_data()\n self.config.from_args(\n ignore_errors=ignore_errors, omit=omit, include=include,\n show_missing=show_missing,\n )\n reporter = SummaryReporter(self, self.config)\n return reporter.report(morfs, outfile=file)", "response": "Write a summary report to file."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef annotate(self, morfs=None, directory=None, ignore_errors=None,\n omit=None, include=None):\n \"\"\"Annotate a list of modules.\n\n Each module in `morfs` is annotated. The source is written to a new\n file, named with a \",cover\" suffix, with each line prefixed with a\n marker to indicate the coverage of the line. Covered lines have \">\",\n excluded lines have \"-\", and missing lines have \"!\".\n\n See `coverage.report()` for other arguments.\n\n \"\"\"\n self._harvest_data()\n self.config.from_args(\n ignore_errors=ignore_errors, omit=omit, include=include\n )\n reporter = AnnotateReporter(self, self.config)\n reporter.report(morfs, directory=directory)", "response": "Annotate a list of modules."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef html_report(self, morfs=None, directory=None, ignore_errors=None,\n omit=None, include=None, extra_css=None, title=None):\n \"\"\"Generate an HTML report.\n\n The HTML is written to `directory`. The file \"index.html\" is the\n overview starting point, with links to more detailed pages for\n individual modules.\n\n `extra_css` is a path to a file of other CSS to apply on the page.\n It will be copied into the HTML directory.\n\n `title` is a text string (not HTML) to use as the title of the HTML\n report.\n\n See `coverage.report()` for other arguments.\n\n Returns a float, the total percentage covered.\n\n \"\"\"\n self._harvest_data()\n self.config.from_args(\n ignore_errors=ignore_errors, omit=omit, include=include,\n html_dir=directory, extra_css=extra_css, html_title=title,\n )\n reporter = HtmlReporter(self, self.config)\n return reporter.report(morfs)", "response": "Generate an HTML report."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef xml_report(self, morfs=None, outfile=None, ignore_errors=None,\n omit=None, include=None):\n \"\"\"Generate an XML report of coverage results.\n\n The report is compatible with Cobertura reports.\n\n Each module in `morfs` is included in the report. `outfile` is the\n path to write the file to, \"-\" will write to stdout.\n\n See `coverage.report()` for other arguments.\n\n Returns a float, the total percentage covered.\n\n \"\"\"\n self._harvest_data()\n self.config.from_args(\n ignore_errors=ignore_errors, omit=omit, include=include,\n xml_output=outfile,\n )\n file_to_close = None\n delete_file = False\n if self.config.xml_output:\n if self.config.xml_output == '-':\n outfile = sys.stdout\n else:\n outfile = open(self.config.xml_output, \"w\")\n file_to_close = outfile\n try:\n try:\n reporter = XmlReporter(self, self.config)\n return reporter.report(morfs, outfile=outfile)\n except CoverageException:\n delete_file = True\n raise\n finally:\n if file_to_close:\n file_to_close.close()\n if delete_file:\n file_be_gone(self.config.xml_output)", "response": "Generate an XML report of the Cobertura coverage results."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sysinfo(self):\n\n import coverage as covmod\n import platform, re\n\n try:\n implementation = platform.python_implementation()\n except AttributeError:\n implementation = \"unknown\"\n\n info = [\n ('version', covmod.__version__),\n ('coverage', covmod.__file__),\n ('cover_dir', self.cover_dir),\n ('pylib_dirs', self.pylib_dirs),\n ('tracer', self.collector.tracer_name()),\n ('config_files', self.config.attempted_config_files),\n ('configs_read', self.config.config_files),\n ('data_path', self.data.filename),\n ('python', sys.version.replace('\\n', '')),\n ('platform', platform.platform()),\n ('implementation', implementation),\n ('executable', sys.executable),\n ('cwd', os.getcwd()),\n ('path', sys.path),\n ('environment', sorted([\n (\"%s = %s\" % (k, v)) for k, v in iitems(os.environ)\n if re.search(r\"^COV|^PY\", k)\n ])),\n ('command_line', \" \".join(getattr(sys, 'argv', ['???']))),\n ]\n if self.source_match:\n info.append(('source_match', self.source_match.info()))\n if self.include_match:\n info.append(('include_match', self.include_match.info()))\n if self.omit_match:\n info.append(('omit_match', self.omit_match.info()))\n if self.cover_match:\n info.append(('cover_match', self.cover_match.info()))\n if self.pylib_match:\n info.append(('pylib_match', self.pylib_match.info()))\n\n return info", "response": "Return a list of ( key value ) pairs showing internal information."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef display(*objs, **kwargs):\n include = kwargs.get('include')\n exclude = kwargs.get('exclude')\n\n from IPython.core.interactiveshell import InteractiveShell\n inst = InteractiveShell.instance()\n format = inst.display_formatter.format\n publish = inst.display_pub.publish\n\n for obj in objs:\n format_dict = format(obj, include=include, exclude=exclude)\n publish('IPython.core.display.display', format_dict)", "response": "Display a Python object in all frontends."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef display_pretty(*objs, **kwargs):\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_pretty(obj)\n else:\n display(*objs, include=['text/plain'])", "response": "Display the pretty representation of an object."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndisplay the HTML representation of a Python object.", "response": "def display_html(*objs, **kwargs):\n \"\"\"Display the HTML representation of an object.\n\n Parameters\n ----------\n objs : tuple of objects\n The Python objects to display, or if raw=True raw HTML data to\n display.\n raw : bool\n Are the data objects raw data or Python objects that need to be\n formatted before display? [default: False]\n \"\"\"\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_html(obj)\n else:\n display(*objs, include=['text/plain','text/html'])"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndisplays the SVG representation of a Python object.", "response": "def display_svg(*objs, **kwargs):\n \"\"\"Display the SVG representation of an object.\n\n Parameters\n ----------\n objs : tuple of objects\n The Python objects to display, or if raw=True raw svg data to\n display.\n raw : bool\n Are the data objects raw data or Python objects that need to be\n formatted before display? [default: False]\n \"\"\"\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_svg(obj)\n else:\n display(*objs, include=['text/plain','image/svg+xml'])"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef display_png(*objs, **kwargs):\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_png(obj)\n else:\n display(*objs, include=['text/plain','image/png'])", "response": "Display the PNG representation of a Python object."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndisplay the JPEG representation of a Python object.", "response": "def display_jpeg(*objs, **kwargs):\n \"\"\"Display the JPEG representation of an object.\n\n Parameters\n ----------\n objs : tuple of objects\n The Python objects to display, or if raw=True raw JPEG data to\n display.\n raw : bool\n Are the data objects raw data or Python objects that need to be\n formatted before display? [default: False]\n \"\"\"\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_jpeg(obj)\n else:\n display(*objs, include=['text/plain','image/jpeg'])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndisplaying the LaTeX representation of a Python object.", "response": "def display_latex(*objs, **kwargs):\n \"\"\"Display the LaTeX representation of an object.\n\n Parameters\n ----------\n objs : tuple of objects\n The Python objects to display, or if raw=True raw latex data to\n display.\n raw : bool\n Are the data objects raw data or Python objects that need to be\n formatted before display? [default: False]\n \"\"\"\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_latex(obj)\n else:\n display(*objs, include=['text/plain','text/latex'])"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndisplays the JSON representation of a Python object.", "response": "def display_json(*objs, **kwargs):\n \"\"\"Display the JSON representation of an object.\n\n Note that not many frontends support displaying JSON.\n\n Parameters\n ----------\n objs : tuple of objects\n The Python objects to display, or if raw=True raw json data to\n display.\n raw : bool\n Are the data objects raw data or Python objects that need to be\n formatted before display? [default: False]\n \"\"\"\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_json(obj)\n else:\n display(*objs, include=['text/plain','application/json'])"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndisplaying the Javascript representation of a Python object.", "response": "def display_javascript(*objs, **kwargs):\n \"\"\"Display the Javascript representation of an object.\n\n Parameters\n ----------\n objs : tuple of objects\n The Python objects to display, or if raw=True raw javascript data to\n display.\n raw : bool\n Are the data objects raw data or Python objects that need to be\n formatted before display? [default: False]\n \"\"\"\n raw = kwargs.pop('raw',False)\n if raw:\n for obj in objs:\n publish_javascript(obj)\n else:\n display(*objs, include=['text/plain','application/javascript'])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef clear_output(stdout=True, stderr=True, other=True):\n from IPython.core.interactiveshell import InteractiveShell\n if InteractiveShell.initialized():\n InteractiveShell.instance().display_pub.clear_output(\n stdout=stdout, stderr=stderr, other=other,\n )\n else:\n from IPython.utils import io\n if stdout:\n print('\\033[2K\\r', file=io.stdout, end='')\n io.stdout.flush()\n if stderr:\n print('\\033[2K\\r', file=io.stderr, end='')\n io.stderr.flush()", "response": "Clear the output of the current cell receiving data."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreloading the raw data from file or URL.", "response": "def reload(self):\n \"\"\"Reload the raw data from file or URL.\"\"\"\n if self.filename is not None:\n with open(self.filename, self._read_flags) as f:\n self.data = f.read()\n elif self.url is not None:\n try:\n import urllib2\n response = urllib2.urlopen(self.url)\n self.data = response.read()\n # extract encoding from header, if there is one:\n encoding = None\n for sub in response.headers['content-type'].split(';'):\n sub = sub.strip()\n if sub.startswith('charset'):\n encoding = sub.split('=')[-1].strip()\n break\n # decode data, if an encoding was specified\n if encoding:\n self.data = self.data.decode(encoding, 'replace')\n except:\n self.data = None"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pip_version_check(session):\n import pip # imported here to prevent circular imports\n pypi_version = None\n\n try:\n state = load_selfcheck_statefile()\n\n current_time = datetime.datetime.utcnow()\n # Determine if we need to refresh the state\n if \"last_check\" in state.state and \"pypi_version\" in state.state:\n last_check = datetime.datetime.strptime(\n state.state[\"last_check\"],\n SELFCHECK_DATE_FMT\n )\n if total_seconds(current_time - last_check) < 7 * 24 * 60 * 60:\n pypi_version = state.state[\"pypi_version\"]\n\n # Refresh the version if we need to or just see if we need to warn\n if pypi_version is None:\n resp = session.get(\n PyPI.pip_json_url,\n headers={\"Accept\": \"application/json\"},\n )\n resp.raise_for_status()\n pypi_version = resp.json()[\"info\"][\"version\"]\n\n # save that we've performed a check\n state.save(pypi_version, current_time)\n\n pip_version = pkg_resources.parse_version(pip.__version__)\n\n # Determine if our pypi_version is older\n if pip_version < pkg_resources.parse_version(pypi_version):\n logger.warning(\n \"You are using pip version %s, however version %s is \"\n \"available.\\nYou should consider upgrading via the \"\n \"'pip install --upgrade pip' command.\" % (pip.__version__,\n pypi_version)\n )\n\n except Exception:\n logger.debug(\n \"There was an error checking the latest version of pip\",\n exc_info=True,\n )", "response": "Checks if the current version of the pip is older than the latest version of the pip."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfinding the full path to a command using which.", "response": "def _find_cmd(cmd):\n \"\"\"Find the full path to a command using which.\"\"\"\n\n path = sp.Popen(['/usr/bin/env', 'which', cmd],\n stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0]\n return py3compat.bytes_to_str(path)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getoutput_pexpect(self, cmd):\n try:\n return pexpect.run(self.sh, args=['-c', cmd]).replace('\\r\\n', '\\n')\n except KeyboardInterrupt:\n print('^C', file=sys.stderr, end='')", "response": "Run a command and return its stdout and stderr as a string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute a command in a subshell and return the exitstatus of the child process.", "response": "def system(self, cmd):\n \"\"\"Execute a command in a subshell.\n\n Parameters\n ----------\n cmd : str\n A command to be executed in the system shell.\n\n Returns\n -------\n int : child's exitstatus\n \"\"\"\n # Get likely encoding for the output.\n enc = DEFAULT_ENCODING\n \n # Patterns to match on the output, for pexpect. We read input and\n # allow either a short timeout or EOF\n patterns = [pexpect.TIMEOUT, pexpect.EOF]\n # the index of the EOF pattern in the list.\n # even though we know it's 1, this call means we don't have to worry if\n # we change the above list, and forget to change this value:\n EOF_index = patterns.index(pexpect.EOF)\n # The size of the output stored so far in the process output buffer.\n # Since pexpect only appends to this buffer, each time we print we\n # record how far we've printed, so that next time we only print *new*\n # content from the buffer.\n out_size = 0\n try:\n # Since we're not really searching the buffer for text patterns, we\n # can set pexpect's search window to be tiny and it won't matter.\n # We only search for the 'patterns' timeout or EOF, which aren't in\n # the text itself.\n #child = pexpect.spawn(pcmd, searchwindowsize=1)\n if hasattr(pexpect, 'spawnb'):\n child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U\n else:\n child = pexpect.spawn(self.sh, args=['-c', cmd]) # Vanilla Pexpect\n flush = sys.stdout.flush\n while True:\n # res is the index of the pattern that caused the match, so we\n # know whether we've finished (if we matched EOF) or not\n res_idx = child.expect_list(patterns, self.read_timeout)\n print(child.before[out_size:].decode(enc, 'replace'), end='')\n flush()\n if res_idx==EOF_index:\n break\n # Update the pointer to what we've already printed\n out_size = len(child.before)\n except KeyboardInterrupt:\n # We need to send ^C to the process. The ascii code for '^C' is 3\n # (the character is known as ETX for 'End of Text', see\n # curses.ascii.ETX).\n child.sendline(chr(3))\n # Read and print any more output the program might produce on its\n # way out.\n try:\n out_size = len(child.before)\n child.expect_list(patterns, self.terminate_timeout)\n print(child.before[out_size:].decode(enc, 'replace'), end='')\n sys.stdout.flush()\n except KeyboardInterrupt:\n # Impatient users tend to type it multiple times\n pass\n finally:\n # Ensure the subprocess really is terminated\n child.terminate(force=True)\n # add isalive check, to ensure exitstatus is set:\n child.isalive()\n return child.exitstatus"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nforward read events from an FD over a socket pair.", "response": "def forward_read_events(fd, context=None):\n \"\"\"Forward read events from an FD over a socket.\n\n This method wraps a file in a socket pair, so it can\n be polled for read events by select (specifically zmq.eventloop.ioloop)\n \"\"\"\n if context is None:\n context = zmq.Context.instance()\n push = context.socket(zmq.PUSH)\n push.setsockopt(zmq.LINGER, -1)\n pull = context.socket(zmq.PULL)\n addr='inproc://%s'%uuid.uuid4()\n push.bind(addr)\n pull.connect(addr)\n forwarder = ForwarderThread(push, fd)\n forwarder.start()\n return pull"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nloops through lines in self. fd and send them over self. sock.", "response": "def run(self):\n \"\"\"Loop through lines in self.fd, and send them over self.sock.\"\"\"\n line = self.fd.readline()\n # allow for files opened in unicode mode\n if isinstance(line, unicode):\n send = self.sock.send_unicode\n else:\n send = self.sock.send\n while line:\n send(line)\n line = self.fd.readline()\n # line == '' means EOF\n self.fd.close()\n self.sock.close()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a launcher class for a given class name and kind.", "response": "def find_launcher_class(clsname, kind):\n \"\"\"Return a launcher for a given clsname and kind.\n\n Parameters\n ==========\n clsname : str\n The full name of the launcher class, either with or without the\n module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF,\n WindowsHPC).\n kind : str\n Either 'EngineSet' or 'Controller'.\n \"\"\"\n if '.' not in clsname:\n # not a module, presume it's the raw name in apps.launcher\n if kind and kind not in clsname:\n # doesn't match necessary full class name, assume it's\n # just 'PBS' or 'MPI' prefix:\n clsname = clsname + kind + 'Launcher'\n clsname = 'IPython.parallel.apps.launcher.'+clsname\n klass = import_item(clsname)\n return klass"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting the app for the stop subcommand.", "response": "def start(self):\n \"\"\"Start the app for the stop subcommand.\"\"\"\n try:\n pid = self.get_pid_from_file()\n except PIDFileError:\n self.log.critical(\n 'Could not read pid file, cluster is probably not running.'\n )\n # Here I exit with a unusual exit status that other processes\n # can watch for to learn how I existed.\n self.remove_pid_file()\n self.exit(ALREADY_STOPPED)\n\n if not self.check_pid(pid):\n self.log.critical(\n 'Cluster [pid=%r] is not running.' % pid\n )\n self.remove_pid_file()\n # Here I exit with a unusual exit status that other processes\n # can watch for to learn how I existed.\n self.exit(ALREADY_STOPPED)\n\n elif os.name=='posix':\n sig = self.signal\n self.log.info(\n \"Stopping cluster [pid=%r] with [signal=%r]\" % (pid, sig)\n )\n try:\n os.kill(pid, sig)\n except OSError:\n self.log.error(\"Stopping cluster failed, assuming already dead.\",\n exc_info=True)\n self.remove_pid_file()\n elif os.name=='nt':\n try:\n # kill the whole tree\n p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)\n except (CalledProcessError, OSError):\n self.log.error(\"Stopping cluster failed, assuming already dead.\",\n exc_info=True)\n self.remove_pid_file()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef build_launcher(self, clsname, kind=None):\n try:\n klass = find_launcher_class(clsname, kind)\n except (ImportError, KeyError):\n self.log.fatal(\"Could not import launcher class: %r\"%clsname)\n self.exit(1)\n\n launcher = klass(\n work_dir=u'.', config=self.config, log=self.log,\n profile_dir=self.profile_dir.location, cluster_id=self.cluster_id,\n )\n return launcher", "response": "import and instantiate a Launcher based on importstring"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef start(self):\n self.log.info(\"IPython cluster: started\")\n # First see if the cluster is already running\n\n # Now log and daemonize\n self.log.info(\n 'Starting engines with [daemon=%r]' % self.daemonize\n )\n # TODO: Get daemonize working on Windows or as a Windows Server.\n if self.daemonize:\n if os.name=='posix':\n daemonize()\n\n dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)\n dc.start()\n # Now write the new pid file AFTER our new forked pid is active.\n # self.write_pid_file()\n try:\n self.loop.start()\n except KeyboardInterrupt:\n pass\n except zmq.ZMQError as e:\n if e.errno == errno.EINTR:\n pass\n else:\n raise", "response": "Start the engines app."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef start(self):\n # First see if the cluster is already running\n try:\n pid = self.get_pid_from_file()\n except PIDFileError:\n pass\n else:\n if self.check_pid(pid):\n self.log.critical(\n 'Cluster is already running with [pid=%s]. '\n 'use \"ipcluster stop\" to stop the cluster.' % pid\n )\n # Here I exit with a unusual exit status that other processes\n # can watch for to learn how I existed.\n self.exit(ALREADY_STARTED)\n else:\n self.remove_pid_file()\n\n\n # Now log and daemonize\n self.log.info(\n 'Starting ipcluster with [daemon=%r]' % self.daemonize\n )\n # TODO: Get daemonize working on Windows or as a Windows Server.\n if self.daemonize:\n if os.name=='posix':\n daemonize()\n\n dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)\n dc.start()\n dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)\n dc.start()\n # Now write the new pid file AFTER our new forked pid is active.\n self.write_pid_file()\n try:\n self.loop.start()\n except KeyboardInterrupt:\n pass\n except zmq.ZMQError as e:\n if e.errno == errno.EINTR:\n pass\n else:\n raise\n finally:\n self.remove_pid_file()", "response": "Start the ipcluster application."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_app_wx(*args, **kwargs):\n import wx\n app = wx.GetApp()\n if app is None:\n if not kwargs.has_key('redirect'):\n kwargs['redirect'] = False\n app = wx.PySimpleApp(*args, **kwargs)\n return app", "response": "Create a wx app or return an exiting one."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nbeing the wx event loop running?", "response": "def is_event_loop_running_wx(app=None):\n \"\"\"Is the wx event loop running.\"\"\"\n if app is None:\n app = get_app_wx()\n if hasattr(app, '_in_event_loop'):\n return app._in_event_loop\n else:\n return app.IsMainLoopRunning()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nstart the wx event loop in a consistent manner.", "response": "def start_event_loop_wx(app=None):\n \"\"\"Start the wx event loop in a consistent manner.\"\"\"\n if app is None:\n app = get_app_wx()\n if not is_event_loop_running_wx(app):\n app._in_event_loop = True\n app.MainLoop()\n app._in_event_loop = False\n else:\n app._in_event_loop = True"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a new qt4 app or return an existing one.", "response": "def get_app_qt4(*args, **kwargs):\n \"\"\"Create a new qt4 app or return an existing one.\"\"\"\n from IPython.external.qt_for_kernel import QtGui\n app = QtGui.QApplication.instance()\n if app is None:\n if not args:\n args = ([''],)\n app = QtGui.QApplication(*args, **kwargs)\n return app"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef is_event_loop_running_qt4(app=None):\n if app is None:\n app = get_app_qt4([''])\n if hasattr(app, '_in_event_loop'):\n return app._in_event_loop\n else:\n # Does qt4 provide a other way to detect this?\n return False", "response": "Is the qt4 event loop running?"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nstarting the qt4 event loop in a consistent manner.", "response": "def start_event_loop_qt4(app=None):\n \"\"\"Start the qt4 event loop in a consistent manner.\"\"\"\n if app is None:\n app = get_app_qt4([''])\n if not is_event_loop_running_qt4(app):\n app._in_event_loop = True\n app.exec_()\n app._in_event_loop = False\n else:\n app._in_event_loop = True"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking namespace packages' __init__ for declare_namespace", "response": "def check_package(self, package, package_dir):\n \"\"\"Check namespace packages' __init__ for declare_namespace\"\"\"\n try:\n return self.packages_checked[package]\n except KeyError:\n pass\n\n init_py = _build_py.check_package(self, package, package_dir)\n self.packages_checked[package] = init_py\n\n if not init_py or not self.distribution.namespace_packages:\n return init_py\n\n for pkg in self.distribution.namespace_packages:\n if pkg==package or pkg.startswith(package+'.'):\n break\n else:\n return init_py\n\n f = open(init_py,'rbU')\n if 'declare_namespace'.encode() not in f.read():\n from distutils import log\n log.warn(\n \"WARNING: %s is a namespace package, but its __init__.py does\\n\"\n \"not declare_namespace(); setuptools 0.7 will REQUIRE this!\\n\"\n '(See the setuptools manual under \"Namespace Packages\" for '\n \"details.)\\n\", package\n )\n f.close()\n return init_py"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef blank_canvas(width, height):\n canvas = np.zeros((height, width, 3), dtype=np.uint8)\n return canvas.view(Canvas)", "response": "Return a blank canvas to annotate."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef draw_cross(self, position, color=(255, 0, 0), radius=4):\n y, x = position\n for xmod in np.arange(-radius, radius+1, 1):\n xpos = x + xmod\n if xpos < 0:\n continue # Negative indices will draw on the opposite side.\n if xpos >= self.shape[1]:\n continue # Out of bounds.\n self[int(y), int(xpos)] = color\n for ymod in np.arange(-radius, radius+1, 1):\n ypos = y + ymod\n if ypos < 0:\n continue # Negative indices will draw on the opposite side.\n if ypos >= self.shape[0]:\n continue # Out of bounds.\n self[int(ypos), int(x)] = color", "response": "Draw a cross on the canvas."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndraws a line between pos1 and pos2 on the canvas.", "response": "def draw_line(self, pos1, pos2, color=(255, 0, 0)):\n \"\"\"Draw a line between pos1 and pos2 on the canvas.\n\n :param pos1: position 1 (row, col) tuple\n :param pos2: position 2 (row, col) tuple\n :param color: RGB tuple\n \"\"\"\n r1, c1 = tuple([int(round(i, 0)) for i in pos1])\n r2, c2 = tuple([int(round(i, 0)) for i in pos2])\n rr, cc = skimage.draw.line(r1, c1, r2, c2)\n self[rr, cc] = color"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef text_at(self, text, position, color=(255, 255, 255),\n size=12, antialias=False, center=False):\n \"\"\"Write text at x, y top left corner position.\n\n By default the x and y coordinates represent the top left hand corner\n of the text. The text can be centered vertically and horizontally by\n using setting the ``center`` option to ``True``.\n\n :param text: text to write\n :param position: (row, col) tuple\n :param color: RGB tuple\n :param size: font size\n :param antialias: whether or not the text should be antialiased\n :param center: whether or not the text should be centered on the\n input coordinate\n \"\"\"\n def antialias_value(value, normalisation):\n return int(round(value * normalisation))\n\n def antialias_rgb(color, normalisation):\n return tuple([antialias_value(v, normalisation) for v in color])\n\n def set_color(xpos, ypos, color):\n try:\n self[ypos, xpos] = color\n except IndexError:\n pass\n\n y, x = position\n font = PIL.ImageFont.truetype(DEFAULT_FONT_PATH, size=size)\n mask = font.getmask(text)\n width, height = mask.size\n if center:\n x = x - (width // 2)\n y = y - (height // 2)\n for ystep in range(height):\n for xstep in range(width):\n normalisation = mask[ystep * width + xstep] / 255.\n if antialias:\n if normalisation != 0:\n rgb_color = antialias_rgb(color, normalisation)\n set_color(x + xstep, y+ystep, rgb_color)\n else:\n if normalisation > .5:\n set_color(x + xstep, y + ystep, color)", "response": "Write text at x y top left corner position."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a canvas from a grayscale image.", "response": "def from_grayscale(im, channels_on=(True, True, True)):\n \"\"\"Return a canvas from a grayscale image.\n\n :param im: single channel image\n :channels_on: channels to populate with input image\n :returns: :class:`jicbioimage.illustrate.Canvas`\n \"\"\"\n xdim, ydim = im.shape\n canvas = np.zeros((xdim, ydim, 3), dtype=np.uint8)\n for i, include in enumerate(channels_on):\n if include:\n canvas[:, :, i] = im\n return canvas.view(AnnotatedImage)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a unique ID of a given length.", "response": "def get_uuid(length=32, version=1):\n \"\"\"\n Returns a unique ID of a given length.\n User `version=2` for cross-systems uniqueness.\n \"\"\"\n if version == 1:\n return uuid.uuid1().hex[:length]\n else:\n return uuid.uuid4().hex[:length]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_dict_to_encoded_url(data):\n unicode_data = dict([(k, smart_str(v)) for k, v in data.items()])\n encoded = urllib.urlencode(unicode_data)\n return encoded", "response": "Converts a dict to an encoded URL."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert an encoded URL to a dict.", "response": "def get_encoded_url_to_dict(string):\n \"\"\"\n Converts an encoded URL to a dict.\n Example: given string = 'a=1&b=2' it returns {'a': 1, 'b': 2}\n \"\"\"\n data = urllib.parse.parse_qsl(string, keep_blank_values=True)\n data = dict(data)\n return data"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbuild a unique key from get data", "response": "def get_unique_key_from_get(get_dict):\n \"\"\"\n Build a unique key from get data\n \"\"\"\n site = Site.objects.get_current()\n key = get_dict_to_encoded_url(get_dict)\n cache_key = '{}_{}'.format(site.domain, key)\n return hashlib.md5(cache_key).hexdigest()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngive a decimal number returns a string bitfield of length = len", "response": "def tobin(deci_num, len=32):\n \"\"\"\n Given a decimal number, returns a string bitfield of length = len\n Example: given deci_num = 1 and len = 10, it return 0000000001\n \"\"\"\n bitstr = \"\".join(map(lambda y: str((deci_num >> y) & 1), range(len - 1, -1, -1)))\n return bitstr"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nvalidates and email address.", "response": "def is_valid_email(email):\n \"\"\"\n Validates and email address.\n Note: valid emails must follow the @<.extension> patterns.\n \"\"\"\n try:\n validate_email(email)\n except ValidationError:\n return False\n if simple_email_re.match(email):\n return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_domain(url):\n if 'http' not in url.lower():\n url = 'http://{}'.format(url)\n return urllib.parse.urlparse(url).hostname", "response": "Returns the domain name portion of a URL"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_url_args(url):\n url_data = urllib.parse.urlparse(url)\n arg_dict = urllib.parse.parse_qs(url_data.query)\n return arg_dict", "response": "Returns a dictionary from a URL params"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef learn(env,\n network,\n seed=None,\n lr=5e-4,\n total_timesteps=100000,\n buffer_size=50000,\n exploration_fraction=0.1,\n exploration_final_eps=0.02,\n train_freq=1,\n batch_size=32,\n print_freq=100,\n checkpoint_freq=10000,\n checkpoint_path=None,\n learning_starts=1000,\n gamma=1.0,\n target_network_update_freq=500,\n prioritized_replay=False,\n prioritized_replay_alpha=0.6,\n prioritized_replay_beta0=0.4,\n prioritized_replay_beta_iters=None,\n prioritized_replay_eps=1e-6,\n param_noise=False,\n callback=None,\n load_path=None,\n **network_kwargs\n ):\n \"\"\"Train a deepq model.\n\n Parameters\n -------\n env: gym.Env\n environment to train on\n network: string or a function\n neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models\n (mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which\n will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)\n seed: int or None\n prng seed. The runs with the same seed \"should\" give the same results. If None, no seeding is used.\n lr: float\n learning rate for adam optimizer\n total_timesteps: int\n number of env steps to optimizer for\n buffer_size: int\n size of the replay buffer\n exploration_fraction: float\n fraction of entire training period over which the exploration rate is annealed\n exploration_final_eps: float\n final value of random action probability\n train_freq: int\n update the model every `train_freq` steps.\n set to None to disable printing\n batch_size: int\n size of a batched sampled from replay buffer for training\n print_freq: int\n how often to print out training progress\n set to None to disable printing\n checkpoint_freq: int\n how often to save the model. This is so that the best version is restored\n at the end of the training. If you do not wish to restore the best version at\n the end of the training set this variable to None.\n learning_starts: int\n how many steps of the model to collect transitions for before learning starts\n gamma: float\n discount factor\n target_network_update_freq: int\n update the target network every `target_network_update_freq` steps.\n prioritized_replay: True\n if True prioritized replay buffer will be used.\n prioritized_replay_alpha: float\n alpha parameter for prioritized replay buffer\n prioritized_replay_beta0: float\n initial value of beta for prioritized replay buffer\n prioritized_replay_beta_iters: int\n number of iterations over which beta will be annealed from initial value\n to 1.0. If set to None equals to total_timesteps.\n prioritized_replay_eps: float\n epsilon to add to the TD errors when updating priorities.\n param_noise: bool\n whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)\n callback: (locals, globals) -> None\n function called at every steps with state of the algorithm.\n If callback returns true training stops.\n load_path: str\n path to load the model from. (default: None)\n **network_kwargs\n additional keyword arguments to pass to the network builder.\n\n Returns\n -------\n act: ActWrapper\n Wrapper over act function. Adds ability to save it and load it.\n See header of baselines/deepq/categorical.py for details on the act function.\n \"\"\"\n # Create all the functions necessary to train the model\n\n sess = get_session()\n set_global_seeds(seed)\n\n q_func = build_q_func(network, **network_kwargs)\n\n # capture the shape outside the closure so that the env object is not serialized\n # by cloudpickle when serializing make_obs_ph\n\n observation_space = env.observation_space\n def make_obs_ph(name):\n return ObservationInput(observation_space, name=name)\n\n act, train, update_target, debug = deepq.build_train(\n make_obs_ph=make_obs_ph,\n q_func=q_func,\n num_actions=env.action_space.n,\n optimizer=tf.train.AdamOptimizer(learning_rate=lr),\n gamma=gamma,\n grad_norm_clipping=10,\n param_noise=param_noise\n )\n\n act_params = {\n 'make_obs_ph': make_obs_ph,\n 'q_func': q_func,\n 'num_actions': env.action_space.n,\n }\n\n act = ActWrapper(act, act_params)\n\n # Create the replay buffer\n if prioritized_replay:\n replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)\n if prioritized_replay_beta_iters is None:\n prioritized_replay_beta_iters = total_timesteps\n beta_schedule = LinearSchedule(prioritized_replay_beta_iters,\n initial_p=prioritized_replay_beta0,\n final_p=1.0)\n else:\n replay_buffer = ReplayBuffer(buffer_size)\n beta_schedule = None\n # Create the schedule for exploration starting from 1.\n exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * total_timesteps),\n initial_p=1.0,\n final_p=exploration_final_eps)\n\n # Initialize the parameters and copy them to the target network.\n U.initialize()\n update_target()\n\n episode_rewards = [0.0]\n saved_mean_reward = None\n obs = env.reset()\n reset = True\n\n with tempfile.TemporaryDirectory() as td:\n td = checkpoint_path or td\n\n model_file = os.path.join(td, \"model\")\n model_saved = False\n\n if tf.train.latest_checkpoint(td) is not None:\n load_variables(model_file)\n logger.log('Loaded model from {}'.format(model_file))\n model_saved = True\n elif load_path is not None:\n load_variables(load_path)\n logger.log('Loaded model from {}'.format(load_path))\n\n\n for t in range(total_timesteps):\n if callback is not None:\n if callback(locals(), globals()):\n break\n # Take action and update exploration to the newest value\n kwargs = {}\n if not param_noise:\n update_eps = exploration.value(t)\n update_param_noise_threshold = 0.\n else:\n update_eps = 0.\n # Compute the threshold such that the KL divergence between perturbed and non-perturbed\n # policy is comparable to eps-greedy exploration with eps = exploration.value(t).\n # See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017\n # for detailed explanation.\n update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))\n kwargs['reset'] = reset\n kwargs['update_param_noise_threshold'] = update_param_noise_threshold\n kwargs['update_param_noise_scale'] = True\n action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]\n env_action = action\n reset = False\n new_obs, rew, done, _ = env.step(env_action)\n # Store transition in the replay buffer.\n replay_buffer.add(obs, action, rew, new_obs, float(done))\n obs = new_obs\n\n episode_rewards[-1] += rew\n if done:\n obs = env.reset()\n episode_rewards.append(0.0)\n reset = True\n\n if t > learning_starts and t % train_freq == 0:\n # Minimize the error in Bellman's equation on a batch sampled from replay buffer.\n if prioritized_replay:\n experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))\n (obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience\n else:\n obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)\n weights, batch_idxes = np.ones_like(rewards), None\n td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)\n if prioritized_replay:\n new_priorities = np.abs(td_errors) + prioritized_replay_eps\n replay_buffer.update_priorities(batch_idxes, new_priorities)\n\n if t > learning_starts and t % target_network_update_freq == 0:\n # Update target network periodically.\n update_target()\n\n mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)\n num_episodes = len(episode_rewards)\n if done and print_freq is not None and len(episode_rewards) % print_freq == 0:\n logger.record_tabular(\"steps\", t)\n logger.record_tabular(\"episodes\", num_episodes)\n logger.record_tabular(\"mean 100 episode reward\", mean_100ep_reward)\n logger.record_tabular(\"% time spent exploring\", int(100 * exploration.value(t)))\n logger.dump_tabular()\n\n if (checkpoint_freq is not None and t > learning_starts and\n num_episodes > 100 and t % checkpoint_freq == 0):\n if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:\n if print_freq is not None:\n logger.log(\"Saving model due to mean reward increase: {} -> {}\".format(\n saved_mean_reward, mean_100ep_reward))\n save_variables(model_file)\n model_saved = True\n saved_mean_reward = mean_100ep_reward\n if model_saved:\n if print_freq is not None:\n logger.log(\"Restored model with mean reward: {}\".format(saved_mean_reward))\n load_variables(model_file)\n\n return act", "response": "Train a deepq model on a neural network."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves the current state of the current state to a pickle located at path", "response": "def save_act(self, path=None):\n \"\"\"Save model to a pickle located at `path`\"\"\"\n if path is None:\n path = os.path.join(logger.get_dir(), \"model.pkl\")\n\n with tempfile.TemporaryDirectory() as td:\n save_variables(os.path.join(td, \"model\"))\n arc_name = os.path.join(td, \"packed.zip\")\n with zipfile.ZipFile(arc_name, 'w') as zipf:\n for root, dirs, files in os.walk(td):\n for fname in files:\n file_path = os.path.join(root, fname)\n if file_path != arc_name:\n zipf.write(file_path, os.path.relpath(file_path, td))\n with open(arc_name, \"rb\") as f:\n model_data = f.read()\n with open(path, \"wb\") as f:\n cloudpickle.dump((model_data, self._act_params), f)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef mlp(num_layers=2, num_hidden=64, activation=tf.tanh, layer_norm=False):\n def network_fn(X):\n h = tf.layers.flatten(X)\n for i in range(num_layers):\n h = fc(h, 'mlp_fc{}'.format(i), nh=num_hidden, init_scale=np.sqrt(2))\n if layer_norm:\n h = tf.contrib.layers.layer_norm(h, center=True, scale=True)\n h = activation(h)\n\n return h\n\n return network_fn", "response": "A function that builds a stack of fully - connected layers for a given input tensor."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds LSTM network for a given set of parameters.", "response": "def lstm(nlstm=128, layer_norm=False):\n \"\"\"\n Builds LSTM (Long-Short Term Memory) network to be used in a policy.\n Note that the resulting function returns not only the output of the LSTM\n (i.e. hidden state of lstm for each step in the sequence), but also a dictionary\n with auxiliary tensors to be set as policy attributes.\n\n Specifically,\n S is a placeholder to feed current state (LSTM state has to be managed outside policy)\n M is a placeholder for the mask (used to mask out observations after the end of the episode, but can be used for other purposes too)\n initial_state is a numpy array containing initial lstm state (usually zeros)\n state is the output LSTM state (to be fed into S at the next call)\n\n\n An example of usage of lstm-based policy can be found here: common/tests/test_doc_examples.py/test_lstm_example\n\n Parameters:\n ----------\n\n nlstm: int LSTM hidden state size\n\n layer_norm: bool if True, layer-normalized version of LSTM is used\n\n Returns:\n -------\n\n function that builds LSTM with a given input tensor / placeholder\n \"\"\"\n\n def network_fn(X, nenv=1):\n nbatch = X.shape[0]\n nsteps = nbatch // nenv\n\n h = tf.layers.flatten(X)\n\n M = tf.placeholder(tf.float32, [nbatch]) #mask (done t-1)\n S = tf.placeholder(tf.float32, [nenv, 2*nlstm]) #states\n\n xs = batch_to_seq(h, nenv, nsteps)\n ms = batch_to_seq(M, nenv, nsteps)\n\n if layer_norm:\n h5, snew = utils.lnlstm(xs, ms, S, scope='lnlstm', nh=nlstm)\n else:\n h5, snew = utils.lstm(xs, ms, S, scope='lstm', nh=nlstm)\n\n h = seq_to_batch(h5)\n initial_state = np.zeros(S.shape.as_list(), dtype=float)\n\n return h, {'S':S, 'M':M, 'state':snew, 'initial_state':initial_state}\n\n return network_fn"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef conv_only(convs=[(32, 8, 4), (64, 4, 2), (64, 3, 1)], **conv_kwargs):\n '''\n convolutions-only net\n\n Parameters:\n ----------\n\n conv: list of triples (filter_number, filter_size, stride) specifying parameters for each layer.\n\n Returns:\n\n function that takes tensorflow tensor as input and returns the output of the last convolutional layer\n\n '''\n\n def network_fn(X):\n out = tf.cast(X, tf.float32) / 255.\n with tf.variable_scope(\"convnet\"):\n for num_outputs, kernel_size, stride in convs:\n out = layers.convolution2d(out,\n num_outputs=num_outputs,\n kernel_size=kernel_size,\n stride=stride,\n activation_fn=tf.nn.relu,\n **conv_kwargs)\n\n return out\n return network_fn", "response": "Convolutions - only network."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_network_builder(name):\n if callable(name):\n return name\n elif name in mapping:\n return mapping[name]\n else:\n raise ValueError('Unknown network type: {}'.format(name))", "response": "Returns a function that can be used to build a network."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef mlp(hiddens=[], layer_norm=False):\n return lambda *args, **kwargs: _mlp(hiddens, layer_norm=layer_norm, *args, **kwargs)", "response": "This model takes as input an observation and returns values of all actions."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a wrapped SubprocVecEnv for Atari and MuJoCo.", "response": "def make_vec_env(env_id, env_type, num_env, seed,\n wrapper_kwargs=None,\n start_index=0,\n reward_scale=1.0,\n flatten_dict_observations=True,\n gamestate=None):\n \"\"\"\n Create a wrapped, monitored SubprocVecEnv for Atari and MuJoCo.\n \"\"\"\n wrapper_kwargs = wrapper_kwargs or {}\n mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else 0\n seed = seed + 10000 * mpi_rank if seed is not None else None\n logger_dir = logger.get_dir()\n def make_thunk(rank):\n return lambda: make_env(\n env_id=env_id,\n env_type=env_type,\n mpi_rank=mpi_rank,\n subrank=rank,\n seed=seed,\n reward_scale=reward_scale,\n gamestate=gamestate,\n flatten_dict_observations=flatten_dict_observations,\n wrapper_kwargs=wrapper_kwargs,\n logger_dir=logger_dir\n )\n\n set_global_seeds(seed)\n if num_env > 1:\n return SubprocVecEnv([make_thunk(i + start_index) for i in range(num_env)])\n else:\n return DummyVecEnv([make_thunk(start_index)])"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef make_mujoco_env(env_id, seed, reward_scale=1.0):\n rank = MPI.COMM_WORLD.Get_rank()\n myseed = seed + 1000 * rank if seed is not None else None\n set_global_seeds(myseed)\n env = gym.make(env_id)\n logger_path = None if logger.get_dir() is None else os.path.join(logger.get_dir(), str(rank))\n env = Monitor(env, logger_path, allow_early_resets=True)\n env.seed(seed)\n if reward_scale != 1.0:\n from baselines.common.retro_wrappers import RewardScaler\n env = RewardScaler(env, reward_scale)\n return env", "response": "Create a wrapped gym. Env for MuJoCo."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_robotics_env(env_id, seed, rank=0):\n set_global_seeds(seed)\n env = gym.make(env_id)\n env = FlattenDictWrapper(env, ['observation', 'desired_goal'])\n env = Monitor(\n env, logger.get_dir() and os.path.join(logger.get_dir(), str(rank)),\n info_keywords=('is_success',))\n env.seed(seed)\n return env", "response": "Create a wrapped gym. Env for MuJoCo."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates an argparse. ArgumentParser for run_mujoco. py.", "response": "def common_arg_parser():\n \"\"\"\n Create an argparse.ArgumentParser for run_mujoco.py.\n \"\"\"\n parser = arg_parser()\n parser.add_argument('--env', help='environment ID', type=str, default='Reacher-v2')\n parser.add_argument('--env_type', help='type of environment, used when the environment type cannot be automatically determined', type=str)\n parser.add_argument('--seed', help='RNG seed', type=int, default=None)\n parser.add_argument('--alg', help='Algorithm', type=str, default='ppo2')\n parser.add_argument('--num_timesteps', type=float, default=1e6),\n parser.add_argument('--network', help='network type (mlp, cnn, lstm, cnn_lstm, conv_only)', default=None)\n parser.add_argument('--gamestate', help='game state to load (so far only used in retro games)', default=None)\n parser.add_argument('--num_env', help='Number of environment copies being run in parallel. When not specified, set to number of cpus for Atari, and to 1 for Mujoco', default=None, type=int)\n parser.add_argument('--reward_scale', help='Reward scale factor. Default: 1.0', default=1.0, type=float)\n parser.add_argument('--save_path', help='Path to save trained model to', default=None, type=str)\n parser.add_argument('--save_video_interval', help='Save video every x steps (0 = disabled)', default=0, type=int)\n parser.add_argument('--save_video_length', help='Length of recorded video. Default: 200', default=200, type=int)\n parser.add_argument('--play', default=False, action='store_true')\n return parser"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef robotics_arg_parser():\n parser = arg_parser()\n parser.add_argument('--env', help='environment ID', type=str, default='FetchReach-v0')\n parser.add_argument('--seed', help='RNG seed', type=int, default=None)\n parser.add_argument('--num-timesteps', type=int, default=int(1e6))\n return parser", "response": "Create an argparse. ArgumentParser for run_mujoco. py.\n "} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse unknown arguments into a dicitonary", "response": "def parse_unknown_args(args):\n \"\"\"\n Parse arguments not consumed by arg parser into a dicitonary\n \"\"\"\n retval = {}\n preceded_by_key = False\n for arg in args:\n if arg.startswith('--'):\n if '=' in arg:\n key = arg.split('=')[0][2:]\n value = arg.split('=')[1]\n retval[key] = value\n else:\n key = arg[2:]\n preceded_by_key = True\n elif preceded_by_key:\n retval[key] = arg\n preceded_by_key = False\n\n return retval"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef clear_mpi_env_vars():\n removed_environment = {}\n for k, v in list(os.environ.items()):\n for prefix in ['OMPI_', 'PMI_']:\n if k.startswith(prefix):\n removed_environment[k] = v\n del os.environ[k]\n try:\n yield\n finally:\n os.environ.update(removed_environment)", "response": "Context manager to clear MPI environment variables."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nlearn policy using PPO algorithm.", "response": "def learn(*, network, env, total_timesteps, eval_env = None, seed=None, nsteps=2048, ent_coef=0.0, lr=3e-4,\n vf_coef=0.5, max_grad_norm=0.5, gamma=0.99, lam=0.95,\n log_interval=10, nminibatches=4, noptepochs=4, cliprange=0.2,\n save_interval=0, load_path=None, model_fn=None, **network_kwargs):\n '''\n Learn policy using PPO algorithm (https://arxiv.org/abs/1707.06347)\n\n Parameters:\n ----------\n\n network: policy network architecture. Either string (mlp, lstm, lnlstm, cnn_lstm, cnn, cnn_small, conv_only - see baselines.common/models.py for full list)\n specifying the standard network architecture, or a function that takes tensorflow tensor as input and returns\n tuple (output_tensor, extra_feed) where output tensor is the last network layer output, extra_feed is None for feed-forward\n neural nets, and extra_feed is a dictionary describing how to feed state into the network for recurrent neural nets.\n See common/models.py/lstm for more details on using recurrent nets in policies\n\n env: baselines.common.vec_env.VecEnv environment. Needs to be vectorized for parallel environment simulation.\n The environments produced by gym.make can be wrapped using baselines.common.vec_env.DummyVecEnv class.\n\n\n nsteps: int number of steps of the vectorized environment per update (i.e. batch size is nsteps * nenv where\n nenv is number of environment copies simulated in parallel)\n\n total_timesteps: int number of timesteps (i.e. number of actions taken in the environment)\n\n ent_coef: float policy entropy coefficient in the optimization objective\n\n lr: float or function learning rate, constant or a schedule function [0,1] -> R+ where 1 is beginning of the\n training and 0 is the end of the training.\n\n vf_coef: float value function loss coefficient in the optimization objective\n\n max_grad_norm: float or None gradient norm clipping coefficient\n\n gamma: float discounting factor\n\n lam: float advantage estimation discounting factor (lambda in the paper)\n\n log_interval: int number of timesteps between logging events\n\n nminibatches: int number of training minibatches per update. For recurrent policies,\n should be smaller or equal than number of environments run in parallel.\n\n noptepochs: int number of training epochs per update\n\n cliprange: float or function clipping range, constant or schedule function [0,1] -> R+ where 1 is beginning of the training\n and 0 is the end of the training\n\n save_interval: int number of timesteps between saving events\n\n load_path: str path to load the model from\n\n **network_kwargs: keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network\n For instance, 'mlp' network architecture has arguments num_hidden and num_layers.\n\n\n\n '''\n\n set_global_seeds(seed)\n\n if isinstance(lr, float): lr = constfn(lr)\n else: assert callable(lr)\n if isinstance(cliprange, float): cliprange = constfn(cliprange)\n else: assert callable(cliprange)\n total_timesteps = int(total_timesteps)\n\n policy = build_policy(env, network, **network_kwargs)\n\n # Get the nb of env\n nenvs = env.num_envs\n\n # Get state_space and action_space\n ob_space = env.observation_space\n ac_space = env.action_space\n\n # Calculate the batch_size\n nbatch = nenvs * nsteps\n nbatch_train = nbatch // nminibatches\n\n # Instantiate the model object (that creates act_model and train_model)\n if model_fn is None:\n from baselines.ppo2.model import Model\n model_fn = Model\n\n model = model_fn(policy=policy, ob_space=ob_space, ac_space=ac_space, nbatch_act=nenvs, nbatch_train=nbatch_train,\n nsteps=nsteps, ent_coef=ent_coef, vf_coef=vf_coef,\n max_grad_norm=max_grad_norm)\n\n if load_path is not None:\n model.load(load_path)\n # Instantiate the runner object\n runner = Runner(env=env, model=model, nsteps=nsteps, gamma=gamma, lam=lam)\n if eval_env is not None:\n eval_runner = Runner(env = eval_env, model = model, nsteps = nsteps, gamma = gamma, lam= lam)\n\n epinfobuf = deque(maxlen=100)\n if eval_env is not None:\n eval_epinfobuf = deque(maxlen=100)\n\n # Start total timer\n tfirststart = time.perf_counter()\n\n nupdates = total_timesteps//nbatch\n for update in range(1, nupdates+1):\n assert nbatch % nminibatches == 0\n # Start timer\n tstart = time.perf_counter()\n frac = 1.0 - (update - 1.0) / nupdates\n # Calculate the learning rate\n lrnow = lr(frac)\n # Calculate the cliprange\n cliprangenow = cliprange(frac)\n # Get minibatch\n obs, returns, masks, actions, values, neglogpacs, states, epinfos = runner.run() #pylint: disable=E0632\n if eval_env is not None:\n eval_obs, eval_returns, eval_masks, eval_actions, eval_values, eval_neglogpacs, eval_states, eval_epinfos = eval_runner.run() #pylint: disable=E0632\n\n epinfobuf.extend(epinfos)\n if eval_env is not None:\n eval_epinfobuf.extend(eval_epinfos)\n\n # Here what we're going to do is for each minibatch calculate the loss and append it.\n mblossvals = []\n if states is None: # nonrecurrent version\n # Index of each element of batch_size\n # Create the indices array\n inds = np.arange(nbatch)\n for _ in range(noptepochs):\n # Randomize the indexes\n np.random.shuffle(inds)\n # 0 to batch_size with batch_train_size step\n for start in range(0, nbatch, nbatch_train):\n end = start + nbatch_train\n mbinds = inds[start:end]\n slices = (arr[mbinds] for arr in (obs, returns, masks, actions, values, neglogpacs))\n mblossvals.append(model.train(lrnow, cliprangenow, *slices))\n else: # recurrent version\n assert nenvs % nminibatches == 0\n envsperbatch = nenvs // nminibatches\n envinds = np.arange(nenvs)\n flatinds = np.arange(nenvs * nsteps).reshape(nenvs, nsteps)\n for _ in range(noptepochs):\n np.random.shuffle(envinds)\n for start in range(0, nenvs, envsperbatch):\n end = start + envsperbatch\n mbenvinds = envinds[start:end]\n mbflatinds = flatinds[mbenvinds].ravel()\n slices = (arr[mbflatinds] for arr in (obs, returns, masks, actions, values, neglogpacs))\n mbstates = states[mbenvinds]\n mblossvals.append(model.train(lrnow, cliprangenow, *slices, mbstates))\n\n # Feedforward --> get losses --> update\n lossvals = np.mean(mblossvals, axis=0)\n # End timer\n tnow = time.perf_counter()\n # Calculate the fps (frame per second)\n fps = int(nbatch / (tnow - tstart))\n if update % log_interval == 0 or update == 1:\n # Calculates if value function is a good predicator of the returns (ev > 1)\n # or if it's just worse than predicting nothing (ev =< 0)\n ev = explained_variance(values, returns)\n logger.logkv(\"serial_timesteps\", update*nsteps)\n logger.logkv(\"nupdates\", update)\n logger.logkv(\"total_timesteps\", update*nbatch)\n logger.logkv(\"fps\", fps)\n logger.logkv(\"explained_variance\", float(ev))\n logger.logkv('eprewmean', safemean([epinfo['r'] for epinfo in epinfobuf]))\n logger.logkv('eplenmean', safemean([epinfo['l'] for epinfo in epinfobuf]))\n if eval_env is not None:\n logger.logkv('eval_eprewmean', safemean([epinfo['r'] for epinfo in eval_epinfobuf]) )\n logger.logkv('eval_eplenmean', safemean([epinfo['l'] for epinfo in eval_epinfobuf]) )\n logger.logkv('time_elapsed', tnow - tfirststart)\n for (lossval, lossname) in zip(lossvals, model.loss_names):\n logger.logkv(lossname, lossval)\n if MPI is None or MPI.COMM_WORLD.Get_rank() == 0:\n logger.dumpkvs()\n if save_interval and (update % save_interval == 0 or update == 1) and logger.get_dir() and (MPI is None or MPI.COMM_WORLD.Get_rank() == 0):\n checkdir = osp.join(logger.get_dir(), 'checkpoints')\n os.makedirs(checkdir, exist_ok=True)\n savepath = osp.join(checkdir, '%.5i'%update)\n print('Saving to', savepath)\n model.save(savepath)\n return model"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cg(f_Ax, b, cg_iters=10, callback=None, verbose=False, residual_tol=1e-10):\n p = b.copy()\n r = b.copy()\n x = np.zeros_like(b)\n rdotr = r.dot(r)\n\n fmtstr = \"%10i %10.3g %10.3g\"\n titlestr = \"%10s %10s %10s\"\n if verbose: print(titlestr % (\"iter\", \"residual norm\", \"soln norm\"))\n\n for i in range(cg_iters):\n if callback is not None:\n callback(x)\n if verbose: print(fmtstr % (i, rdotr, np.linalg.norm(x)))\n z = f_Ax(p)\n v = rdotr / p.dot(z)\n x += v*p\n r -= v*z\n newrdotr = r.dot(r)\n mu = newrdotr/rdotr\n p = r + mu*p\n\n rdotr = newrdotr\n if rdotr < residual_tol:\n break\n\n if callback is not None:\n callback(x)\n if verbose: print(fmtstr % (i+1, rdotr, np.linalg.norm(x))) # pylint: disable=W0631\n return x", "response": "This function computes the CNV - based version of the CNV algorithm."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a tensor that can be used to feed observations into the batch size appropriate to the observation space.", "response": "def observation_placeholder(ob_space, batch_size=None, name='Ob'):\n '''\n Create placeholder to feed observations into of the size appropriate to the observation space\n\n Parameters:\n ----------\n\n ob_space: gym.Space observation space\n\n batch_size: int size of the batch to be fed into input. Can be left None in most cases.\n\n name: str name of the placeholder\n\n Returns:\n -------\n\n tensorflow placeholder tensor\n '''\n\n assert isinstance(ob_space, Discrete) or isinstance(ob_space, Box) or isinstance(ob_space, MultiDiscrete), \\\n 'Can only deal with Discrete and Box observation spaces for now'\n\n dtype = ob_space.dtype\n if dtype == np.int8:\n dtype = np.uint8\n\n return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef observation_input(ob_space, batch_size=None, name='Ob'):\n '''\n Create placeholder to feed observations into of the size appropriate to the observation space, and add input\n encoder of the appropriate type.\n '''\n\n placeholder = observation_placeholder(ob_space, batch_size, name)\n return placeholder, encode_observation(ob_space, placeholder)", "response": "Create a placeholder to feed observations into the observation space."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef encode_observation(ob_space, placeholder):\n '''\n Encode input in the way that is appropriate to the observation space\n\n Parameters:\n ----------\n\n ob_space: gym.Space observation space\n\n placeholder: tf.placeholder observation input placeholder\n '''\n if isinstance(ob_space, Discrete):\n return tf.to_float(tf.one_hot(placeholder, ob_space.n))\n elif isinstance(ob_space, Box):\n return tf.to_float(placeholder)\n elif isinstance(ob_space, MultiDiscrete):\n placeholder = tf.cast(placeholder, tf.int32)\n one_hots = [tf.to_float(tf.one_hot(placeholder[..., i], ob_space.nvec[i])) for i in range(placeholder.shape[-1])]\n return tf.concat(one_hots, axis=-1)\n else:\n raise NotImplementedError", "response": "Encode input in the way that is appropriate to the observation space\n "} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates the rollouts for the current time horizon T.", "response": "def generate_rollouts(self):\n \"\"\"Performs `rollout_batch_size` rollouts in parallel for time horizon `T` with the current\n policy acting on it accordingly.\n \"\"\"\n self.reset_all_rollouts()\n\n # compute observations\n o = np.empty((self.rollout_batch_size, self.dims['o']), np.float32) # observations\n ag = np.empty((self.rollout_batch_size, self.dims['g']), np.float32) # achieved goals\n o[:] = self.initial_o\n ag[:] = self.initial_ag\n\n # generate episodes\n obs, achieved_goals, acts, goals, successes = [], [], [], [], []\n dones = []\n info_values = [np.empty((self.T - 1, self.rollout_batch_size, self.dims['info_' + key]), np.float32) for key in self.info_keys]\n Qs = []\n for t in range(self.T):\n policy_output = self.policy.get_actions(\n o, ag, self.g,\n compute_Q=self.compute_Q,\n noise_eps=self.noise_eps if not self.exploit else 0.,\n random_eps=self.random_eps if not self.exploit else 0.,\n use_target_net=self.use_target_net)\n\n if self.compute_Q:\n u, Q = policy_output\n Qs.append(Q)\n else:\n u = policy_output\n\n if u.ndim == 1:\n # The non-batched case should still have a reasonable shape.\n u = u.reshape(1, -1)\n\n o_new = np.empty((self.rollout_batch_size, self.dims['o']))\n ag_new = np.empty((self.rollout_batch_size, self.dims['g']))\n success = np.zeros(self.rollout_batch_size)\n # compute new states and observations\n obs_dict_new, _, done, info = self.venv.step(u)\n o_new = obs_dict_new['observation']\n ag_new = obs_dict_new['achieved_goal']\n success = np.array([i.get('is_success', 0.0) for i in info])\n\n if any(done):\n # here we assume all environments are done is ~same number of steps, so we terminate rollouts whenever any of the envs returns done\n # trick with using vecenvs is not to add the obs from the environments that are \"done\", because those are already observations\n # after a reset\n break\n\n for i, info_dict in enumerate(info):\n for idx, key in enumerate(self.info_keys):\n info_values[idx][t, i] = info[i][key]\n\n if np.isnan(o_new).any():\n self.logger.warn('NaN caught during rollout generation. Trying again...')\n self.reset_all_rollouts()\n return self.generate_rollouts()\n\n dones.append(done)\n obs.append(o.copy())\n achieved_goals.append(ag.copy())\n successes.append(success.copy())\n acts.append(u.copy())\n goals.append(self.g.copy())\n o[...] = o_new\n ag[...] = ag_new\n obs.append(o.copy())\n achieved_goals.append(ag.copy())\n\n episode = dict(o=obs,\n u=acts,\n g=goals,\n ag=achieved_goals)\n for key, value in zip(self.info_keys, info_values):\n episode['info_{}'.format(key)] = value\n\n # stats\n successful = np.array(successes)[-1, :]\n assert successful.shape == (self.rollout_batch_size,)\n success_rate = np.mean(successful)\n self.success_history.append(success_rate)\n if self.compute_Q:\n self.Q_history.append(np.mean(Qs))\n self.n_episodes += self.rollout_batch_size\n\n return convert_episode_to_batch_major(episode)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_policy(self, path):\n with open(path, 'wb') as f:\n pickle.dump(self.policy, f)", "response": "Save the current policy to a file."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef logs(self, prefix='worker'):\n logs = []\n logs += [('success_rate', np.mean(self.success_history))]\n if self.compute_Q:\n logs += [('mean_Q', np.mean(self.Q_history))]\n logs += [('episode', self.n_episodes)]\n\n if prefix != '' and not prefix.endswith('/'):\n return [(prefix + '/' + key, val) for key, val in logs]\n else:\n return logs", "response": "Generates a dictionary that contains all collected statistics."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nperform one - sided EMA on a new resource.", "response": "def one_sided_ema(xolds, yolds, low=None, high=None, n=512, decay_steps=1., low_counts_threshold=1e-8):\n '''\n perform one-sided (causal) EMA (exponential moving average)\n smoothing and resampling to an even grid with n points.\n Does not do extrapolation, so we assume\n xolds[0] <= low && high <= xolds[-1]\n\n Arguments:\n\n xolds: array or list - x values of data. Needs to be sorted in ascending order\n yolds: array of list - y values of data. Has to have the same length as xolds\n\n low: float - min value of the new x grid. By default equals to xolds[0]\n high: float - max value of the new x grid. By default equals to xolds[-1]\n\n n: int - number of points in new x grid\n\n decay_steps: float - EMA decay factor, expressed in new x grid steps.\n\n low_counts_threshold: float or int\n - y values with counts less than this value will be set to NaN\n\n Returns:\n tuple sum_ys, count_ys where\n xs - array with new x grid\n ys - array of EMA of y at each point of the new x grid\n count_ys - array of EMA of y counts at each point of the new x grid\n\n '''\n\n low = xolds[0] if low is None else low\n high = xolds[-1] if high is None else high\n\n assert xolds[0] <= low, 'low = {} < xolds[0] = {} - extrapolation not permitted!'.format(low, xolds[0])\n assert xolds[-1] >= high, 'high = {} > xolds[-1] = {} - extrapolation not permitted!'.format(high, xolds[-1])\n assert len(xolds) == len(yolds), 'length of xolds ({}) and yolds ({}) do not match!'.format(len(xolds), len(yolds))\n\n\n xolds = xolds.astype('float64')\n yolds = yolds.astype('float64')\n\n luoi = 0 # last unused old index\n sum_y = 0.\n count_y = 0.\n xnews = np.linspace(low, high, n)\n decay_period = (high - low) / (n - 1) * decay_steps\n interstep_decay = np.exp(- 1. / decay_steps)\n sum_ys = np.zeros_like(xnews)\n count_ys = np.zeros_like(xnews)\n for i in range(n):\n xnew = xnews[i]\n sum_y *= interstep_decay\n count_y *= interstep_decay\n while True:\n xold = xolds[luoi]\n if xold <= xnew:\n decay = np.exp(- (xnew - xold) / decay_period)\n sum_y += decay * yolds[luoi]\n count_y += decay\n luoi += 1\n else:\n break\n if luoi >= len(xolds):\n break\n sum_ys[i] = sum_y\n count_ys[i] = count_y\n\n ys = sum_ys / count_ys\n ys[count_ys < low_counts_threshold] = np.nan\n\n return xnews, ys, count_ys"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef symmetric_ema(xolds, yolds, low=None, high=None, n=512, decay_steps=1., low_counts_threshold=1e-8):\n '''\n perform symmetric EMA (exponential moving average)\n smoothing and resampling to an even grid with n points.\n Does not do extrapolation, so we assume\n xolds[0] <= low && high <= xolds[-1]\n\n Arguments:\n\n xolds: array or list - x values of data. Needs to be sorted in ascending order\n yolds: array of list - y values of data. Has to have the same length as xolds\n\n low: float - min value of the new x grid. By default equals to xolds[0]\n high: float - max value of the new x grid. By default equals to xolds[-1]\n\n n: int - number of points in new x grid\n\n decay_steps: float - EMA decay factor, expressed in new x grid steps.\n\n low_counts_threshold: float or int\n - y values with counts less than this value will be set to NaN\n\n Returns:\n tuple sum_ys, count_ys where\n xs - array with new x grid\n ys - array of EMA of y at each point of the new x grid\n count_ys - array of EMA of y counts at each point of the new x grid\n\n '''\n xs, ys1, count_ys1 = one_sided_ema(xolds, yolds, low, high, n, decay_steps, low_counts_threshold=0)\n _, ys2, count_ys2 = one_sided_ema(-xolds[::-1], yolds[::-1], -high, -low, n, decay_steps, low_counts_threshold=0)\n ys2 = ys2[::-1]\n count_ys2 = count_ys2[::-1]\n count_ys = count_ys1 + count_ys2\n ys = (ys1 * count_ys1 + ys2 * count_ys2) / count_ys\n ys[count_ys < low_counts_threshold] = np.nan\n return xs, ys, count_ys", "response": "Perform a symmetric EMA on the current language."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_results(root_dir_or_dirs, enable_progress=True, enable_monitor=True, verbose=False):\n '''\n load summaries of runs from a list of directories (including subdirectories)\n Arguments:\n\n enable_progress: bool - if True, will attempt to load data from progress.csv files (data saved by logger). Default: True\n\n enable_monitor: bool - if True, will attempt to load data from monitor.csv files (data saved by Monitor environment wrapper). Default: True\n\n verbose: bool - if True, will print out list of directories from which the data is loaded. Default: False\n\n\n Returns:\n List of Result objects with the following fields:\n - dirname - path to the directory data was loaded from\n - metadata - run metadata (such as command-line arguments and anything else in metadata.json file\n - monitor - if enable_monitor is True, this field contains pandas dataframe with loaded monitor.csv file (or aggregate of all *.monitor.csv files in the directory)\n - progress - if enable_progress is True, this field contains pandas dataframe with loaded progress.csv file\n '''\n import re\n if isinstance(root_dir_or_dirs, str):\n rootdirs = [osp.expanduser(root_dir_or_dirs)]\n else:\n rootdirs = [osp.expanduser(d) for d in root_dir_or_dirs]\n allresults = []\n for rootdir in rootdirs:\n assert osp.exists(rootdir), \"%s doesn't exist\"%rootdir\n for dirname, dirs, files in os.walk(rootdir):\n if '-proc' in dirname:\n files[:] = []\n continue\n monitor_re = re.compile(r'(\\d+\\.)?(\\d+\\.)?monitor\\.csv')\n if set(['metadata.json', 'monitor.json', 'progress.json', 'progress.csv']).intersection(files) or \\\n any([f for f in files if monitor_re.match(f)]): # also match monitor files like 0.1.monitor.csv\n # used to be uncommented, which means do not go deeper than current directory if any of the data files\n # are found\n # dirs[:] = []\n result = {'dirname' : dirname}\n if \"metadata.json\" in files:\n with open(osp.join(dirname, \"metadata.json\"), \"r\") as fh:\n result['metadata'] = json.load(fh)\n progjson = osp.join(dirname, \"progress.json\")\n progcsv = osp.join(dirname, \"progress.csv\")\n if enable_progress:\n if osp.exists(progjson):\n result['progress'] = pandas.DataFrame(read_json(progjson))\n elif osp.exists(progcsv):\n try:\n result['progress'] = read_csv(progcsv)\n except pandas.errors.EmptyDataError:\n print('skipping progress file in ', dirname, 'empty data')\n else:\n if verbose: print('skipping %s: no progress file'%dirname)\n\n if enable_monitor:\n try:\n result['monitor'] = pandas.DataFrame(monitor.load_results(dirname))\n except monitor.LoadMonitorResultsError:\n print('skipping %s: no monitor files'%dirname)\n except Exception as e:\n print('exception loading monitor file in %s: %s'%(dirname, e))\n\n if result.get('monitor') is not None or result.get('progress') is not None:\n allresults.append(Result(**result))\n if verbose:\n print('successfully loaded %s'%dirname)\n\n if verbose: print('loaded %i results'%len(allresults))\n return allresults", "response": "Load all the results from a directory."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nplot multiple results objects in a single Seaborn language.", "response": "def plot_results(\n allresults, *,\n xy_fn=default_xy_fn,\n split_fn=default_split_fn,\n group_fn=default_split_fn,\n average_group=False,\n shaded_std=True,\n shaded_err=True,\n figsize=None,\n legend_outside=False,\n resample=0,\n smooth_step=1.0\n):\n '''\n Plot multiple Results objects\n\n xy_fn: function Result -> x,y - function that converts results objects into tuple of x and y values.\n By default, x is cumsum of episode lengths, and y is episode rewards\n\n split_fn: function Result -> hashable - function that converts results objects into keys to split curves into sub-panels by.\n That is, the results r for which split_fn(r) is different will be put on different sub-panels.\n By default, the portion of r.dirname between last / and - is returned. The sub-panels are\n stacked vertically in the figure.\n\n group_fn: function Result -> hashable - function that converts results objects into keys to group curves by.\n That is, the results r for which group_fn(r) is the same will be put into the same group.\n Curves in the same group have the same color (if average_group is False), or averaged over\n (if average_group is True). The default value is the same as default value for split_fn\n\n average_group: bool - if True, will average the curves in the same group and plot the mean. Enables resampling\n (if resample = 0, will use 512 steps)\n\n shaded_std: bool - if True (default), the shaded region corresponding to standard deviation of the group of curves will be\n shown (only applicable if average_group = True)\n\n shaded_err: bool - if True (default), the shaded region corresponding to error in mean estimate of the group of curves\n (that is, standard deviation divided by square root of number of curves) will be\n shown (only applicable if average_group = True)\n\n figsize: tuple or None - size of the resulting figure (including sub-panels). By default, width is 6 and height is 6 times number of\n sub-panels.\n\n\n legend_outside: bool - if True, will place the legend outside of the sub-panels.\n\n resample: int - if not zero, size of the uniform grid in x direction to resample onto. Resampling is performed via symmetric\n EMA smoothing (see the docstring for symmetric_ema).\n Default is zero (no resampling). Note that if average_group is True, resampling is necessary; in that case, default\n value is 512.\n\n smooth_step: float - when resampling (i.e. when resample > 0 or average_group is True), use this EMA decay parameter (in units of the new grid step).\n See docstrings for decay_steps in symmetric_ema or one_sided_ema functions.\n\n '''\n\n if split_fn is None: split_fn = lambda _ : ''\n if group_fn is None: group_fn = lambda _ : ''\n sk2r = defaultdict(list) # splitkey2results\n for result in allresults:\n splitkey = split_fn(result)\n sk2r[splitkey].append(result)\n assert len(sk2r) > 0\n assert isinstance(resample, int), \"0: don't resample. : that many samples\"\n nrows = len(sk2r)\n ncols = 1\n figsize = figsize or (6, 6 * nrows)\n f, axarr = plt.subplots(nrows, ncols, sharex=False, squeeze=False, figsize=figsize)\n\n groups = list(set(group_fn(result) for result in allresults))\n\n default_samples = 512\n if average_group:\n resample = resample or default_samples\n\n for (isplit, sk) in enumerate(sorted(sk2r.keys())):\n g2l = {}\n g2c = defaultdict(int)\n sresults = sk2r[sk]\n gresults = defaultdict(list)\n ax = axarr[isplit][0]\n for result in sresults:\n group = group_fn(result)\n g2c[group] += 1\n x, y = xy_fn(result)\n if x is None: x = np.arange(len(y))\n x, y = map(np.asarray, (x, y))\n if average_group:\n gresults[group].append((x,y))\n else:\n if resample:\n x, y, counts = symmetric_ema(x, y, x[0], x[-1], resample, decay_steps=smooth_step)\n l, = ax.plot(x, y, color=COLORS[groups.index(group) % len(COLORS)])\n g2l[group] = l\n if average_group:\n for group in sorted(groups):\n xys = gresults[group]\n if not any(xys):\n continue\n color = COLORS[groups.index(group) % len(COLORS)]\n origxs = [xy[0] for xy in xys]\n minxlen = min(map(len, origxs))\n def allequal(qs):\n return all((q==qs[0]).all() for q in qs[1:])\n if resample:\n low = max(x[0] for x in origxs)\n high = min(x[-1] for x in origxs)\n usex = np.linspace(low, high, resample)\n ys = []\n for (x, y) in xys:\n ys.append(symmetric_ema(x, y, low, high, resample, decay_steps=smooth_step)[1])\n else:\n assert allequal([x[:minxlen] for x in origxs]),\\\n 'If you want to average unevenly sampled data, set resample='\n usex = origxs[0]\n ys = [xy[1][:minxlen] for xy in xys]\n ymean = np.mean(ys, axis=0)\n ystd = np.std(ys, axis=0)\n ystderr = ystd / np.sqrt(len(ys))\n l, = axarr[isplit][0].plot(usex, ymean, color=color)\n g2l[group] = l\n if shaded_err:\n ax.fill_between(usex, ymean - ystderr, ymean + ystderr, color=color, alpha=.4)\n if shaded_std:\n ax.fill_between(usex, ymean - ystd, ymean + ystd, color=color, alpha=.2)\n\n\n # https://matplotlib.org/users/legend_guide.html\n plt.tight_layout()\n if any(g2l.keys()):\n ax.legend(\n g2l.values(),\n ['%s (%i)'%(g, g2c[g]) for g in g2l] if average_group else g2l.keys(),\n loc=2 if legend_outside else None,\n bbox_to_anchor=(1,1) if legend_outside else None)\n ax.set_title(sk)\n return f, axarr"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check_synced(localval, comm=None):\n comm = comm or MPI.COMM_WORLD\n vals = comm.gather(localval)\n if comm.rank == 0:\n assert all(val==vals[0] for val in vals[1:])", "response": "Checks that all local variables on all workers are the same value"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef copy_obs_dict(obs):\n return {k: np.copy(v) for k, v in obs.items()}", "response": "Deep - copy an observation dict."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef obs_space_info(obs_space):\n if isinstance(obs_space, gym.spaces.Dict):\n assert isinstance(obs_space.spaces, OrderedDict)\n subspaces = obs_space.spaces\n else:\n subspaces = {None: obs_space}\n keys = []\n shapes = {}\n dtypes = {}\n for key, box in subspaces.items():\n keys.append(key)\n shapes[key] = box.shape\n dtypes[key] = box.dtype\n return keys, shapes, dtypes", "response": "Get dict - structured information about a gym. Space object."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ntrains an agent with given network architecture on a given environment.", "response": "def learn(network, env, seed=None, nsteps=20, total_timesteps=int(80e6), q_coef=0.5, ent_coef=0.01,\n max_grad_norm=10, lr=7e-4, lrschedule='linear', rprop_epsilon=1e-5, rprop_alpha=0.99, gamma=0.99,\n log_interval=100, buffer_size=50000, replay_ratio=4, replay_start=10000, c=10.0,\n trust_region=True, alpha=0.99, delta=1, load_path=None, **network_kwargs):\n\n '''\n Main entrypoint for ACER (Actor-Critic with Experience Replay) algorithm (https://arxiv.org/pdf/1611.01224.pdf)\n Train an agent with given network architecture on a given environment using ACER.\n\n Parameters:\n ----------\n\n network: policy network architecture. Either string (mlp, lstm, lnlstm, cnn_lstm, cnn, cnn_small, conv_only - see baselines.common/models.py for full list)\n specifying the standard network architecture, or a function that takes tensorflow tensor as input and returns\n tuple (output_tensor, extra_feed) where output tensor is the last network layer output, extra_feed is None for feed-forward\n neural nets, and extra_feed is a dictionary describing how to feed state into the network for recurrent neural nets.\n See baselines.common/policies.py/lstm for more details on using recurrent nets in policies\n\n env: environment. Needs to be vectorized for parallel environment simulation.\n The environments produced by gym.make can be wrapped using baselines.common.vec_env.DummyVecEnv class.\n\n nsteps: int, number of steps of the vectorized environment per update (i.e. batch size is nsteps * nenv where\n nenv is number of environment copies simulated in parallel) (default: 20)\n\n nstack: int, size of the frame stack, i.e. number of the frames passed to the step model. Frames are stacked along channel dimension\n (last image dimension) (default: 4)\n\n total_timesteps: int, number of timesteps (i.e. number of actions taken in the environment) (default: 80M)\n\n q_coef: float, value function loss coefficient in the optimization objective (analog of vf_coef for other actor-critic methods)\n\n ent_coef: float, policy entropy coefficient in the optimization objective (default: 0.01)\n\n max_grad_norm: float, gradient norm clipping coefficient. If set to None, no clipping. (default: 10),\n\n lr: float, learning rate for RMSProp (current implementation has RMSProp hardcoded in) (default: 7e-4)\n\n lrschedule: schedule of learning rate. Can be 'linear', 'constant', or a function [0..1] -> [0..1] that takes fraction of the training progress as input and\n returns fraction of the learning rate (specified as lr) as output\n\n rprop_epsilon: float, RMSProp epsilon (stabilizes square root computation in denominator of RMSProp update) (default: 1e-5)\n\n rprop_alpha: float, RMSProp decay parameter (default: 0.99)\n\n gamma: float, reward discounting factor (default: 0.99)\n\n log_interval: int, number of updates between logging events (default: 100)\n\n buffer_size: int, size of the replay buffer (default: 50k)\n\n replay_ratio: int, now many (on average) batches of data to sample from the replay buffer take after batch from the environment (default: 4)\n\n replay_start: int, the sampling from the replay buffer does not start until replay buffer has at least that many samples (default: 10k)\n\n c: float, importance weight clipping factor (default: 10)\n\n trust_region bool, whether or not algorithms estimates the gradient KL divergence between the old and updated policy and uses it to determine step size (default: True)\n\n delta: float, max KL divergence between the old policy and updated policy (default: 1)\n\n alpha: float, momentum factor in the Polyak (exponential moving average) averaging of the model parameters (default: 0.99)\n\n load_path: str, path to load the model from (default: None)\n\n **network_kwargs: keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network\n For instance, 'mlp' network architecture has arguments num_hidden and num_layers.\n\n '''\n\n print(\"Running Acer Simple\")\n print(locals())\n set_global_seeds(seed)\n if not isinstance(env, VecFrameStack):\n env = VecFrameStack(env, 1)\n\n policy = build_policy(env, network, estimate_q=True, **network_kwargs)\n nenvs = env.num_envs\n ob_space = env.observation_space\n ac_space = env.action_space\n\n nstack = env.nstack\n model = Model(policy=policy, ob_space=ob_space, ac_space=ac_space, nenvs=nenvs, nsteps=nsteps,\n ent_coef=ent_coef, q_coef=q_coef, gamma=gamma,\n max_grad_norm=max_grad_norm, lr=lr, rprop_alpha=rprop_alpha, rprop_epsilon=rprop_epsilon,\n total_timesteps=total_timesteps, lrschedule=lrschedule, c=c,\n trust_region=trust_region, alpha=alpha, delta=delta)\n\n runner = Runner(env=env, model=model, nsteps=nsteps)\n if replay_ratio > 0:\n buffer = Buffer(env=env, nsteps=nsteps, size=buffer_size)\n else:\n buffer = None\n nbatch = nenvs*nsteps\n acer = Acer(runner, model, buffer, log_interval)\n acer.tstart = time.time()\n\n for acer.steps in range(0, total_timesteps, nbatch): #nbatch samples, 1 on_policy call and multiple off-policy calls\n acer.call(on_policy=True)\n if replay_ratio > 0 and buffer.has_atleast(replay_start):\n n = np.random.poisson(replay_ratio)\n for _ in range(n):\n acer.call(on_policy=False) # no simulation steps in this\n\n return model"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef apply_stats(self, statsUpdates):\n\n def updateAccumStats():\n if self._full_stats_init:\n return tf.cond(tf.greater(self.sgd_step, self._cold_iter), lambda: tf.group(*self._apply_stats(statsUpdates, accumulate=True, accumulateCoeff=1. / self._stats_accum_iter)), tf.no_op)\n else:\n return tf.group(*self._apply_stats(statsUpdates, accumulate=True, accumulateCoeff=1. / self._stats_accum_iter))\n\n def updateRunningAvgStats(statsUpdates, fac_iter=1):\n # return tf.cond(tf.greater_equal(self.factor_step,\n # tf.convert_to_tensor(fac_iter)), lambda:\n # tf.group(*self._apply_stats(stats_list, varlist)), tf.no_op)\n return tf.group(*self._apply_stats(statsUpdates))\n\n if self._async_stats:\n # asynchronous stats update\n update_stats = self._apply_stats(statsUpdates)\n\n queue = tf.FIFOQueue(1, [item.dtype for item in update_stats], shapes=[\n item.get_shape() for item in update_stats])\n enqueue_op = queue.enqueue(update_stats)\n\n def dequeue_stats_op():\n return queue.dequeue()\n self.qr_stats = tf.train.QueueRunner(queue, [enqueue_op])\n update_stats_op = tf.cond(tf.equal(queue.size(), tf.convert_to_tensor(\n 0)), tf.no_op, lambda: tf.group(*[dequeue_stats_op(), ]))\n else:\n # synchronous stats update\n update_stats_op = tf.cond(tf.greater_equal(\n self.stats_step, self._stats_accum_iter), lambda: updateRunningAvgStats(statsUpdates), updateAccumStats)\n self._update_stats_op = update_stats_op\n return update_stats_op", "response": "compute stats and update the new stats to the running average\n "} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntiles N images into one big PxQ image", "response": "def tile_images(img_nhwc):\n \"\"\"\n Tile N images into one big PxQ image\n (P,Q) are chosen to be as close as possible, and if N\n is square, then P=Q.\n\n input: img_nhwc, list or array of images, ndim=4 once turned into array\n n = batch index, h = height, w = width, c = channel\n returns:\n bigim_HWc, ndarray with ndim=3\n \"\"\"\n img_nhwc = np.asarray(img_nhwc)\n N, h, w, c = img_nhwc.shape\n H = int(np.ceil(np.sqrt(N)))\n W = int(np.ceil(float(N)/H))\n img_nhwc = np.array(list(img_nhwc) + [img_nhwc[0]*0 for _ in range(N, H*W)])\n img_HWhwc = img_nhwc.reshape(H, W, h, w, c)\n img_HhWwc = img_HWhwc.transpose(0, 2, 1, 3, 4)\n img_Hh_Ww_c = img_HhWwc.reshape(H*h, W*w, c)\n return img_Hh_Ww_c"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sum(self, start=0, end=None):\n return super(SumSegmentTree, self).reduce(start, end)", "response": "Returns the sum of the segments in the tree."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfinding the index i in the array that is greater than or equal to the given prefixsum.", "response": "def find_prefixsum_idx(self, prefixsum):\n \"\"\"Find the highest index `i` in the array such that\n sum(arr[0] + arr[1] + ... + arr[i - i]) <= prefixsum\n\n if array values are probabilities, this function\n allows to sample indexes according to the discrete\n probability efficiently.\n\n Parameters\n ----------\n perfixsum: float\n upperbound on the sum of array prefix\n\n Returns\n -------\n idx: int\n highest index satisfying the prefixsum constraint\n \"\"\"\n assert 0 <= prefixsum <= self.sum() + 1e-5\n idx = 1\n while idx < self._capacity: # while non-leaf\n if self._value[2 * idx] > prefixsum:\n idx = 2 * idx\n else:\n prefixsum -= self._value[2 * idx]\n idx = 2 * idx + 1\n return idx - self._capacity"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the minimum entry in the tree.", "response": "def min(self, start=0, end=None):\n \"\"\"Returns min(arr[start], ..., arr[end])\"\"\"\n\n return super(MinSegmentTree, self).reduce(start, end)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _subproc_worker(pipe, parent_pipe, env_fn_wrapper, obs_bufs, obs_shapes, obs_dtypes, keys):\n def _write_obs(maybe_dict_obs):\n flatdict = obs_to_dict(maybe_dict_obs)\n for k in keys:\n dst = obs_bufs[k].get_obj()\n dst_np = np.frombuffer(dst, dtype=obs_dtypes[k]).reshape(obs_shapes[k]) # pylint: disable=W0212\n np.copyto(dst_np, flatdict[k])\n\n env = env_fn_wrapper.x()\n parent_pipe.close()\n try:\n while True:\n cmd, data = pipe.recv()\n if cmd == 'reset':\n pipe.send(_write_obs(env.reset()))\n elif cmd == 'step':\n obs, reward, done, info = env.step(data)\n if done:\n obs = env.reset()\n pipe.send((_write_obs(obs), reward, done, info))\n elif cmd == 'render':\n pipe.send(env.render(mode='rgb_array'))\n elif cmd == 'close':\n pipe.send(None)\n break\n else:\n raise RuntimeError('Got unrecognized cmd %s' % cmd)\n except KeyboardInterrupt:\n print('ShmemVecEnv worker: got KeyboardInterrupt')\n finally:\n env.close()", "response": "This function is a worker function that handles the processing of a single environment instance."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntraining a policy with given network architecture on a given environment.", "response": "def learn(\n network,\n env,\n seed=None,\n nsteps=5,\n total_timesteps=int(80e6),\n vf_coef=0.5,\n ent_coef=0.01,\n max_grad_norm=0.5,\n lr=7e-4,\n lrschedule='linear',\n epsilon=1e-5,\n alpha=0.99,\n gamma=0.99,\n log_interval=100,\n load_path=None,\n **network_kwargs):\n\n '''\n Main entrypoint for A2C algorithm. Train a policy with given network architecture on a given environment using a2c algorithm.\n\n Parameters:\n -----------\n\n network: policy network architecture. Either string (mlp, lstm, lnlstm, cnn_lstm, cnn, cnn_small, conv_only - see baselines.common/models.py for full list)\n specifying the standard network architecture, or a function that takes tensorflow tensor as input and returns\n tuple (output_tensor, extra_feed) where output tensor is the last network layer output, extra_feed is None for feed-forward\n neural nets, and extra_feed is a dictionary describing how to feed state into the network for recurrent neural nets.\n See baselines.common/policies.py/lstm for more details on using recurrent nets in policies\n\n\n env: RL environment. Should implement interface similar to VecEnv (baselines.common/vec_env) or be wrapped with DummyVecEnv (baselines.common/vec_env/dummy_vec_env.py)\n\n\n seed: seed to make random number sequence in the alorightm reproducible. By default is None which means seed from system noise generator (not reproducible)\n\n nsteps: int, number of steps of the vectorized environment per update (i.e. batch size is nsteps * nenv where\n nenv is number of environment copies simulated in parallel)\n\n total_timesteps: int, total number of timesteps to train on (default: 80M)\n\n vf_coef: float, coefficient in front of value function loss in the total loss function (default: 0.5)\n\n ent_coef: float, coeffictiant in front of the policy entropy in the total loss function (default: 0.01)\n\n max_gradient_norm: float, gradient is clipped to have global L2 norm no more than this value (default: 0.5)\n\n lr: float, learning rate for RMSProp (current implementation has RMSProp hardcoded in) (default: 7e-4)\n\n lrschedule: schedule of learning rate. Can be 'linear', 'constant', or a function [0..1] -> [0..1] that takes fraction of the training progress as input and\n returns fraction of the learning rate (specified as lr) as output\n\n epsilon: float, RMSProp epsilon (stabilizes square root computation in denominator of RMSProp update) (default: 1e-5)\n\n alpha: float, RMSProp decay parameter (default: 0.99)\n\n gamma: float, reward discounting parameter (default: 0.99)\n\n log_interval: int, specifies how frequently the logs are printed out (default: 100)\n\n **network_kwargs: keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network\n For instance, 'mlp' network architecture has arguments num_hidden and num_layers.\n\n '''\n\n\n\n set_global_seeds(seed)\n\n # Get the nb of env\n nenvs = env.num_envs\n policy = build_policy(env, network, **network_kwargs)\n\n # Instantiate the model object (that creates step_model and train_model)\n model = Model(policy=policy, env=env, nsteps=nsteps, ent_coef=ent_coef, vf_coef=vf_coef,\n max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps, lrschedule=lrschedule)\n if load_path is not None:\n model.load(load_path)\n\n # Instantiate the runner object\n runner = Runner(env, model, nsteps=nsteps, gamma=gamma)\n epinfobuf = deque(maxlen=100)\n\n # Calculate the batch_size\n nbatch = nenvs*nsteps\n\n # Start total timer\n tstart = time.time()\n\n for update in range(1, total_timesteps//nbatch+1):\n # Get mini batch of experiences\n obs, states, rewards, masks, actions, values, epinfos = runner.run()\n epinfobuf.extend(epinfos)\n\n policy_loss, value_loss, policy_entropy = model.train(obs, states, rewards, masks, actions, values)\n nseconds = time.time()-tstart\n\n # Calculate the fps (frame per second)\n fps = int((update*nbatch)/nseconds)\n if update % log_interval == 0 or update == 1:\n # Calculates if value function is a good predicator of the returns (ev > 1)\n # or if it's just worse than predicting nothing (ev =< 0)\n ev = explained_variance(values, rewards)\n logger.record_tabular(\"nupdates\", update)\n logger.record_tabular(\"total_timesteps\", update*nbatch)\n logger.record_tabular(\"fps\", fps)\n logger.record_tabular(\"policy_entropy\", float(policy_entropy))\n logger.record_tabular(\"value_loss\", float(value_loss))\n logger.record_tabular(\"explained_variance\", float(ev))\n logger.record_tabular(\"eprewmean\", safemean([epinfo['r'] for epinfo in epinfobuf]))\n logger.record_tabular(\"eplenmean\", safemean([epinfo['l'] for epinfo in epinfobuf]))\n logger.dump_tabular()\n return model"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nswap and then flatten axes 0 and 1", "response": "def sf01(arr):\n \"\"\"\n swap and then flatten axes 0 and 1\n \"\"\"\n s = arr.shape\n return arr.swapaxes(0, 1).reshape(s[0] * s[1], *s[2:])"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncompute next action given the observation data.", "response": "def step(self, observation, **extra_feed):\n \"\"\"\n Compute next action(s) given the observation(s)\n\n Parameters:\n ----------\n\n observation observation data (either single or a batch)\n\n **extra_feed additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__)\n\n Returns:\n -------\n (action, value estimate, next state, negative log likelihood of the action under current policy parameters) tuple\n \"\"\"\n\n a, v, state, neglogp = self._evaluate([self.action, self.vf, self.state, self.neglogp], observation, **extra_feed)\n if state.size == 0:\n state = None\n return a, v, state, neglogp"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef value(self, ob, *args, **kwargs):\n return self._evaluate(self.vf, ob, *args, **kwargs)", "response": "Compute the value estimate of the current value for the given observation"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nprinting the number of seconds in human readable format.", "response": "def pretty_eta(seconds_left):\n \"\"\"Print the number of seconds in human readable format.\n\n Examples:\n 2 days\n 2 hours and 37 minutes\n less than a minute\n\n Paramters\n ---------\n seconds_left: int\n Number of seconds to be converted to the ETA\n Returns\n -------\n eta: str\n String representing the pretty ETA.\n \"\"\"\n minutes_left = seconds_left // 60\n seconds_left %= 60\n hours_left = minutes_left // 60\n minutes_left %= 60\n days_left = hours_left // 24\n hours_left %= 24\n\n def helper(cnt, name):\n return \"{} {}{}\".format(str(cnt), name, ('s' if cnt > 1 else ''))\n\n if days_left > 0:\n msg = helper(days_left, 'day')\n if hours_left > 0:\n msg += ' and ' + helper(hours_left, 'hour')\n return msg\n if hours_left > 0:\n msg = helper(hours_left, 'hour')\n if minutes_left > 0:\n msg += ' and ' + helper(minutes_left, 'minute')\n return msg\n if minutes_left > 0:\n return helper(minutes_left, 'minute')\n return 'less than a minute'"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef boolean_flag(parser, name, default=False, help=None):\n dest = name.replace('-', '_')\n parser.add_argument(\"--\" + name, action=\"store_true\", default=default, dest=dest, help=help)\n parser.add_argument(\"--no-\" + name, action=\"store_false\", dest=dest)", "response": "Adds a boolean flag to argparse parser."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_wrapper_by_name(env, classname):\n currentenv = env\n while True:\n if classname == currentenv.class_name():\n return currentenv\n elif isinstance(currentenv, gym.Wrapper):\n currentenv = currentenv.env\n else:\n raise ValueError(\"Couldn't find wrapper named %s\" % classname)", "response": "Given an environment possibly wrapped multiple times returns a wrapper of class classname"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef relatively_safe_pickle_dump(obj, path, compression=False):\n temp_storage = path + \".relatively_safe\"\n if compression:\n # Using gzip here would be simpler, but the size is limited to 2GB\n with tempfile.NamedTemporaryFile() as uncompressed_file:\n pickle.dump(obj, uncompressed_file)\n uncompressed_file.file.flush()\n with zipfile.ZipFile(temp_storage, \"w\", compression=zipfile.ZIP_DEFLATED) as myzip:\n myzip.write(uncompressed_file.name, \"data\")\n else:\n with open(temp_storage, \"wb\") as f:\n pickle.dump(obj, f)\n os.rename(temp_storage, path)", "response": "This function is just like regular pickle dump except that it will not overwrite the original file."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nupdate the estimate. Parameters ---------- new_val: float new observated value of estimated quantity.", "response": "def update(self, new_val):\n \"\"\"Update the estimate.\n\n Parameters\n ----------\n new_val: float\n new observated value of estimated quantity.\n \"\"\"\n if self._value is None:\n self._value = new_val\n else:\n self._value = self._gamma * self._value + (1.0 - self._gamma) * new_val"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstore provided method args as instance attributes.", "response": "def store_args(method):\n \"\"\"Stores provided method args as instance attributes.\n \"\"\"\n argspec = inspect.getfullargspec(method)\n defaults = {}\n if argspec.defaults is not None:\n defaults = dict(\n zip(argspec.args[-len(argspec.defaults):], argspec.defaults))\n if argspec.kwonlydefaults is not None:\n defaults.update(argspec.kwonlydefaults)\n arg_names = argspec.args[1:]\n\n @functools.wraps(method)\n def wrapper(*positional_args, **keyword_args):\n self = positional_args[0]\n # Get default arg values\n args = defaults.copy()\n # Add provided arg values\n for name, value in zip(arg_names, positional_args[1:]):\n args[name] = value\n args.update(keyword_args)\n self.__dict__.update(args)\n return method(*positional_args, **keyword_args)\n\n return wrapper"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef import_function(spec):\n mod_name, fn_name = spec.split(':')\n module = importlib.import_module(mod_name)\n fn = getattr(module, fn_name)\n return fn", "response": "Import a function identified by a string like pkg. module : fn_name."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef flatten_grads(var_list, grads):\n return tf.concat([tf.reshape(grad, [U.numel(v)])\n for (v, grad) in zip(var_list, grads)], 0)", "response": "Flattens a variables and their gradients."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef nn(input, layers_sizes, reuse=None, flatten=False, name=\"\"):\n for i, size in enumerate(layers_sizes):\n activation = tf.nn.relu if i < len(layers_sizes) - 1 else None\n input = tf.layers.dense(inputs=input,\n units=size,\n kernel_initializer=tf.contrib.layers.xavier_initializer(),\n reuse=reuse,\n name=name + '_' + str(i))\n if activation:\n input = activation(input)\n if flatten:\n assert layers_sizes[-1] == 1\n input = tf.reshape(input, [-1])\n return input", "response": "Creates a simple neural network with a given number of layers."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts an episode to have the batch dimension in the major ( first ) dimension.", "response": "def convert_episode_to_batch_major(episode):\n \"\"\"Converts an episode to have the batch dimension in the major (first)\n dimension.\n \"\"\"\n episode_batch = {}\n for key in episode.keys():\n val = np.array(episode[key]).copy()\n # make inputs batch-major instead of time-major\n episode_batch[key] = val.swapaxes(0, 1)\n\n return episode_batch"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_vtarg_and_adv(seg, gamma, lam):\n new = np.append(seg[\"new\"], 0) # last element is only used for last vtarg, but we already zeroed it if last new = 1\n vpred = np.append(seg[\"vpred\"], seg[\"nextvpred\"])\n T = len(seg[\"rew\"])\n seg[\"adv\"] = gaelam = np.empty(T, 'float32')\n rew = seg[\"rew\"]\n lastgaelam = 0\n for t in reversed(range(T)):\n nonterminal = 1-new[t+1]\n delta = rew[t] + gamma * vpred[t+1] * nonterminal - vpred[t]\n gaelam[t] = lastgaelam = delta + gamma * lam * nonterminal * lastgaelam\n seg[\"tdlamret\"] = seg[\"adv\"] + seg[\"vpred\"]", "response": "Compute target value using TD lambda and advantage with GAE lambda"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nswitching between two operations depending on a scalar value.", "response": "def switch(condition, then_expression, else_expression):\n \"\"\"Switches between two operations depending on a scalar value (int or bool).\n Note that both `then_expression` and `else_expression`\n should be symbolic tensors of the *same shape*.\n\n # Arguments\n condition: scalar tensor.\n then_expression: TensorFlow operation.\n else_expression: TensorFlow operation.\n \"\"\"\n x_shape = copy.copy(then_expression.get_shape())\n x = tf.cond(tf.cast(condition, 'bool'),\n lambda: then_expression,\n lambda: else_expression)\n x.set_shape(x_shape)\n return x"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreference : https://en. wikicode. org / wiki / Huber_loss", "response": "def huber_loss(x, delta=1.0):\n \"\"\"Reference: https://en.wikipedia.org/wiki/Huber_loss\"\"\"\n return tf.where(\n tf.abs(x) < delta,\n tf.square(x) * 0.5,\n delta * (tf.abs(x) - 0.5 * delta)\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets default session or create one with a given config", "response": "def get_session(config=None):\n \"\"\"Get default session or create one with a given config\"\"\"\n sess = tf.get_default_session()\n if sess is None:\n sess = make_session(config=config, make_default=True)\n return sess"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_session(config=None, num_cpu=None, make_default=False, graph=None):\n if num_cpu is None:\n num_cpu = int(os.getenv('RCALL_NUM_CPU', multiprocessing.cpu_count()))\n if config is None:\n config = tf.ConfigProto(\n allow_soft_placement=True,\n inter_op_parallelism_threads=num_cpu,\n intra_op_parallelism_threads=num_cpu)\n config.gpu_options.allow_growth = True\n\n if make_default:\n return tf.InteractiveSession(config=config, graph=graph)\n else:\n return tf.Session(config=config, graph=graph)", "response": "Returns a session that will use num_cpu CPU s only"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef initialize():\n new_variables = set(tf.global_variables()) - ALREADY_INITIALIZED\n get_session().run(tf.variables_initializer(new_variables))\n ALREADY_INITIALIZED.update(new_variables)", "response": "Initialize all the uninitialized variables in the global scope."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef adjust_shape(placeholder, data):\n '''\n adjust shape of the data to the shape of the placeholder if possible.\n If shape is incompatible, AssertionError is thrown\n\n Parameters:\n placeholder tensorflow input placeholder\n\n data input data to be (potentially) reshaped to be fed into placeholder\n\n Returns:\n reshaped data\n '''\n\n if not isinstance(data, np.ndarray) and not isinstance(data, list):\n return data\n if isinstance(data, list):\n data = np.array(data)\n\n placeholder_shape = [x or -1 for x in placeholder.shape.as_list()]\n\n assert _check_shape(placeholder_shape, data.shape), \\\n 'Shape of data {} is not compatible with shape of the placeholder {}'.format(data.shape, placeholder_shape)\n\n return np.reshape(data, placeholder_shape)", "response": "Adjust the shape of the data to the shape of the placeholder if possible."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if two shapes are compatible", "response": "def _check_shape(placeholder_shape, data_shape):\n ''' check if two shapes are compatible (i.e. differ only by dimensions of size 1, or by the batch dimension)'''\n\n return True\n squeezed_placeholder_shape = _squeeze_shape(placeholder_shape)\n squeezed_data_shape = _squeeze_shape(data_shape)\n\n for i, s_data in enumerate(squeezed_data_shape):\n s_placeholder = squeezed_placeholder_shape[i]\n if s_placeholder != -1 and s_data != s_placeholder:\n return False\n\n return True"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef profile(n):\n def decorator_with_name(func):\n def func_wrapper(*args, **kwargs):\n with profile_kv(n):\n return func(*args, **kwargs)\n return func_wrapper\n return decorator_with_name", "response": "Usage:\n @profile(\"my_func\")\n def my_func(): code"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconfiguring environment for DeepMind - style Atari.", "response": "def wrap_deepmind(env, episode_life=True, clip_rewards=True, frame_stack=False, scale=False):\n \"\"\"Configure environment for DeepMind-style Atari.\n \"\"\"\n if episode_life:\n env = EpisodicLifeEnv(env)\n if 'FIRE' in env.unwrapped.get_action_meanings():\n env = FireResetEnv(env)\n env = WarpFrame(env)\n if scale:\n env = ScaledFloatFrame(env)\n if clip_rewards:\n env = ClipRewardEnv(env)\n if frame_stack:\n env = FrameStack(env, 4)\n return env"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreset the state machine.", "response": "def reset(self, **kwargs):\n \"\"\"Reset only when lives are exhausted.\n This way all states are still reachable even though lives are episodic,\n and the learner need not know about any of this behind-the-scenes.\n \"\"\"\n if self.was_real_done:\n obs = self.env.reset(**kwargs)\n else:\n # no-op step to advance from terminal/lost life state\n obs, _, _, _ = self.env.step(0)\n self.lives = self.env.unwrapped.ale.lives()\n return obs"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsends the root node s parameters to every worker.", "response": "def sync_from_root(sess, variables, comm=None):\n \"\"\"\n Send the root node's parameters to every worker.\n Arguments:\n sess: the TensorFlow session.\n variables: all parameter variables including optimizer's\n \"\"\"\n if comm is None: comm = MPI.COMM_WORLD\n import tensorflow as tf\n values = comm.bcast(sess.run(variables))\n sess.run([tf.assign(var, val)\n for (var, val) in zip(variables, values)])"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncounting the number of GPUs on this machine.", "response": "def gpu_count():\n \"\"\"\n Count the GPUs on this machine.\n \"\"\"\n if shutil.which('nvidia-smi') is None:\n return 0\n output = subprocess.check_output(['nvidia-smi', '--query-gpu=gpu_name', '--format=csv'])\n return max(0, len(output.split(b'\\n')) - 2)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef setup_mpi_gpus():\n if 'CUDA_VISIBLE_DEVICES' not in os.environ:\n if sys.platform == 'darwin': # This Assumes if you're on OSX you're just\n ids = [] # doing a smoke test and don't want GPUs\n else:\n lrank, _lsize = get_local_rank_size(MPI.COMM_WORLD)\n ids = [lrank]\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = \",\".join(map(str, ids))", "response": "Set CUDA_VISIBLE_DEVICES to MPI rank if not already set"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the rank of each process on a given machine and the number of processes on that machine.", "response": "def get_local_rank_size(comm):\n \"\"\"\n Returns the rank of each process on its machine\n The processes on a given machine will be assigned ranks\n 0, 1, 2, ..., N-1,\n where N is the number of processes on this machine.\n\n Useful if you want to assign one gpu per machine\n \"\"\"\n this_node = platform.node()\n ranks_nodes = comm.allgather((comm.Get_rank(), this_node))\n node2rankssofar = defaultdict(int)\n local_rank = None\n for (rank, node) in ranks_nodes:\n if rank == comm.Get_rank():\n local_rank = node2rankssofar[node]\n node2rankssofar[node] += 1\n assert local_rank is not None\n return local_rank, node2rankssofar[this_node]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nshare the file with the specified path.", "response": "def share_file(comm, path):\n \"\"\"\n Copies the file from rank 0 to all other ranks\n Puts it in the same place on all machines\n \"\"\"\n localrank, _ = get_local_rank_size(comm)\n if comm.Get_rank() == 0:\n with open(path, 'rb') as fh:\n data = fh.read()\n comm.bcast(data)\n else:\n data = comm.bcast(None)\n if localrank == 0:\n os.makedirs(os.path.dirname(path), exist_ok=True)\n with open(path, 'wb') as fh:\n fh.write(data)\n comm.Barrier()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nperform a reduction operation over dicts d", "response": "def dict_gather(comm, d, op='mean', assert_all_have_data=True):\n \"\"\"\n Perform a reduction operation over dicts\n \"\"\"\n if comm is None: return d\n alldicts = comm.allgather(d)\n size = comm.size\n k2li = defaultdict(list)\n for d in alldicts:\n for (k,v) in d.items():\n k2li[k].append(v)\n result = {}\n for (k,li) in k2li.items():\n if assert_all_have_data:\n assert len(li)==size, \"only %i out of %i MPI workers have sent '%s'\" % (len(li), size, k)\n if op=='mean':\n result[k] = np.mean(li, axis=0)\n elif op=='sum':\n result[k] = np.sum(li, axis=0)\n else:\n assert 0, op\n return result"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef mpi_weighted_mean(comm, local_name2valcount):\n all_name2valcount = comm.gather(local_name2valcount)\n if comm.rank == 0:\n name2sum = defaultdict(float)\n name2count = defaultdict(float)\n for n2vc in all_name2valcount:\n for (name, (val, count)) in n2vc.items():\n try:\n val = float(val)\n except ValueError:\n if comm.rank == 0:\n warnings.warn('WARNING: tried to compute mean on non-float {}={}'.format(name, val))\n else:\n name2sum[name] += val * count\n name2count[name] += count\n return {name : name2sum[name] / name2count[name] for name in name2sum}\n else:\n return {}", "response": "Perform a weighted average over dicts that are each on a different node\n Output a dictionary that maps the names of the node to the mean of the nodes in the tree."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef learn(*,\n network,\n env,\n total_timesteps,\n timesteps_per_batch=1024, # what to train on\n max_kl=0.001,\n cg_iters=10,\n gamma=0.99,\n lam=1.0, # advantage estimation\n seed=None,\n ent_coef=0.0,\n cg_damping=1e-2,\n vf_stepsize=3e-4,\n vf_iters =3,\n max_episodes=0, max_iters=0, # time constraint\n callback=None,\n load_path=None,\n **network_kwargs\n ):\n '''\n learn a policy function with TRPO algorithm\n\n Parameters:\n ----------\n\n network neural network to learn. Can be either string ('mlp', 'cnn', 'lstm', 'lnlstm' for basic types)\n or function that takes input placeholder and returns tuple (output, None) for feedforward nets\n or (output, (state_placeholder, state_output, mask_placeholder)) for recurrent nets\n\n env environment (one of the gym environments or wrapped via baselines.common.vec_env.VecEnv-type class\n\n timesteps_per_batch timesteps per gradient estimation batch\n\n max_kl max KL divergence between old policy and new policy ( KL(pi_old || pi) )\n\n ent_coef coefficient of policy entropy term in the optimization objective\n\n cg_iters number of iterations of conjugate gradient algorithm\n\n cg_damping conjugate gradient damping\n\n vf_stepsize learning rate for adam optimizer used to optimie value function loss\n\n vf_iters number of iterations of value function optimization iterations per each policy optimization step\n\n total_timesteps max number of timesteps\n\n max_episodes max number of episodes\n\n max_iters maximum number of policy optimization iterations\n\n callback function to be called with (locals(), globals()) each policy optimization step\n\n load_path str, path to load the model from (default: None, i.e. no model is loaded)\n\n **network_kwargs keyword arguments to the policy / network builder. See baselines.common/policies.py/build_policy and arguments to a particular type of network\n\n Returns:\n -------\n\n learnt model\n\n '''\n\n if MPI is not None:\n nworkers = MPI.COMM_WORLD.Get_size()\n rank = MPI.COMM_WORLD.Get_rank()\n else:\n nworkers = 1\n rank = 0\n\n cpus_per_worker = 1\n U.get_session(config=tf.ConfigProto(\n allow_soft_placement=True,\n inter_op_parallelism_threads=cpus_per_worker,\n intra_op_parallelism_threads=cpus_per_worker\n ))\n\n\n policy = build_policy(env, network, value_network='copy', **network_kwargs)\n set_global_seeds(seed)\n\n np.set_printoptions(precision=3)\n # Setup losses and stuff\n # ----------------------------------------\n ob_space = env.observation_space\n ac_space = env.action_space\n\n ob = observation_placeholder(ob_space)\n with tf.variable_scope(\"pi\"):\n pi = policy(observ_placeholder=ob)\n with tf.variable_scope(\"oldpi\"):\n oldpi = policy(observ_placeholder=ob)\n\n atarg = tf.placeholder(dtype=tf.float32, shape=[None]) # Target advantage function (if applicable)\n ret = tf.placeholder(dtype=tf.float32, shape=[None]) # Empirical return\n\n ac = pi.pdtype.sample_placeholder([None])\n\n kloldnew = oldpi.pd.kl(pi.pd)\n ent = pi.pd.entropy()\n meankl = tf.reduce_mean(kloldnew)\n meanent = tf.reduce_mean(ent)\n entbonus = ent_coef * meanent\n\n vferr = tf.reduce_mean(tf.square(pi.vf - ret))\n\n ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac)) # advantage * pnew / pold\n surrgain = tf.reduce_mean(ratio * atarg)\n\n optimgain = surrgain + entbonus\n losses = [optimgain, meankl, entbonus, surrgain, meanent]\n loss_names = [\"optimgain\", \"meankl\", \"entloss\", \"surrgain\", \"entropy\"]\n\n dist = meankl\n\n all_var_list = get_trainable_variables(\"pi\")\n # var_list = [v for v in all_var_list if v.name.split(\"/\")[1].startswith(\"pol\")]\n # vf_var_list = [v for v in all_var_list if v.name.split(\"/\")[1].startswith(\"vf\")]\n var_list = get_pi_trainable_variables(\"pi\")\n vf_var_list = get_vf_trainable_variables(\"pi\")\n\n vfadam = MpiAdam(vf_var_list)\n\n get_flat = U.GetFlat(var_list)\n set_from_flat = U.SetFromFlat(var_list)\n klgrads = tf.gradients(dist, var_list)\n flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name=\"flat_tan\")\n shapes = [var.get_shape().as_list() for var in var_list]\n start = 0\n tangents = []\n for shape in shapes:\n sz = U.intprod(shape)\n tangents.append(tf.reshape(flat_tangent[start:start+sz], shape))\n start += sz\n gvp = tf.add_n([tf.reduce_sum(g*tangent) for (g, tangent) in zipsame(klgrads, tangents)]) #pylint: disable=E1111\n fvp = U.flatgrad(gvp, var_list)\n\n assign_old_eq_new = U.function([],[], updates=[tf.assign(oldv, newv)\n for (oldv, newv) in zipsame(get_variables(\"oldpi\"), get_variables(\"pi\"))])\n\n compute_losses = U.function([ob, ac, atarg], losses)\n compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])\n compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)\n compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))\n\n @contextmanager\n def timed(msg):\n if rank == 0:\n print(colorize(msg, color='magenta'))\n tstart = time.time()\n yield\n print(colorize(\"done in %.3f seconds\"%(time.time() - tstart), color='magenta'))\n else:\n yield\n\n def allmean(x):\n assert isinstance(x, np.ndarray)\n if MPI is not None:\n out = np.empty_like(x)\n MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)\n out /= nworkers\n else:\n out = np.copy(x)\n\n return out\n\n U.initialize()\n if load_path is not None:\n pi.load(load_path)\n\n th_init = get_flat()\n if MPI is not None:\n MPI.COMM_WORLD.Bcast(th_init, root=0)\n\n set_from_flat(th_init)\n vfadam.sync()\n print(\"Init param sum\", th_init.sum(), flush=True)\n\n # Prepare for rollouts\n # ----------------------------------------\n seg_gen = traj_segment_generator(pi, env, timesteps_per_batch, stochastic=True)\n\n episodes_so_far = 0\n timesteps_so_far = 0\n iters_so_far = 0\n tstart = time.time()\n lenbuffer = deque(maxlen=40) # rolling buffer for episode lengths\n rewbuffer = deque(maxlen=40) # rolling buffer for episode rewards\n\n if sum([max_iters>0, total_timesteps>0, max_episodes>0])==0:\n # noththing to be done\n return pi\n\n assert sum([max_iters>0, total_timesteps>0, max_episodes>0]) < 2, \\\n 'out of max_iters, total_timesteps, and max_episodes only one should be specified'\n\n while True:\n if callback: callback(locals(), globals())\n if total_timesteps and timesteps_so_far >= total_timesteps:\n break\n elif max_episodes and episodes_so_far >= max_episodes:\n break\n elif max_iters and iters_so_far >= max_iters:\n break\n logger.log(\"********** Iteration %i ************\"%iters_so_far)\n\n with timed(\"sampling\"):\n seg = seg_gen.__next__()\n add_vtarg_and_adv(seg, gamma, lam)\n\n # ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))\n ob, ac, atarg, tdlamret = seg[\"ob\"], seg[\"ac\"], seg[\"adv\"], seg[\"tdlamret\"]\n vpredbefore = seg[\"vpred\"] # predicted value function before udpate\n atarg = (atarg - atarg.mean()) / atarg.std() # standardized advantage function estimate\n\n if hasattr(pi, \"ret_rms\"): pi.ret_rms.update(tdlamret)\n if hasattr(pi, \"ob_rms\"): pi.ob_rms.update(ob) # update running mean/std for policy\n\n args = seg[\"ob\"], seg[\"ac\"], atarg\n fvpargs = [arr[::5] for arr in args]\n def fisher_vector_product(p):\n return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p\n\n assign_old_eq_new() # set old parameter values to new parameter values\n with timed(\"computegrad\"):\n *lossbefore, g = compute_lossandgrad(*args)\n lossbefore = allmean(np.array(lossbefore))\n g = allmean(g)\n if np.allclose(g, 0):\n logger.log(\"Got zero gradient. not updating\")\n else:\n with timed(\"cg\"):\n stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank==0)\n assert np.isfinite(stepdir).all()\n shs = .5*stepdir.dot(fisher_vector_product(stepdir))\n lm = np.sqrt(shs / max_kl)\n # logger.log(\"lagrange multiplier:\", lm, \"gnorm:\", np.linalg.norm(g))\n fullstep = stepdir / lm\n expectedimprove = g.dot(fullstep)\n surrbefore = lossbefore[0]\n stepsize = 1.0\n thbefore = get_flat()\n for _ in range(10):\n thnew = thbefore + fullstep * stepsize\n set_from_flat(thnew)\n meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))\n improve = surr - surrbefore\n logger.log(\"Expected: %.3f Actual: %.3f\"%(expectedimprove, improve))\n if not np.isfinite(meanlosses).all():\n logger.log(\"Got non-finite value of losses -- bad!\")\n elif kl > max_kl * 1.5:\n logger.log(\"violated KL constraint. shrinking step.\")\n elif improve < 0:\n logger.log(\"surrogate didn't improve. shrinking step.\")\n else:\n logger.log(\"Stepsize OK!\")\n break\n stepsize *= .5\n else:\n logger.log(\"couldn't compute a good step\")\n set_from_flat(thbefore)\n if nworkers > 1 and iters_so_far % 20 == 0:\n paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum())) # list of tuples\n assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])\n\n for (lossname, lossval) in zip(loss_names, meanlosses):\n logger.record_tabular(lossname, lossval)\n\n with timed(\"vf\"):\n\n for _ in range(vf_iters):\n for (mbob, mbret) in dataset.iterbatches((seg[\"ob\"], seg[\"tdlamret\"]),\n include_final_partial_batch=False, batch_size=64):\n g = allmean(compute_vflossandgrad(mbob, mbret))\n vfadam.update(g, vf_stepsize)\n\n logger.record_tabular(\"ev_tdlam_before\", explained_variance(vpredbefore, tdlamret))\n\n lrlocal = (seg[\"ep_lens\"], seg[\"ep_rets\"]) # local values\n if MPI is not None:\n listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples\n else:\n listoflrpairs = [lrlocal]\n\n lens, rews = map(flatten_lists, zip(*listoflrpairs))\n lenbuffer.extend(lens)\n rewbuffer.extend(rews)\n\n logger.record_tabular(\"EpLenMean\", np.mean(lenbuffer))\n logger.record_tabular(\"EpRewMean\", np.mean(rewbuffer))\n logger.record_tabular(\"EpThisIter\", len(lens))\n episodes_so_far += len(lens)\n timesteps_so_far += sum(lens)\n iters_so_far += 1\n\n logger.record_tabular(\"EpisodesSoFar\", episodes_so_far)\n logger.record_tabular(\"TimestepsSoFar\", timesteps_so_far)\n logger.record_tabular(\"TimeElapsed\", time.time() - tstart)\n\n if rank==0:\n logger.dump_tabular()\n\n return pi", "response": "Learn a new neural network with TRPO algorithm."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute discounted sums along 0th dimension of x. inputs ------ x: ndarray gamma: float outputs ------- y: ndarray with same shape as x, satisfying y[t] = x[t] + gamma*x[t+1] + gamma^2*x[t+2] + ... + gamma^k x[t+k], where k = len(x) - t - 1", "response": "def discount(x, gamma):\n \"\"\"\n computes discounted sums along 0th dimension of x.\n\n inputs\n ------\n x: ndarray\n gamma: float\n\n outputs\n -------\n y: ndarray with same shape as x, satisfying\n\n y[t] = x[t] + gamma*x[t+1] + gamma^2*x[t+2] + ... + gamma^k x[t+k],\n where k = len(x) - t - 1\n\n \"\"\"\n assert x.ndim >= 1\n return scipy.signal.lfilter([1],[1,-gamma],x[::-1], axis=0)[::-1]"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the fraction of variance that ypred explains about y.", "response": "def explained_variance(ypred,y):\n \"\"\"\n Computes fraction of variance that ypred explains about y.\n Returns 1 - Var[y-ypred] / Var[y]\n\n interpretation:\n ev=0 => might as well have predicted zero\n ev=1 => perfect prediction\n ev<0 => worse than just predicting zero\n\n \"\"\"\n assert y.ndim == 1 and ypred.ndim == 1\n vary = np.var(y)\n return np.nan if vary==0 else 1 - np.var(y-ypred)/vary"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef discount_with_boundaries(X, New, gamma):\n Y = np.zeros_like(X)\n T = X.shape[0]\n Y[T-1] = X[T-1]\n for t in range(T-2, -1, -1):\n Y[t] = X[t] + gamma * Y[t+1] * (1 - New[t+1])\n return Y", "response": "discount the episode with boundaries"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sample(self, batch_size):\n idxes = [random.randint(0, len(self._storage) - 1) for _ in range(batch_size)]\n return self._encode_sample(idxes)", "response": "Sample a batch of experiences."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add(self, *args, **kwargs):\n idx = self._next_idx\n super().add(*args, **kwargs)\n self._it_sum[idx] = self._max_priority ** self._alpha\n self._it_min[idx] = self._max_priority ** self._alpha", "response": "Add a new entry to the replay buffer."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsample a batch of experiences and return a tuple of importance weights and idxes of sampled experiences.", "response": "def sample(self, batch_size, beta):\n \"\"\"Sample a batch of experiences.\n\n compared to ReplayBuffer.sample\n it also returns importance weights and idxes\n of sampled experiences.\n\n\n Parameters\n ----------\n batch_size: int\n How many transitions to sample.\n beta: float\n To what degree to use importance weights\n (0 - no corrections, 1 - full correction)\n\n Returns\n -------\n obs_batch: np.array\n batch of observations\n act_batch: np.array\n batch of actions executed given obs_batch\n rew_batch: np.array\n rewards received as results of executing act_batch\n next_obs_batch: np.array\n next set of observations seen after executing act_batch\n done_mask: np.array\n done_mask[i] = 1 if executing act_batch[i] resulted in\n the end of an episode and 0 otherwise.\n weights: np.array\n Array of shape (batch_size,) and dtype np.float32\n denoting importance weight of each sampled transition\n idxes: np.array\n Array of shape (batch_size,) and dtype np.int32\n idexes in buffer of sampled experiences\n \"\"\"\n assert beta > 0\n\n idxes = self._sample_proportional(batch_size)\n\n weights = []\n p_min = self._it_min.min() / self._it_sum.sum()\n max_weight = (p_min * len(self._storage)) ** (-beta)\n\n for idx in idxes:\n p_sample = self._it_sum[idx] / self._it_sum.sum()\n weight = (p_sample * len(self._storage)) ** (-beta)\n weights.append(weight / max_weight)\n weights = np.array(weights)\n encoded_sample = self._encode_sample(idxes)\n return tuple(list(encoded_sample) + [weights, idxes])"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_priorities(self, idxes, priorities):\n assert len(idxes) == len(priorities)\n for idx, priority in zip(idxes, priorities):\n assert priority > 0\n assert 0 <= idx < len(self._storage)\n self._it_sum[idx] = priority ** self._alpha\n self._it_min[idx] = priority ** self._alpha\n\n self._max_priority = max(self._max_priority, priority)", "response": "Updates the priorities of sampled transitions at the given idxes."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwraps environment for retro games using deepmind - style Atari in wrap_deepmind", "response": "def wrap_deepmind_retro(env, scale=True, frame_stack=4):\n \"\"\"\n Configure environment for retro games, using config similar to DeepMind-style Atari in wrap_deepmind\n \"\"\"\n env = WarpFrame(env)\n env = ClipRewardEnv(env)\n if frame_stack > 1:\n env = FrameStack(env, frame_stack)\n if scale:\n env = ScaledFloatFrame(env)\n return env"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a list of variables in a given scope.", "response": "def scope_vars(scope, trainable_only=False):\n \"\"\"\n Get variables inside a scope\n The scope can be specified as a string\n Parameters\n ----------\n scope: str or VariableScope\n scope in which the variables reside.\n trainable_only: bool\n whether or not to return only the variables that were marked as trainable.\n Returns\n -------\n vars: [tf.Variable]\n list of variables in `scope`.\n \"\"\"\n return tf.get_collection(\n tf.GraphKeys.TRAINABLE_VARIABLES if trainable_only else tf.GraphKeys.GLOBAL_VARIABLES,\n scope=scope if isinstance(scope, str) else scope.name\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef build_act(make_obs_ph, q_func, num_actions, scope=\"deepq\", reuse=None):\n with tf.variable_scope(scope, reuse=reuse):\n observations_ph = make_obs_ph(\"observation\")\n stochastic_ph = tf.placeholder(tf.bool, (), name=\"stochastic\")\n update_eps_ph = tf.placeholder(tf.float32, (), name=\"update_eps\")\n\n eps = tf.get_variable(\"eps\", (), initializer=tf.constant_initializer(0))\n\n q_values = q_func(observations_ph.get(), num_actions, scope=\"q_func\")\n deterministic_actions = tf.argmax(q_values, axis=1)\n\n batch_size = tf.shape(observations_ph.get())[0]\n random_actions = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=num_actions, dtype=tf.int64)\n chose_random = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps\n stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)\n\n output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)\n update_eps_expr = eps.assign(tf.cond(update_eps_ph >= 0, lambda: update_eps_ph, lambda: eps))\n _act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph],\n outputs=output_actions,\n givens={update_eps_ph: -1.0, stochastic_ph: True},\n updates=[update_eps_expr])\n def act(ob, stochastic=True, update_eps=-1):\n return _act(ob, stochastic, update_eps)\n return act", "response": "Builds the act function for the object store."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef build_act_with_param_noise(make_obs_ph, q_func, num_actions, scope=\"deepq\", reuse=None, param_noise_filter_func=None):\n if param_noise_filter_func is None:\n param_noise_filter_func = default_param_noise_filter\n\n with tf.variable_scope(scope, reuse=reuse):\n observations_ph = make_obs_ph(\"observation\")\n stochastic_ph = tf.placeholder(tf.bool, (), name=\"stochastic\")\n update_eps_ph = tf.placeholder(tf.float32, (), name=\"update_eps\")\n update_param_noise_threshold_ph = tf.placeholder(tf.float32, (), name=\"update_param_noise_threshold\")\n update_param_noise_scale_ph = tf.placeholder(tf.bool, (), name=\"update_param_noise_scale\")\n reset_ph = tf.placeholder(tf.bool, (), name=\"reset\")\n\n eps = tf.get_variable(\"eps\", (), initializer=tf.constant_initializer(0))\n param_noise_scale = tf.get_variable(\"param_noise_scale\", (), initializer=tf.constant_initializer(0.01), trainable=False)\n param_noise_threshold = tf.get_variable(\"param_noise_threshold\", (), initializer=tf.constant_initializer(0.05), trainable=False)\n\n # Unmodified Q.\n q_values = q_func(observations_ph.get(), num_actions, scope=\"q_func\")\n\n # Perturbable Q used for the actual rollout.\n q_values_perturbed = q_func(observations_ph.get(), num_actions, scope=\"perturbed_q_func\")\n # We have to wrap this code into a function due to the way tf.cond() works. See\n # https://stackoverflow.com/questions/37063952/confused-by-the-behavior-of-tf-cond for\n # a more detailed discussion.\n def perturb_vars(original_scope, perturbed_scope):\n all_vars = scope_vars(absolute_scope_name(original_scope))\n all_perturbed_vars = scope_vars(absolute_scope_name(perturbed_scope))\n assert len(all_vars) == len(all_perturbed_vars)\n perturb_ops = []\n for var, perturbed_var in zip(all_vars, all_perturbed_vars):\n if param_noise_filter_func(perturbed_var):\n # Perturb this variable.\n op = tf.assign(perturbed_var, var + tf.random_normal(shape=tf.shape(var), mean=0., stddev=param_noise_scale))\n else:\n # Do not perturb, just assign.\n op = tf.assign(perturbed_var, var)\n perturb_ops.append(op)\n assert len(perturb_ops) == len(all_vars)\n return tf.group(*perturb_ops)\n\n # Set up functionality to re-compute `param_noise_scale`. This perturbs yet another copy\n # of the network and measures the effect of that perturbation in action space. If the perturbation\n # is too big, reduce scale of perturbation, otherwise increase.\n q_values_adaptive = q_func(observations_ph.get(), num_actions, scope=\"adaptive_q_func\")\n perturb_for_adaption = perturb_vars(original_scope=\"q_func\", perturbed_scope=\"adaptive_q_func\")\n kl = tf.reduce_sum(tf.nn.softmax(q_values) * (tf.log(tf.nn.softmax(q_values)) - tf.log(tf.nn.softmax(q_values_adaptive))), axis=-1)\n mean_kl = tf.reduce_mean(kl)\n def update_scale():\n with tf.control_dependencies([perturb_for_adaption]):\n update_scale_expr = tf.cond(mean_kl < param_noise_threshold,\n lambda: param_noise_scale.assign(param_noise_scale * 1.01),\n lambda: param_noise_scale.assign(param_noise_scale / 1.01),\n )\n return update_scale_expr\n\n # Functionality to update the threshold for parameter space noise.\n update_param_noise_threshold_expr = param_noise_threshold.assign(tf.cond(update_param_noise_threshold_ph >= 0,\n lambda: update_param_noise_threshold_ph, lambda: param_noise_threshold))\n\n # Put everything together.\n deterministic_actions = tf.argmax(q_values_perturbed, axis=1)\n batch_size = tf.shape(observations_ph.get())[0]\n random_actions = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=num_actions, dtype=tf.int64)\n chose_random = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps\n stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)\n\n output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)\n update_eps_expr = eps.assign(tf.cond(update_eps_ph >= 0, lambda: update_eps_ph, lambda: eps))\n updates = [\n update_eps_expr,\n tf.cond(reset_ph, lambda: perturb_vars(original_scope=\"q_func\", perturbed_scope=\"perturbed_q_func\"), lambda: tf.group(*[])),\n tf.cond(update_param_noise_scale_ph, lambda: update_scale(), lambda: tf.Variable(0., trainable=False)),\n update_param_noise_threshold_expr,\n ]\n _act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph, reset_ph, update_param_noise_threshold_ph, update_param_noise_scale_ph],\n outputs=output_actions,\n givens={update_eps_ph: -1.0, stochastic_ph: True, reset_ph: False, update_param_noise_threshold_ph: False, update_param_noise_scale_ph: False},\n updates=updates)\n def act(ob, reset=False, update_param_noise_threshold=False, update_param_noise_scale=False, stochastic=True, update_eps=-1):\n return _act(ob, stochastic, update_eps, reset, update_param_noise_threshold, update_param_noise_scale)\n return act", "response": "Builds the act function that can be used to explorate parameter space noise exploration."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef build_train(make_obs_ph, q_func, num_actions, optimizer, grad_norm_clipping=None, gamma=1.0,\n double_q=True, scope=\"deepq\", reuse=None, param_noise=False, param_noise_filter_func=None):\n \"\"\"Creates the train function:\n\n Parameters\n ----------\n make_obs_ph: str -> tf.placeholder or TfInput\n a function that takes a name and creates a placeholder of input with that name\n q_func: (tf.Variable, int, str, bool) -> tf.Variable\n the model that takes the following inputs:\n observation_in: object\n the output of observation placeholder\n num_actions: int\n number of actions\n scope: str\n reuse: bool\n should be passed to outer variable scope\n and returns a tensor of shape (batch_size, num_actions) with values of every action.\n num_actions: int\n number of actions\n reuse: bool\n whether or not to reuse the graph variables\n optimizer: tf.train.Optimizer\n optimizer to use for the Q-learning objective.\n grad_norm_clipping: float or None\n clip gradient norms to this value. If None no clipping is performed.\n gamma: float\n discount rate.\n double_q: bool\n if true will use Double Q Learning (https://arxiv.org/abs/1509.06461).\n In general it is a good idea to keep it enabled.\n scope: str or VariableScope\n optional scope for variable_scope.\n reuse: bool or None\n whether or not the variables should be reused. To be able to reuse the scope must be given.\n param_noise: bool\n whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)\n param_noise_filter_func: tf.Variable -> bool\n function that decides whether or not a variable should be perturbed. Only applicable\n if param_noise is True. If set to None, default_param_noise_filter is used by default.\n\n Returns\n -------\n act: (tf.Variable, bool, float) -> tf.Variable\n function to select and action given observation.\n` See the top of the file for details.\n train: (object, np.array, np.array, object, np.array, np.array) -> np.array\n optimize the error in Bellman's equation.\n` See the top of the file for details.\n update_target: () -> ()\n copy the parameters from optimized Q function to the target Q function.\n` See the top of the file for details.\n debug: {str: function}\n a bunch of functions to print debug data like q_values.\n \"\"\"\n if param_noise:\n act_f = build_act_with_param_noise(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse,\n param_noise_filter_func=param_noise_filter_func)\n else:\n act_f = build_act(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse)\n\n with tf.variable_scope(scope, reuse=reuse):\n # set up placeholders\n obs_t_input = make_obs_ph(\"obs_t\")\n act_t_ph = tf.placeholder(tf.int32, [None], name=\"action\")\n rew_t_ph = tf.placeholder(tf.float32, [None], name=\"reward\")\n obs_tp1_input = make_obs_ph(\"obs_tp1\")\n done_mask_ph = tf.placeholder(tf.float32, [None], name=\"done\")\n importance_weights_ph = tf.placeholder(tf.float32, [None], name=\"weight\")\n\n # q network evaluation\n q_t = q_func(obs_t_input.get(), num_actions, scope=\"q_func\", reuse=True) # reuse parameters from act\n q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=tf.get_variable_scope().name + \"/q_func\")\n\n # target q network evalution\n q_tp1 = q_func(obs_tp1_input.get(), num_actions, scope=\"target_q_func\")\n target_q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=tf.get_variable_scope().name + \"/target_q_func\")\n\n # q scores for actions which we know were selected in the given state.\n q_t_selected = tf.reduce_sum(q_t * tf.one_hot(act_t_ph, num_actions), 1)\n\n # compute estimate of best possible value starting from state at t + 1\n if double_q:\n q_tp1_using_online_net = q_func(obs_tp1_input.get(), num_actions, scope=\"q_func\", reuse=True)\n q_tp1_best_using_online_net = tf.argmax(q_tp1_using_online_net, 1)\n q_tp1_best = tf.reduce_sum(q_tp1 * tf.one_hot(q_tp1_best_using_online_net, num_actions), 1)\n else:\n q_tp1_best = tf.reduce_max(q_tp1, 1)\n q_tp1_best_masked = (1.0 - done_mask_ph) * q_tp1_best\n\n # compute RHS of bellman equation\n q_t_selected_target = rew_t_ph + gamma * q_tp1_best_masked\n\n # compute the error (potentially clipped)\n td_error = q_t_selected - tf.stop_gradient(q_t_selected_target)\n errors = U.huber_loss(td_error)\n weighted_error = tf.reduce_mean(importance_weights_ph * errors)\n\n # compute optimization op (potentially with gradient clipping)\n if grad_norm_clipping is not None:\n gradients = optimizer.compute_gradients(weighted_error, var_list=q_func_vars)\n for i, (grad, var) in enumerate(gradients):\n if grad is not None:\n gradients[i] = (tf.clip_by_norm(grad, grad_norm_clipping), var)\n optimize_expr = optimizer.apply_gradients(gradients)\n else:\n optimize_expr = optimizer.minimize(weighted_error, var_list=q_func_vars)\n\n # update_target_fn will be called periodically to copy Q network to target Q network\n update_target_expr = []\n for var, var_target in zip(sorted(q_func_vars, key=lambda v: v.name),\n sorted(target_q_func_vars, key=lambda v: v.name)):\n update_target_expr.append(var_target.assign(var))\n update_target_expr = tf.group(*update_target_expr)\n\n # Create callable functions\n train = U.function(\n inputs=[\n obs_t_input,\n act_t_ph,\n rew_t_ph,\n obs_tp1_input,\n done_mask_ph,\n importance_weights_ph\n ],\n outputs=td_error,\n updates=[optimize_expr]\n )\n update_target = U.function([], [], updates=[update_target_expr])\n\n q_values = U.function([obs_t_input], q_t)\n\n return act_f, train, update_target, {'q_values': q_values}", "response": "Builds the train function for the given object store."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef profile_tf_runningmeanstd():\n import time\n from baselines.common import tf_util\n\n tf_util.get_session( config=tf.ConfigProto(\n inter_op_parallelism_threads=1,\n intra_op_parallelism_threads=1,\n allow_soft_placement=True\n ))\n\n x = np.random.random((376,))\n\n n_trials = 10000\n rms = RunningMeanStd()\n tfrms = TfRunningMeanStd()\n\n tic1 = time.time()\n for _ in range(n_trials):\n rms.update(x)\n\n tic2 = time.time()\n for _ in range(n_trials):\n tfrms.update(x)\n\n tic3 = time.time()\n\n print('rms update time ({} trials): {} s'.format(n_trials, tic2 - tic1))\n print('tfrms update time ({} trials): {} s'.format(n_trials, tic3 - tic2))\n\n\n tic1 = time.time()\n for _ in range(n_trials):\n z1 = rms.mean\n\n tic2 = time.time()\n for _ in range(n_trials):\n z2 = tfrms.mean\n\n assert z1 == z2\n\n tic3 = time.time()\n\n print('rms get mean time ({} trials): {} s'.format(n_trials, tic2 - tic1))\n print('tfrms get mean time ({} trials): {} s'.format(n_trials, tic3 - tic2))\n\n\n\n '''\n options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) #pylint: disable=E1101\n run_metadata = tf.RunMetadata()\n profile_opts = dict(options=options, run_metadata=run_metadata)\n\n\n\n from tensorflow.python.client import timeline\n fetched_timeline = timeline.Timeline(run_metadata.step_stats) #pylint: disable=E1101\n chrome_trace = fetched_timeline.generate_chrome_trace_format()\n outfile = '/tmp/timeline.json'\n with open(outfile, 'wt') as f:\n f.write(chrome_trace)\n print('Successfully saved profile to {}. Exiting.'.format(outfile))\n exit(0)\n '''", "response": "profile a running mean standard deviation"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_sample_her_transitions(replay_strategy, replay_k, reward_fun):\n if replay_strategy == 'future':\n future_p = 1 - (1. / (1 + replay_k))\n else: # 'replay_strategy' == 'none'\n future_p = 0\n\n def _sample_her_transitions(episode_batch, batch_size_in_transitions):\n \"\"\"episode_batch is {key: array(buffer_size x T x dim_key)}\n \"\"\"\n T = episode_batch['u'].shape[1]\n rollout_batch_size = episode_batch['u'].shape[0]\n batch_size = batch_size_in_transitions\n\n # Select which episodes and time steps to use.\n episode_idxs = np.random.randint(0, rollout_batch_size, batch_size)\n t_samples = np.random.randint(T, size=batch_size)\n transitions = {key: episode_batch[key][episode_idxs, t_samples].copy()\n for key in episode_batch.keys()}\n\n # Select future time indexes proportional with probability future_p. These\n # will be used for HER replay by substituting in future goals.\n her_indexes = np.where(np.random.uniform(size=batch_size) < future_p)\n future_offset = np.random.uniform(size=batch_size) * (T - t_samples)\n future_offset = future_offset.astype(int)\n future_t = (t_samples + 1 + future_offset)[her_indexes]\n\n # Replace goal with achieved goal but only for the previously-selected\n # HER transitions (as defined by her_indexes). For the other transitions,\n # keep the original goal.\n future_ag = episode_batch['ag'][episode_idxs[her_indexes], future_t]\n transitions['g'][her_indexes] = future_ag\n\n # Reconstruct info dictionary for reward computation.\n info = {}\n for key, value in transitions.items():\n if key.startswith('info_'):\n info[key.replace('info_', '')] = value\n\n # Re-compute reward since we may have substituted the goal.\n reward_params = {k: transitions[k] for k in ['ag_2', 'g']}\n reward_params['info'] = info\n transitions['r'] = reward_fun(**reward_params)\n\n transitions = {k: transitions[k].reshape(batch_size, *transitions[k].shape[1:])\n for k in transitions.keys()}\n\n assert(transitions['u'].shape[0] == batch_size_in_transitions)\n\n return transitions\n\n return _sample_her_transitions", "response": "Creates a sample function that can be used for HER experience replays."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef model(inpt, num_actions, scope, reuse=False):\n with tf.variable_scope(scope, reuse=reuse):\n out = inpt\n out = layers.fully_connected(out, num_outputs=64, activation_fn=tf.nn.tanh)\n out = layers.fully_connected(out, num_outputs=num_actions, activation_fn=None)\n return out", "response": "This model takes as input an observation and returns values of all actions."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a dict of the sample size of the current set of entries", "response": "def sample(self, batch_size):\n \"\"\"Returns a dict {key: array(batch_size x shapes[key])}\n \"\"\"\n buffers = {}\n\n with self.lock:\n assert self.current_size > 0\n for key in self.buffers.keys():\n buffers[key] = self.buffers[key][:self.current_size]\n\n buffers['o_2'] = buffers['o'][:, 1:, :]\n buffers['ag_2'] = buffers['ag'][:, 1:, :]\n\n transitions = self.sample_transitions(buffers, batch_size)\n\n for key in (['r', 'o_2', 'ag_2'] + list(self.buffers.keys())):\n assert key in transitions, \"key %s missing from transitions\" % key\n\n return transitions"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstore the episode into the buffers and the number of transitions that have been stored.", "response": "def store_episode(self, episode_batch):\n \"\"\"episode_batch: array(batch_size x (T or T+1) x dim_key)\n \"\"\"\n batch_sizes = [len(episode_batch[key]) for key in episode_batch.keys()]\n assert np.all(np.array(batch_sizes) == batch_sizes[0])\n batch_size = batch_sizes[0]\n\n with self.lock:\n idxs = self._get_storage_idx(batch_size)\n\n # load inputs into buffers\n for key in self.buffers.keys():\n self.buffers[key][idxs] = episode_batch[key]\n\n self.n_transitions_stored += batch_size * self.T"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef store_episode(self, episode_batch, update_stats=True):\n\n self.buffer.store_episode(episode_batch)\n\n if update_stats:\n # add transitions to normalizer\n episode_batch['o_2'] = episode_batch['o'][:, 1:, :]\n episode_batch['ag_2'] = episode_batch['ag'][:, 1:, :]\n num_normalizing_transitions = transitions_in_episode_batch(episode_batch)\n transitions = self.sample_transitions(episode_batch, num_normalizing_transitions)\n\n o, g, ag = transitions['o'], transitions['g'], transitions['ag']\n transitions['o'], transitions['g'] = self._preprocess_og(o, ag, g)\n # No need to preprocess the o_2 and g_2 since this is only used for stats\n\n self.o_stats.update(transitions['o'])\n self.g_stats.update(transitions['g'])\n\n self.o_stats.recompute_stats()\n self.g_stats.recompute_stats()", "response": "Store an episode in the internal buffer."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts a list of '=' - spaced command - line arguments to a dictionary evaluating python objects when possible", "response": "def parse_cmdline_kwargs(args):\n '''\n convert a list of '='-spaced command-line arguments to a dictionary, evaluating python objects when possible\n '''\n def parse(v):\n\n assert isinstance(v, str)\n try:\n return eval(v)\n except (NameError, SyntaxError):\n return v\n\n return {k: parse(v) for k,v in parse_unknown_args(args).items()}"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a new environment from a function that has not already been created.", "response": "def cached_make_env(make_env):\n \"\"\"\n Only creates a new environment from the provided function if one has not yet already been\n created. This is useful here because we need to infer certain properties of the env, e.g.\n its observation and action spaces, without any intend of actually using it.\n \"\"\"\n if make_env not in CACHED_ENVS:\n env = make_env()\n CACHED_ENVS[make_env] = env\n return CACHED_ENVS[make_env]"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef compute_geometric_median(X, eps=1e-5):\n y = np.mean(X, 0)\n\n while True:\n D = scipy.spatial.distance.cdist(X, [y])\n nonzeros = (D != 0)[:, 0]\n\n Dinv = 1 / D[nonzeros]\n Dinvs = np.sum(Dinv)\n W = Dinv / Dinvs\n T = np.sum(W * X[nonzeros], 0)\n\n num_zeros = len(X) - np.sum(nonzeros)\n if num_zeros == 0:\n y1 = T\n elif num_zeros == len(X):\n return y\n else:\n R = (T - y) * Dinvs\n r = np.linalg.norm(R)\n rinv = 0 if r == 0 else num_zeros/r\n y1 = max(0, 1-rinv)*T + min(1, rinv)*y\n\n if scipy.spatial.distance.euclidean(y, y1) < eps:\n return y1\n\n y = y1", "response": "Computes the geometric median of points in 2D."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nproject the keypoint onto a new position on a new image.", "response": "def project(self, from_shape, to_shape):\n \"\"\"\n Project the keypoint onto a new position on a new image.\n\n E.g. if the keypoint is on its original image at x=(10 of 100 pixels)\n and y=(20 of 100 pixels) and is projected onto a new image with\n size (width=200, height=200), its new position will be (20, 40).\n\n This is intended for cases where the original image is resized.\n It cannot be used for more complex changes (e.g. padding, cropping).\n\n Parameters\n ----------\n from_shape : tuple of int\n Shape of the original image. (Before resize.)\n\n to_shape : tuple of int\n Shape of the new image. (After resize.)\n\n Returns\n -------\n imgaug.Keypoint\n Keypoint object with new coordinates.\n\n \"\"\"\n xy_proj = project_coords([(self.x, self.y)], from_shape, to_shape)\n return self.deepcopy(x=xy_proj[0][0], y=xy_proj[0][1])"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a new keypoint object with new coordinates shifted around the image.", "response": "def shift(self, x=0, y=0):\n \"\"\"\n Move the keypoint around on an image.\n\n Parameters\n ----------\n x : number, optional\n Move by this value on the x axis.\n\n y : number, optional\n Move by this value on the y axis.\n\n Returns\n -------\n imgaug.Keypoint\n Keypoint object with new coordinates.\n\n \"\"\"\n return self.deepcopy(self.x + x, self.y + y)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndraw the keypoint onto a given image.", "response": "def draw_on_image(self, image, color=(0, 255, 0), alpha=1.0, size=3,\n copy=True, raise_if_out_of_image=False):\n \"\"\"\n Draw the keypoint onto a given image.\n\n The keypoint is drawn as a square.\n\n Parameters\n ----------\n image : (H,W,3) ndarray\n The image onto which to draw the keypoint.\n\n color : int or list of int or tuple of int or (3,) ndarray, optional\n The RGB color of the keypoint. If a single int ``C``, then that is\n equivalent to ``(C,C,C)``.\n\n alpha : float, optional\n The opacity of the drawn keypoint, where ``1.0`` denotes a fully\n visible keypoint and ``0.0`` an invisible one.\n\n size : int, optional\n The size of the keypoint. If set to ``S``, each square will have\n size ``S x S``.\n\n copy : bool, optional\n Whether to copy the image before drawing the keypoint.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an exception if the keypoint is outside of the\n image.\n\n Returns\n -------\n image : (H,W,3) ndarray\n Image with drawn keypoint.\n\n \"\"\"\n if copy:\n image = np.copy(image)\n\n if image.ndim == 2:\n assert ia.is_single_number(color), (\n \"Got a 2D image. Expected then 'color' to be a single number, \"\n \"but got %s.\" % (str(color),))\n elif image.ndim == 3 and ia.is_single_number(color):\n color = [color] * image.shape[-1]\n\n input_dtype = image.dtype\n alpha_color = color\n if alpha < 0.01:\n # keypoint invisible, nothing to do\n return image\n elif alpha > 0.99:\n alpha = 1\n else:\n image = image.astype(np.float32, copy=False)\n alpha_color = alpha * np.array(color)\n\n height, width = image.shape[0:2]\n\n y, x = self.y_int, self.x_int\n\n x1 = max(x - size//2, 0)\n x2 = min(x + 1 + size//2, width)\n y1 = max(y - size//2, 0)\n y2 = min(y + 1 + size//2, height)\n\n x1_clipped, x2_clipped = np.clip([x1, x2], 0, width)\n y1_clipped, y2_clipped = np.clip([y1, y2], 0, height)\n\n x1_clipped_ooi = (x1_clipped < 0 or x1_clipped >= width)\n x2_clipped_ooi = (x2_clipped < 0 or x2_clipped >= width+1)\n y1_clipped_ooi = (y1_clipped < 0 or y1_clipped >= height)\n y2_clipped_ooi = (y2_clipped < 0 or y2_clipped >= height+1)\n x_ooi = (x1_clipped_ooi and x2_clipped_ooi)\n y_ooi = (y1_clipped_ooi and y2_clipped_ooi)\n x_zero_size = (x2_clipped - x1_clipped) < 1 # min size is 1px\n y_zero_size = (y2_clipped - y1_clipped) < 1\n if not x_ooi and not y_ooi and not x_zero_size and not y_zero_size:\n if alpha == 1:\n image[y1_clipped:y2_clipped, x1_clipped:x2_clipped] = color\n else:\n image[y1_clipped:y2_clipped, x1_clipped:x2_clipped] = (\n (1 - alpha)\n * image[y1_clipped:y2_clipped, x1_clipped:x2_clipped]\n + alpha_color\n )\n else:\n if raise_if_out_of_image:\n raise Exception(\n \"Cannot draw keypoint x=%.8f, y=%.8f on image with \"\n \"shape %s.\" % (y, x, image.shape))\n\n if image.dtype.name != input_dtype.name:\n if input_dtype.name == \"uint8\":\n image = np.clip(image, 0, 255, out=image)\n image = image.astype(input_dtype, copy=False)\n return image"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating nearby points to this keypoint based on manhattan distance. To generate the first neighbouring points, a distance of S (step size) is moved from the center point (this keypoint) to the top, right, bottom and left, resulting in four new points. From these new points, the pattern is repeated. Overlapping points are ignored. The resulting points have a shape similar to a square rotated by 45 degrees. Parameters ---------- nb_steps : int The number of steps to move from the center point. nb_steps=1 results in a total of 5 output points (1 center point + 4 neighbours). step_size : number The step size to move from every point to its neighbours. return_array : bool, optional Whether to return the generated points as a list of keypoints or an array of shape ``(N,2)``, where ``N`` is the number of generated points and the second axis contains the x- (first value) and y- (second value) coordinates. Returns ------- points : list of imgaug.Keypoint or (N,2) ndarray If return_array was False, then a list of Keypoint. Otherwise a numpy array of shape ``(N,2)``, where ``N`` is the number of generated points and the second axis contains the x- (first value) and y- (second value) coordinates. The center keypoint (the one on which this function was called) is always included.", "response": "def generate_similar_points_manhattan(self, nb_steps, step_size, return_array=False):\n \"\"\"\n Generate nearby points to this keypoint based on manhattan distance.\n\n To generate the first neighbouring points, a distance of S (step size) is moved from the\n center point (this keypoint) to the top, right, bottom and left, resulting in four new\n points. From these new points, the pattern is repeated. Overlapping points are ignored.\n\n The resulting points have a shape similar to a square rotated by 45 degrees.\n\n Parameters\n ----------\n nb_steps : int\n The number of steps to move from the center point. nb_steps=1 results in a total of\n 5 output points (1 center point + 4 neighbours).\n\n step_size : number\n The step size to move from every point to its neighbours.\n\n return_array : bool, optional\n Whether to return the generated points as a list of keypoints or an array\n of shape ``(N,2)``, where ``N`` is the number of generated points and the second axis contains\n the x- (first value) and y- (second value) coordinates.\n\n Returns\n -------\n points : list of imgaug.Keypoint or (N,2) ndarray\n If return_array was False, then a list of Keypoint.\n Otherwise a numpy array of shape ``(N,2)``, where ``N`` is the number of generated points and\n the second axis contains the x- (first value) and y- (second value) coordinates.\n The center keypoint (the one on which this function was called) is always included.\n\n \"\"\"\n # TODO add test\n # Points generates in manhattan style with S steps have a shape similar to a 45deg rotated\n # square. The center line with the origin point has S+1+S = 1+2*S points (S to the left,\n # S to the right). The lines above contain (S+1+S)-2 + (S+1+S)-2-2 + ... + 1 points. E.g.\n # for S=2 it would be 3+1=4 and for S=3 it would be 5+3+1=9. Same for the lines below the\n # center. Hence the total number of points is S+1+S + 2*(S^2).\n points = np.zeros((nb_steps + 1 + nb_steps + 2*(nb_steps**2), 2), dtype=np.float32)\n\n # we start at the bottom-most line and move towards the top-most line\n yy = np.linspace(self.y - nb_steps * step_size, self.y + nb_steps * step_size, nb_steps + 1 + nb_steps)\n\n # bottom-most line contains only one point\n width = 1\n\n nth_point = 0\n for i_y, y in enumerate(yy):\n if width == 1:\n xx = [self.x]\n else:\n xx = np.linspace(self.x - (width-1)//2 * step_size, self.x + (width-1)//2 * step_size, width)\n for x in xx:\n points[nth_point] = [x, y]\n nth_point += 1\n if i_y < nb_steps:\n width += 2\n else:\n width -= 2\n\n if return_array:\n return points\n return [self.deepcopy(x=points[i, 0], y=points[i, 1]) for i in sm.xrange(points.shape[0])]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef copy(self, x=None, y=None):\n return self.deepcopy(x=x, y=y)", "response": "Create a shallow copy of the Keypoint object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a deep copy of the Keypoint object.", "response": "def deepcopy(self, x=None, y=None):\n \"\"\"\n Create a deep copy of the Keypoint object.\n\n Parameters\n ----------\n x : None or number, optional\n Coordinate of the keypoint on the x axis.\n If ``None``, the instance's value will be copied.\n\n y : None or number, optional\n Coordinate of the keypoint on the y axis.\n If ``None``, the instance's value will be copied.\n\n Returns\n -------\n imgaug.Keypoint\n Deep copy.\n\n \"\"\"\n x = self.x if x is None else x\n y = self.y if y is None else y\n return Keypoint(x=x, y=y)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef on(self, image):\n shape = normalize_shape(image)\n if shape[0:2] == self.shape[0:2]:\n return self.deepcopy()\n else:\n keypoints = [kp.project(self.shape, shape) for kp in self.keypoints]\n return self.deepcopy(keypoints, shape)", "response": "Project keypoints from one image to a new one."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef draw_on_image(self, image, color=(0, 255, 0), alpha=1.0, size=3,\n copy=True, raise_if_out_of_image=False):\n \"\"\"\n Draw all keypoints onto a given image.\n\n Each keypoint is marked by a square of a chosen color and size.\n\n Parameters\n ----------\n image : (H,W,3) ndarray\n The image onto which to draw the keypoints.\n This image should usually have the same shape as\n set in KeypointsOnImage.shape.\n\n color : int or list of int or tuple of int or (3,) ndarray, optional\n The RGB color of all keypoints. If a single int ``C``, then that is\n equivalent to ``(C,C,C)``.\n\n alpha : float, optional\n The opacity of the drawn keypoint, where ``1.0`` denotes a fully\n visible keypoint and ``0.0`` an invisible one.\n\n size : int, optional\n The size of each point. If set to ``C``, each square will have\n size ``C x C``.\n\n copy : bool, optional\n Whether to copy the image before drawing the points.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an exception if any keypoint is outside of the image.\n\n Returns\n -------\n image : (H,W,3) ndarray\n Image with drawn keypoints.\n\n \"\"\"\n image = np.copy(image) if copy else image\n for keypoint in self.keypoints:\n image = keypoint.draw_on_image(\n image, color=color, alpha=alpha, size=size, copy=False,\n raise_if_out_of_image=raise_if_out_of_image)\n return image", "response": "Draw all keypoints onto a given image."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef shift(self, x=0, y=0):\n keypoints = [keypoint.shift(x=x, y=y) for keypoint in self.keypoints]\n return self.deepcopy(keypoints)", "response": "Shifts the keypoints around on an image."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts keypoint coordinates to xy - form.", "response": "def to_xy_array(self):\n \"\"\"\n Convert keypoint coordinates to ``(N,2)`` array.\n\n Returns\n -------\n (N, 2) ndarray\n Array containing the coordinates of all keypoints.\n Shape is ``(N,2)`` with coordinates in xy-form.\n\n \"\"\"\n result = np.zeros((len(self.keypoints), 2), dtype=np.float32)\n for i, keypoint in enumerate(self.keypoints):\n result[i, 0] = keypoint.x\n result[i, 1] = keypoint.y\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef from_xy_array(cls, xy, shape):\n keypoints = [Keypoint(x=coord[0], y=coord[1]) for coord in xy]\n return KeypointsOnImage(keypoints, shape)", "response": "Convert an array of xy - coordinates to a KeypointsOnImage object."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns an array of keypoints drawn from the keypoints on image.", "response": "def to_keypoint_image(self, size=1):\n \"\"\"\n Draws a new black image of shape ``(H,W,N)`` in which all keypoint coordinates are set to 255.\n (H=shape height, W=shape width, N=number of keypoints)\n\n This function can be used as a helper when augmenting keypoints with a method that only supports the\n augmentation of images.\n\n Parameters\n -------\n size : int\n Size of each (squared) point.\n\n Returns\n -------\n image : (H,W,N) ndarray\n Image in which the keypoints are marked. H is the height,\n defined in KeypointsOnImage.shape[0] (analogous W). N is the\n number of keypoints.\n\n \"\"\"\n ia.do_assert(len(self.keypoints) > 0)\n height, width = self.shape[0:2]\n image = np.zeros((height, width, len(self.keypoints)), dtype=np.uint8)\n ia.do_assert(size % 2 != 0)\n sizeh = max(0, (size-1)//2)\n for i, keypoint in enumerate(self.keypoints):\n # TODO for float values spread activation over several cells\n # here and do voting at the end\n y = keypoint.y_int\n x = keypoint.x_int\n\n x1 = np.clip(x - sizeh, 0, width-1)\n x2 = np.clip(x + sizeh + 1, 0, width)\n y1 = np.clip(y - sizeh, 0, height-1)\n y2 = np.clip(y + sizeh + 1, 0, height)\n\n if x1 < x2 and y1 < y2:\n image[y1:y2, x1:x2, i] = 128\n if 0 <= y < height and 0 <= x < width:\n image[y, x, i] = 255\n return image"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert an image generated by to_keypoint_image to a KeypointsOnImage object.", "response": "def from_keypoint_image(image, if_not_found_coords={\"x\": -1, \"y\": -1}, threshold=1, nb_channels=None): # pylint: disable=locally-disabled, dangerous-default-value, line-too-long\n \"\"\"\n Converts an image generated by ``to_keypoint_image()`` back to a KeypointsOnImage object.\n\n Parameters\n ----------\n image : (H,W,N) ndarray\n The keypoints image. N is the number of keypoints.\n\n if_not_found_coords : tuple or list or dict or None, optional\n Coordinates to use for keypoints that cannot be found in `image`.\n If this is a list/tuple, it must have two integer values.\n If it is a dictionary, it must have the keys ``x`` and ``y`` with\n each containing one integer value.\n If this is None, then the keypoint will not be added to the final\n KeypointsOnImage object.\n\n threshold : int, optional\n The search for keypoints works by searching for the argmax in\n each channel. This parameters contains the minimum value that\n the max must have in order to be viewed as a keypoint.\n\n nb_channels : None or int, optional\n Number of channels of the image on which the keypoints are placed.\n Some keypoint augmenters require that information.\n If set to None, the keypoint's shape will be set\n to ``(height, width)``, otherwise ``(height, width, nb_channels)``.\n\n Returns\n -------\n out : KeypointsOnImage\n The extracted keypoints.\n\n \"\"\"\n ia.do_assert(len(image.shape) == 3)\n height, width, nb_keypoints = image.shape\n\n drop_if_not_found = False\n if if_not_found_coords is None:\n drop_if_not_found = True\n if_not_found_x = -1\n if_not_found_y = -1\n elif isinstance(if_not_found_coords, (tuple, list)):\n ia.do_assert(len(if_not_found_coords) == 2)\n if_not_found_x = if_not_found_coords[0]\n if_not_found_y = if_not_found_coords[1]\n elif isinstance(if_not_found_coords, dict):\n if_not_found_x = if_not_found_coords[\"x\"]\n if_not_found_y = if_not_found_coords[\"y\"]\n else:\n raise Exception(\"Expected if_not_found_coords to be None or tuple or list or dict, got %s.\" % (\n type(if_not_found_coords),))\n\n keypoints = []\n for i in sm.xrange(nb_keypoints):\n maxidx_flat = np.argmax(image[..., i])\n maxidx_ndim = np.unravel_index(maxidx_flat, (height, width))\n found = (image[maxidx_ndim[0], maxidx_ndim[1], i] >= threshold)\n if found:\n keypoints.append(Keypoint(x=maxidx_ndim[1], y=maxidx_ndim[0]))\n else:\n if drop_if_not_found:\n pass # dont add the keypoint to the result list, i.e. drop it\n else:\n keypoints.append(Keypoint(x=if_not_found_x, y=if_not_found_y))\n\n out_shape = (height, width)\n if nb_channels is not None:\n out_shape += (nb_channels,)\n return KeypointsOnImage(keypoints, shape=out_shape)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_distance_maps(self, inverted=False):\n ia.do_assert(len(self.keypoints) > 0)\n height, width = self.shape[0:2]\n distance_maps = np.zeros((height, width, len(self.keypoints)), dtype=np.float32)\n\n yy = np.arange(0, height)\n xx = np.arange(0, width)\n grid_xx, grid_yy = np.meshgrid(xx, yy)\n\n for i, keypoint in enumerate(self.keypoints):\n y, x = keypoint.y, keypoint.x\n distance_maps[:, :, i] = (grid_xx - x) ** 2 + (grid_yy - y) ** 2\n distance_maps = np.sqrt(distance_maps)\n if inverted:\n return 1/(distance_maps+1)\n return distance_maps", "response": "Generates a distance map for the keypoints in the keypoint system."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting a distance maps array to a KeypointsOnImage object.", "response": "def from_distance_maps(distance_maps, inverted=False, if_not_found_coords={\"x\": -1, \"y\": -1}, threshold=None, # pylint: disable=locally-disabled, dangerous-default-value, line-too-long\n nb_channels=None):\n \"\"\"\n Converts maps generated by ``to_distance_maps()`` back to a KeypointsOnImage object.\n\n Parameters\n ----------\n distance_maps : (H,W,N) ndarray\n The distance maps. N is the number of keypoints.\n\n inverted : bool, optional\n Whether the given distance maps were generated in inverted or normal mode.\n\n if_not_found_coords : tuple or list or dict or None, optional\n Coordinates to use for keypoints that cannot be found in ``distance_maps``.\n If this is a list/tuple, it must have two integer values.\n If it is a dictionary, it must have the keys ``x`` and ``y``, with each\n containing one integer value.\n If this is None, then the keypoint will not be added to the final\n KeypointsOnImage object.\n\n threshold : float, optional\n The search for keypoints works by searching for the argmin (non-inverted) or\n argmax (inverted) in each channel. This parameters contains the maximum (non-inverted)\n or minimum (inverted) value to accept in order to view a hit as a keypoint.\n Use None to use no min/max.\n\n nb_channels : None or int, optional\n Number of channels of the image on which the keypoints are placed.\n Some keypoint augmenters require that information.\n If set to None, the keypoint's shape will be set\n to ``(height, width)``, otherwise ``(height, width, nb_channels)``.\n\n Returns\n -------\n imgaug.KeypointsOnImage\n The extracted keypoints.\n\n \"\"\"\n ia.do_assert(len(distance_maps.shape) == 3)\n height, width, nb_keypoints = distance_maps.shape\n\n drop_if_not_found = False\n if if_not_found_coords is None:\n drop_if_not_found = True\n if_not_found_x = -1\n if_not_found_y = -1\n elif isinstance(if_not_found_coords, (tuple, list)):\n ia.do_assert(len(if_not_found_coords) == 2)\n if_not_found_x = if_not_found_coords[0]\n if_not_found_y = if_not_found_coords[1]\n elif isinstance(if_not_found_coords, dict):\n if_not_found_x = if_not_found_coords[\"x\"]\n if_not_found_y = if_not_found_coords[\"y\"]\n else:\n raise Exception(\"Expected if_not_found_coords to be None or tuple or list or dict, got %s.\" % (\n type(if_not_found_coords),))\n\n keypoints = []\n for i in sm.xrange(nb_keypoints):\n # TODO introduce voting here among all distance values that have min/max values\n if inverted:\n hitidx_flat = np.argmax(distance_maps[..., i])\n else:\n hitidx_flat = np.argmin(distance_maps[..., i])\n hitidx_ndim = np.unravel_index(hitidx_flat, (height, width))\n if not inverted and threshold is not None:\n found = (distance_maps[hitidx_ndim[0], hitidx_ndim[1], i] < threshold)\n elif inverted and threshold is not None:\n found = (distance_maps[hitidx_ndim[0], hitidx_ndim[1], i] >= threshold)\n else:\n found = True\n if found:\n keypoints.append(Keypoint(x=hitidx_ndim[1], y=hitidx_ndim[0]))\n else:\n if drop_if_not_found:\n pass # dont add the keypoint to the result list, i.e. drop it\n else:\n keypoints.append(Keypoint(x=if_not_found_x, y=if_not_found_y))\n\n out_shape = (height, width)\n if nb_channels is not None:\n out_shape += (nb_channels,)\n return KeypointsOnImage(keypoints, shape=out_shape)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef copy(self, keypoints=None, shape=None):\n result = copy.copy(self)\n if keypoints is not None:\n result.keypoints = keypoints\n if shape is not None:\n result.shape = shape\n return result", "response": "Create a shallow copy of the KeypointsOnImage object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a deep copy of the KeypointsOnImage object.", "response": "def deepcopy(self, keypoints=None, shape=None):\n \"\"\"\n Create a deep copy of the KeypointsOnImage object.\n\n Parameters\n ----------\n keypoints : None or list of imgaug.Keypoint, optional\n List of keypoints on the image. If ``None``, the instance's\n keypoints will be copied.\n\n shape : tuple of int, optional\n The shape of the image on which the keypoints are placed.\n If ``None``, the instance's shape will be copied.\n\n Returns\n -------\n imgaug.KeypointsOnImage\n Deep copy.\n\n \"\"\"\n # for some reason deepcopy is way slower here than manual copy\n if keypoints is None:\n keypoints = [kp.deepcopy() for kp in self.keypoints]\n if shape is None:\n shape = tuple(self.shape)\n return KeypointsOnImage(keypoints, shape)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef contains(self, other):\n if isinstance(other, tuple):\n x, y = other\n else:\n x, y = other.x, other.y\n return self.x1 <= x <= self.x2 and self.y1 <= y <= self.y2", "response": "Return True if the bounding box contains the point other."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nprojects the bounding box onto a differently shaped image.", "response": "def project(self, from_shape, to_shape):\n \"\"\"\n Project the bounding box onto a differently shaped image.\n\n E.g. if the bounding box is on its original image at\n x1=(10 of 100 pixels) and y1=(20 of 100 pixels) and is projected onto\n a new image with size (width=200, height=200), its new position will\n be (x1=20, y1=40). (Analogous for x2/y2.)\n\n This is intended for cases where the original image is resized.\n It cannot be used for more complex changes (e.g. padding, cropping).\n\n Parameters\n ----------\n from_shape : tuple of int or ndarray\n Shape of the original image. (Before resize.)\n\n to_shape : tuple of int or ndarray\n Shape of the new image. (After resize.)\n\n Returns\n -------\n out : imgaug.BoundingBox\n BoundingBox object with new coordinates.\n\n \"\"\"\n coords_proj = project_coords([(self.x1, self.y1), (self.x2, self.y2)],\n from_shape, to_shape)\n return self.copy(\n x1=coords_proj[0][0],\n y1=coords_proj[0][1],\n x2=coords_proj[1][0],\n y2=coords_proj[1][1],\n label=self.label)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef extend(self, all_sides=0, top=0, right=0, bottom=0, left=0):\n return BoundingBox(\n x1=self.x1 - all_sides - left,\n x2=self.x2 + all_sides + right,\n y1=self.y1 - all_sides - top,\n y2=self.y2 + all_sides + bottom\n )", "response": "Returns a new bounding box with the size of the bounding box along its sides."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef intersection(self, other, default=None):\n x1_i = max(self.x1, other.x1)\n y1_i = max(self.y1, other.y1)\n x2_i = min(self.x2, other.x2)\n y2_i = min(self.y2, other.y2)\n if x1_i > x2_i or y1_i > y2_i:\n return default\n else:\n return BoundingBox(x1=x1_i, y1=y1_i, x2=x2_i, y2=y2_i)", "response": "Compute the intersection of this bounding box and another."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncomputes the union bounding box of this bounding box and another bounding box.", "response": "def union(self, other):\n \"\"\"\n Compute the union bounding box of this bounding box and another one.\n\n This is equivalent to drawing a bounding box around all corners points of both\n bounding boxes.\n\n Parameters\n ----------\n other : imgaug.BoundingBox\n Other bounding box with which to generate the union.\n\n Returns\n -------\n imgaug.BoundingBox\n Union bounding box of the two bounding boxes.\n\n \"\"\"\n return BoundingBox(\n x1=min(self.x1, other.x1),\n y1=min(self.y1, other.y1),\n x2=max(self.x2, other.x2),\n y2=max(self.y2, other.y2),\n )"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the IoU of this bounding box with another.", "response": "def iou(self, other):\n \"\"\"\n Compute the IoU of this bounding box with another one.\n\n IoU is the intersection over union, defined as::\n\n ``area(intersection(A, B)) / area(union(A, B))``\n ``= area(intersection(A, B)) / (area(A) + area(B) - area(intersection(A, B)))``\n\n Parameters\n ----------\n other : imgaug.BoundingBox\n Other bounding box with which to compare.\n\n Returns\n -------\n float\n IoU between the two bounding boxes.\n\n \"\"\"\n inters = self.intersection(other)\n if inters is None:\n return 0.0\n else:\n area_union = self.area + other.area - inters.area\n return inters.area / area_union if area_union > 0 else 0.0"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if the bounding box is fully inside the image area.", "response": "def is_fully_within_image(self, image):\n \"\"\"\n Estimate whether the bounding box is fully inside the image area.\n\n Parameters\n ----------\n image : (H,W,...) ndarray or tuple of int\n Image dimensions to use.\n If an ndarray, its shape will be used.\n If a tuple, it is assumed to represent the image shape\n and must contain at least two integers.\n\n Returns\n -------\n bool\n True if the bounding box is fully inside the image area. False otherwise.\n\n \"\"\"\n shape = normalize_shape(image)\n height, width = shape[0:2]\n return self.x1 >= 0 and self.x2 < width and self.y1 >= 0 and self.y2 < height"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning True if the bounding box is at least partially inside the image area.", "response": "def is_partly_within_image(self, image):\n \"\"\"\n Estimate whether the bounding box is at least partially inside the image area.\n\n Parameters\n ----------\n image : (H,W,...) ndarray or tuple of int\n Image dimensions to use.\n If an ndarray, its shape will be used.\n If a tuple, it is assumed to represent the image shape\n and must contain at least two integers.\n\n Returns\n -------\n bool\n True if the bounding box is at least partially inside the image area. False otherwise.\n\n \"\"\"\n shape = normalize_shape(image)\n height, width = shape[0:2]\n eps = np.finfo(np.float32).eps\n img_bb = BoundingBox(x1=0, x2=width-eps, y1=0, y2=height-eps)\n return self.intersection(img_bb) is not None"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns True if the bounding box is partially outside of the image area.", "response": "def is_out_of_image(self, image, fully=True, partly=False):\n \"\"\"\n Estimate whether the bounding box is partially or fully outside of the image area.\n\n Parameters\n ----------\n image : (H,W,...) ndarray or tuple of int\n Image dimensions to use. If an ndarray, its shape will be used. If a tuple, it is\n assumed to represent the image shape and must contain at least two integers.\n\n fully : bool, optional\n Whether to return True if the bounding box is fully outside fo the image area.\n\n partly : bool, optional\n Whether to return True if the bounding box is at least partially outside fo the\n image area.\n\n Returns\n -------\n bool\n True if the bounding box is partially/fully outside of the image area, depending\n on defined parameters. False otherwise.\n\n \"\"\"\n if self.is_fully_within_image(image):\n return False\n elif self.is_partly_within_image(image):\n return partly\n else:\n return fully"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef clip_out_of_image(self, image):\n shape = normalize_shape(image)\n\n height, width = shape[0:2]\n ia.do_assert(height > 0)\n ia.do_assert(width > 0)\n\n eps = np.finfo(np.float32).eps\n x1 = np.clip(self.x1, 0, width - eps)\n x2 = np.clip(self.x2, 0, width - eps)\n y1 = np.clip(self.y1, 0, height - eps)\n y2 = np.clip(self.y2, 0, height - eps)\n\n return self.copy(\n x1=x1,\n y1=y1,\n x2=x2,\n y2=y2,\n label=self.label\n )", "response": "Clip off all parts of the bounding box that are outside of the image."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nshifting the bounding box from one or more image sides i. e. move it on the x - axis.", "response": "def shift(self, top=None, right=None, bottom=None, left=None):\n \"\"\"\n Shift the bounding box from one or more image sides, i.e. move it on the x/y-axis.\n\n Parameters\n ----------\n top : None or int, optional\n Amount of pixels by which to shift the bounding box from the top.\n\n right : None or int, optional\n Amount of pixels by which to shift the bounding box from the right.\n\n bottom : None or int, optional\n Amount of pixels by which to shift the bounding box from the bottom.\n\n left : None or int, optional\n Amount of pixels by which to shift the bounding box from the left.\n\n Returns\n -------\n result : imgaug.BoundingBox\n Shifted bounding box.\n\n \"\"\"\n top = top if top is not None else 0\n right = right if right is not None else 0\n bottom = bottom if bottom is not None else 0\n left = left if left is not None else 0\n return self.copy(\n x1=self.x1+left-right,\n x2=self.x2+left-right,\n y1=self.y1+top-bottom,\n y2=self.y2+top-bottom\n )"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef draw_on_image(self, image, color=(0, 255, 0), alpha=1.0, size=1,\n copy=True, raise_if_out_of_image=False, thickness=None):\n \"\"\"\n Draw the bounding box on an image.\n\n Parameters\n ----------\n image : (H,W,C) ndarray(uint8)\n The image onto which to draw the bounding box.\n\n color : iterable of int, optional\n The color to use, corresponding to the channel layout of the image. Usually RGB.\n\n alpha : float, optional\n The transparency of the drawn bounding box, where 1.0 denotes no transparency and\n 0.0 is invisible.\n\n size : int, optional\n The thickness of the bounding box in pixels. If the value is larger than 1, then\n additional pixels will be added around the bounding box (i.e. extension towards the\n outside).\n\n copy : bool, optional\n Whether to copy the input image or change it in-place.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the bounding box is fully outside of the\n image. If set to False, no error will be raised and only the parts inside the image\n will be drawn.\n\n thickness : None or int, optional\n Deprecated.\n\n Returns\n -------\n result : (H,W,C) ndarray(uint8)\n Image with bounding box drawn on it.\n\n \"\"\"\n if thickness is not None:\n ia.warn_deprecated(\n \"Usage of argument 'thickness' in BoundingBox.draw_on_image() \"\n \"is deprecated. The argument was renamed to 'size'.\"\n )\n size = thickness\n\n if raise_if_out_of_image and self.is_out_of_image(image):\n raise Exception(\"Cannot draw bounding box x1=%.8f, y1=%.8f, x2=%.8f, y2=%.8f on image with shape %s.\" % (\n self.x1, self.y1, self.x2, self.y2, image.shape))\n\n result = np.copy(image) if copy else image\n\n if isinstance(color, (tuple, list)):\n color = np.uint8(color)\n\n for i in range(size):\n y1, y2, x1, x2 = self.y1_int, self.y2_int, self.x1_int, self.x2_int\n\n # When y values get into the range (H-0.5, H), the *_int functions round them to H.\n # That is technically sensible, but in the case of drawing means that the border lies\n # just barely outside of the image, making the border disappear, even though the BB\n # is fully inside the image. Here we correct for that because of beauty reasons.\n # Same is the case for x coordinates.\n if self.is_fully_within_image(image):\n y1 = np.clip(y1, 0, image.shape[0]-1)\n y2 = np.clip(y2, 0, image.shape[0]-1)\n x1 = np.clip(x1, 0, image.shape[1]-1)\n x2 = np.clip(x2, 0, image.shape[1]-1)\n\n y = [y1-i, y1-i, y2+i, y2+i]\n x = [x1-i, x2+i, x2+i, x1-i]\n rr, cc = skimage.draw.polygon_perimeter(y, x, shape=result.shape)\n if alpha >= 0.99:\n result[rr, cc, :] = color\n else:\n if ia.is_float_array(result):\n result[rr, cc, :] = (1 - alpha) * result[rr, cc, :] + alpha * color\n result = np.clip(result, 0, 255)\n else:\n input_dtype = result.dtype\n result = result.astype(np.float32)\n result[rr, cc, :] = (1 - alpha) * result[rr, cc, :] + alpha * color\n result = np.clip(result, 0, 255).astype(input_dtype)\n\n return result", "response": "Draw the bounding box on an image."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef extract_from_image(self, image, pad=True, pad_max=None,\n prevent_zero_size=True):\n \"\"\"\n Extract the image pixels within the bounding box.\n\n This function will zero-pad the image if the bounding box is partially/fully outside of\n the image.\n\n Parameters\n ----------\n image : (H,W) ndarray or (H,W,C) ndarray\n The image from which to extract the pixels within the bounding box.\n\n pad : bool, optional\n Whether to zero-pad the image if the object is partially/fully\n outside of it.\n\n pad_max : None or int, optional\n The maximum number of pixels that may be zero-paded on any side,\n i.e. if this has value ``N`` the total maximum of added pixels\n is ``4*N``.\n This option exists to prevent extremely large images as a result of\n single points being moved very far away during augmentation.\n\n prevent_zero_size : bool, optional\n Whether to prevent height or width of the extracted image from becoming zero.\n If this is set to True and height or width of the bounding box is below 1, the height/width will\n be increased to 1. This can be useful to prevent problems, e.g. with image saving or plotting.\n If it is set to False, images will be returned as ``(H', W')`` or ``(H', W', 3)`` with ``H`` or\n ``W`` potentially being 0.\n\n Returns\n -------\n image : (H',W') ndarray or (H',W',C) ndarray\n Pixels within the bounding box. Zero-padded if the bounding box is partially/fully\n outside of the image. If prevent_zero_size is activated, it is guarantueed that ``H'>0``\n and ``W'>0``, otherwise only ``H'>=0`` and ``W'>=0``.\n\n \"\"\"\n pad_top = 0\n pad_right = 0\n pad_bottom = 0\n pad_left = 0\n\n height, width = image.shape[0], image.shape[1]\n x1, x2, y1, y2 = self.x1_int, self.x2_int, self.y1_int, self.y2_int\n\n # When y values get into the range (H-0.5, H), the *_int functions round them to H.\n # That is technically sensible, but in the case of extraction leads to a black border,\n # which is both ugly and unexpected after calling cut_out_of_image(). Here we correct for\n # that because of beauty reasons.\n # Same is the case for x coordinates.\n fully_within = self.is_fully_within_image(image)\n if fully_within:\n y1, y2 = np.clip([y1, y2], 0, height-1)\n x1, x2 = np.clip([x1, x2], 0, width-1)\n\n # TODO add test\n if prevent_zero_size:\n if abs(x2 - x1) < 1:\n x2 = x1 + 1\n if abs(y2 - y1) < 1:\n y2 = y1 + 1\n\n if pad:\n # if the bb is outside of the image area, the following pads the image\n # first with black pixels until the bb is inside the image\n # and only then extracts the image area\n # TODO probably more efficient to initialize an array of zeros\n # and copy only the portions of the bb into that array that are\n # natively inside the image area\n if x1 < 0:\n pad_left = abs(x1)\n x2 = x2 + pad_left\n width = width + pad_left\n x1 = 0\n if y1 < 0:\n pad_top = abs(y1)\n y2 = y2 + pad_top\n height = height + pad_top\n y1 = 0\n if x2 >= width:\n pad_right = x2 - width\n if y2 >= height:\n pad_bottom = y2 - height\n\n paddings = [pad_top, pad_right, pad_bottom, pad_left]\n any_padded = any([val > 0 for val in paddings])\n if any_padded:\n if pad_max is None:\n pad_max = max(paddings)\n\n image = ia.pad(\n image,\n top=min(pad_top, pad_max),\n right=min(pad_right, pad_max),\n bottom=min(pad_bottom, pad_max),\n left=min(pad_left, pad_max)\n )\n return image[y1:y2, x1:x2]\n else:\n within_image = (\n (0, 0, 0, 0)\n <= (x1, y1, x2, y2)\n < (width, height, width, height)\n )\n out_height, out_width = (y2 - y1), (x2 - x1)\n nonzero_height = (out_height > 0)\n nonzero_width = (out_width > 0)\n if within_image and nonzero_height and nonzero_width:\n return image[y1:y2, x1:x2]\n if prevent_zero_size:\n out_height = 1\n out_width = 1\n else:\n out_height = 0\n out_width = 0\n if image.ndim == 2:\n return np.zeros((out_height, out_width), dtype=image.dtype)\n return np.zeros((out_height, out_width, image.shape[-1]),\n dtype=image.dtype)", "response": "This function extracts the image pixels from the image and returns the image as a numpy array."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts the corners of the bounding box to keypoints.", "response": "def to_keypoints(self):\n \"\"\"\n Convert the corners of the bounding box to keypoints (clockwise, starting at top left).\n\n Returns\n -------\n list of imgaug.Keypoint\n Corners of the bounding box as keypoints.\n\n \"\"\"\n # TODO get rid of this deferred import\n from imgaug.augmentables.kps import Keypoint\n\n return [\n Keypoint(x=self.x1, y=self.y1),\n Keypoint(x=self.x2, y=self.y1),\n Keypoint(x=self.x2, y=self.y2),\n Keypoint(x=self.x1, y=self.y2)\n ]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef copy(self, x1=None, y1=None, x2=None, y2=None, label=None):\n return BoundingBox(\n x1=self.x1 if x1 is None else x1,\n x2=self.x2 if x2 is None else x2,\n y1=self.y1 if y1 is None else y1,\n y2=self.y2 if y2 is None else y2,\n label=self.label if label is None else label\n )", "response": "Create a shallow copy of the BoundingBox object."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a deep copy of the BoundingBox object.", "response": "def deepcopy(self, x1=None, y1=None, x2=None, y2=None, label=None):\n \"\"\"\n Create a deep copy of the BoundingBox object.\n\n Parameters\n ----------\n x1 : None or number\n If not None, then the x1 coordinate of the copied object will be set to this value.\n\n y1 : None or number\n If not None, then the y1 coordinate of the copied object will be set to this value.\n\n x2 : None or number\n If not None, then the x2 coordinate of the copied object will be set to this value.\n\n y2 : None or number\n If not None, then the y2 coordinate of the copied object will be set to this value.\n\n label : None or string\n If not None, then the label of the copied object will be set to this value.\n\n Returns\n -------\n imgaug.BoundingBox\n Deep copy.\n\n \"\"\"\n return self.copy(x1=x1, y1=y1, x2=x2, y2=y2, label=label)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef on(self, image):\n shape = normalize_shape(image)\n if shape[0:2] == self.shape[0:2]:\n return self.deepcopy()\n bounding_boxes = [bb.project(self.shape, shape)\n for bb in self.bounding_boxes]\n return BoundingBoxesOnImage(bounding_boxes, shape)", "response": "Project bounding boxes from one image to a new one."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef from_xyxy_array(cls, xyxy, shape):\n ia.do_assert(xyxy.shape[1] == 4, \"Expected input array of shape (N, 4), got shape %s.\" % (xyxy.shape,))\n\n boxes = [BoundingBox(*row) for row in xyxy]\n\n return cls(boxes, shape)", "response": "Convert an array containing xyxy coordinates to a BoundingBoxesOnImage object."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef to_xyxy_array(self, dtype=np.float32):\n xyxy_array = np.zeros((len(self.bounding_boxes), 4), dtype=np.float32)\n\n for i, box in enumerate(self.bounding_boxes):\n xyxy_array[i] = [box.x1, box.y1, box.x2, box.y2]\n\n return xyxy_array.astype(dtype)", "response": "Convert the BoundingBoxesOnImage object to an array of xyxy coordinates."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef draw_on_image(self, image, color=(0, 255, 0), alpha=1.0, size=1,\n copy=True, raise_if_out_of_image=False, thickness=None):\n \"\"\"\n Draw all bounding boxes onto a given image.\n\n Parameters\n ----------\n image : (H,W,3) ndarray\n The image onto which to draw the bounding boxes.\n This image should usually have the same shape as\n set in BoundingBoxesOnImage.shape.\n\n color : int or list of int or tuple of int or (3,) ndarray, optional\n The RGB color of all bounding boxes. If a single int ``C``, then\n that is equivalent to ``(C,C,C)``.\n\n alpha : float, optional\n Alpha/transparency of the bounding box.\n\n size : int, optional\n Thickness in pixels.\n\n copy : bool, optional\n Whether to copy the image before drawing the bounding boxes.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an exception if any bounding box is outside of the\n image.\n\n thickness : None or int, optional\n Deprecated.\n\n Returns\n -------\n image : (H,W,3) ndarray\n Image with drawn bounding boxes.\n\n \"\"\"\n image = np.copy(image) if copy else image\n\n for bb in self.bounding_boxes:\n image = bb.draw_on_image(\n image,\n color=color,\n alpha=alpha,\n size=size,\n copy=False,\n raise_if_out_of_image=raise_if_out_of_image,\n thickness=thickness\n )\n\n return image", "response": "Draw all bounding boxes onto a given image."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nremoving all bounding boxes that are not part of the image.", "response": "def remove_out_of_image(self, fully=True, partly=False):\n \"\"\"\n Remove all bounding boxes that are fully or partially outside of the image.\n\n Parameters\n ----------\n fully : bool, optional\n Whether to remove bounding boxes that are fully outside of the image.\n\n partly : bool, optional\n Whether to remove bounding boxes that are partially outside of the image.\n\n Returns\n -------\n imgaug.BoundingBoxesOnImage\n Reduced set of bounding boxes, with those that were fully/partially outside of\n the image removed.\n\n \"\"\"\n bbs_clean = [bb for bb in self.bounding_boxes\n if not bb.is_out_of_image(self.shape, fully=fully, partly=partly)]\n return BoundingBoxesOnImage(bbs_clean, shape=self.shape)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef clip_out_of_image(self):\n bbs_cut = [bb.clip_out_of_image(self.shape)\n for bb in self.bounding_boxes if bb.is_partly_within_image(self.shape)]\n return BoundingBoxesOnImage(bbs_cut, shape=self.shape)", "response": "Clip off all parts from all bounding boxes that are outside of the image."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nshifting all bounding boxes from one or more image sides i. e. move them on the x - axis.", "response": "def shift(self, top=None, right=None, bottom=None, left=None):\n \"\"\"\n Shift all bounding boxes from one or more image sides, i.e. move them on the x/y-axis.\n\n Parameters\n ----------\n top : None or int, optional\n Amount of pixels by which to shift all bounding boxes from the top.\n\n right : None or int, optional\n Amount of pixels by which to shift all bounding boxes from the right.\n\n bottom : None or int, optional\n Amount of pixels by which to shift all bounding boxes from the bottom.\n\n left : None or int, optional\n Amount of pixels by which to shift all bounding boxes from the left.\n\n Returns\n -------\n imgaug.BoundingBoxesOnImage\n Shifted bounding boxes.\n\n \"\"\"\n bbs_new = [bb.shift(top=top, right=right, bottom=bottom, left=left) for bb in self.bounding_boxes]\n return BoundingBoxesOnImage(bbs_new, shape=self.shape)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a deep copy of the BoundingBoxesOnImage object. Returns ------- imgaug. BoundingBoxesOnImage Deep copy.", "response": "def deepcopy(self):\n \"\"\"\n Create a deep copy of the BoundingBoxesOnImage object.\n\n Returns\n -------\n imgaug.BoundingBoxesOnImage\n Deep copy.\n\n \"\"\"\n # Manual copy is far faster than deepcopy for BoundingBoxesOnImage,\n # so use manual copy here too\n bbs = [bb.deepcopy() for bb in self.bounding_boxes]\n return BoundingBoxesOnImage(bbs, tuple(self.shape))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DirectedEdgeDetect(alpha=0, direction=(0.0, 1.0), name=None, deterministic=False, random_state=None):\n alpha_param = iap.handle_continuous_param(alpha, \"alpha\", value_range=(0, 1.0), tuple_to_uniform=True,\n list_to_choice=True)\n direction_param = iap.handle_continuous_param(direction, \"direction\", value_range=None, tuple_to_uniform=True,\n list_to_choice=True)\n\n def create_matrices(_image, nb_channels, random_state_func):\n alpha_sample = alpha_param.draw_sample(random_state=random_state_func)\n ia.do_assert(0 <= alpha_sample <= 1.0)\n direction_sample = direction_param.draw_sample(random_state=random_state_func)\n\n deg = int(direction_sample * 360) % 360\n rad = np.deg2rad(deg)\n x = np.cos(rad - 0.5*np.pi)\n y = np.sin(rad - 0.5*np.pi)\n direction_vector = np.array([x, y])\n\n matrix_effect = np.array([\n [0, 0, 0],\n [0, 0, 0],\n [0, 0, 0]\n ], dtype=np.float32)\n for x in [-1, 0, 1]:\n for y in [-1, 0, 1]:\n if (x, y) != (0, 0):\n cell_vector = np.array([x, y])\n distance_deg = np.rad2deg(ia.angle_between_vectors(cell_vector, direction_vector))\n distance = distance_deg / 180\n similarity = (1 - distance)**4\n matrix_effect[y+1, x+1] = similarity\n matrix_effect = matrix_effect / np.sum(matrix_effect)\n matrix_effect = matrix_effect * (-1)\n matrix_effect[1, 1] = 1\n\n matrix_nochange = np.array([\n [0, 0, 0],\n [0, 1, 0],\n [0, 0, 0]\n ], dtype=np.float32)\n\n matrix = (1-alpha_sample) * matrix_nochange + alpha_sample * matrix_effect\n\n return [matrix] * nb_channels\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return Convolve(create_matrices, name=name, deterministic=deterministic, random_state=random_state)", "response": "Directed Edge Detect Augmenter that detects edges in a sharpened image."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nnormalize a shape tuple or array to a shape tuple.", "response": "def normalize_shape(shape):\n \"\"\"\n Normalize a shape tuple or array to a shape tuple.\n\n Parameters\n ----------\n shape : tuple of int or ndarray\n The input to normalize. May optionally be an array.\n\n Returns\n -------\n tuple of int\n Shape tuple.\n\n \"\"\"\n if isinstance(shape, tuple):\n return shape\n assert ia.is_np_array(shape), (\n \"Expected tuple of ints or array, got %s.\" % (type(shape),))\n return shape.shape"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nproject coordinates from one image shape to another.", "response": "def project_coords(coords, from_shape, to_shape):\n \"\"\"\n Project coordinates from one image shape to another.\n\n This performs a relative projection, e.g. a point at 60% of the old\n image width will be at 60% of the new image width after projection.\n\n Parameters\n ----------\n coords : ndarray or tuple of number\n Coordinates to project. Either a ``(N,2)`` numpy array or a tuple\n of `(x,y)` coordinates.\n\n from_shape : tuple of int or ndarray\n Old image shape.\n\n to_shape : tuple of int or ndarray\n New image shape.\n\n Returns\n -------\n ndarray\n Projected coordinates as ``(N,2)`` ``float32`` numpy array.\n\n \"\"\"\n from_shape = normalize_shape(from_shape)\n to_shape = normalize_shape(to_shape)\n if from_shape[0:2] == to_shape[0:2]:\n return coords\n\n from_height, from_width = from_shape[0:2]\n to_height, to_width = to_shape[0:2]\n assert all([v > 0 for v in [from_height, from_width, to_height, to_width]])\n\n # make sure to not just call np.float32(coords) here as the following lines\n # perform in-place changes and np.float32(.) only copies if the input\n # was *not* a float32 array\n coords_proj = np.array(coords).astype(np.float32)\n coords_proj[:, 0] = (coords_proj[:, 0] / from_width) * to_width\n coords_proj[:, 1] = (coords_proj[:, 1] / from_height) * to_height\n return coords_proj"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef AdditiveGaussianNoise(loc=0, scale=0, per_channel=False, name=None, deterministic=False, random_state=None):\n loc2 = iap.handle_continuous_param(loc, \"loc\", value_range=None, tuple_to_uniform=True, list_to_choice=True)\n scale2 = iap.handle_continuous_param(scale, \"scale\", value_range=(0, None), tuple_to_uniform=True,\n list_to_choice=True)\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return AddElementwise(iap.Normal(loc=loc2, scale=scale2), per_channel=per_channel, name=name,\n deterministic=deterministic, random_state=random_state)", "response": "Adds a gaussian noise to the internal list of images."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates an augmenter to add poisson noise to images. Poisson noise is comparable to gaussian noise as in ``AdditiveGaussianNoise``, but the values are sampled from a poisson distribution instead of a gaussian distribution. As poisson distributions produce only positive numbers, the sign of the sampled values are here randomly flipped. Values of around ``10.0`` for `lam` lead to visible noise (for uint8). Values of around ``20.0`` for `lam` lead to very visible noise (for uint8). It is recommended to usually set `per_channel` to True. dtype support:: See ``imgaug.augmenters.arithmetic.AddElementwise``. Parameters ---------- lam : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional Lambda parameter of the poisson distribution. Recommended values are around ``0.0`` to ``10.0``. * If a number, exactly that value will be used. * If a tuple ``(a, b)``, a random value from the range ``a <= x <= b`` will be sampled per image. * If a list, then a random value will be sampled from that list per image. * If a StochasticParameter, a value will be sampled from the parameter per image. per_channel : bool or float, optional Whether to use the same noise value per pixel for all channels (False) or to sample a new value for each channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel` will be treated as True, otherwise as False. name : None or str, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. deterministic : bool, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. random_state : None or int or numpy.random.RandomState, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. Examples -------- >>> aug = iaa.AdditivePoissonNoise(lam=5.0) Adds poisson noise sampled from ``Poisson(5.0)`` to images. >>> aug = iaa.AdditivePoissonNoise(lam=(0.0, 10.0)) Adds poisson noise sampled from ``Poisson(x)`` to images, where ``x`` is randomly sampled per image from the interval ``[0.0, 10.0]``. >>> aug = iaa.AdditivePoissonNoise(lam=5.0, per_channel=True) Adds poisson noise sampled from ``Poisson(5.0)`` to images, where the values are different per pixel *and* channel (e.g. a different one for red, green and blue channels for the same pixel). >>> aug = iaa.AdditivePoissonNoise(lam=(0.0, 10.0), per_channel=True) Adds poisson noise sampled from ``Poisson(x)`` to images, with ``x`` being sampled from ``uniform(0.0, 10.0)`` per image, pixel and channel. This is the *recommended* configuration. >>> aug = iaa.AdditivePoissonNoise(lam=2, per_channel=0.5) Adds poisson noise sampled from the distribution ``Poisson(2)`` to images, where the values are sometimes (50 percent of all cases) the same per pixel for all channels and sometimes different (other 50 percent).", "response": "def AdditivePoissonNoise(lam=0, per_channel=False, name=None, deterministic=False, random_state=None):\n \"\"\"\n Create an augmenter to add poisson noise to images.\n\n Poisson noise is comparable to gaussian noise as in ``AdditiveGaussianNoise``, but the values are sampled from\n a poisson distribution instead of a gaussian distribution. As poisson distributions produce only positive numbers,\n the sign of the sampled values are here randomly flipped.\n\n Values of around ``10.0`` for `lam` lead to visible noise (for uint8).\n Values of around ``20.0`` for `lam` lead to very visible noise (for uint8).\n It is recommended to usually set `per_channel` to True.\n\n dtype support::\n\n See ``imgaug.augmenters.arithmetic.AddElementwise``.\n\n Parameters\n ----------\n lam : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional\n Lambda parameter of the poisson distribution. Recommended values are around ``0.0`` to ``10.0``.\n\n * If a number, exactly that value will be used.\n * If a tuple ``(a, b)``, a random value from the range ``a <= x <= b`` will\n be sampled per image.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, a value will be sampled from the\n parameter per image.\n\n per_channel : bool or float, optional\n Whether to use the same noise value per pixel for all channels (False)\n or to sample a new value for each channel (True).\n If this value is a float ``p``, then for ``p`` percent of all images\n `per_channel` will be treated as True, otherwise as False.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Examples\n --------\n >>> aug = iaa.AdditivePoissonNoise(lam=5.0)\n\n Adds poisson noise sampled from ``Poisson(5.0)`` to images.\n\n >>> aug = iaa.AdditivePoissonNoise(lam=(0.0, 10.0))\n\n Adds poisson noise sampled from ``Poisson(x)`` to images, where ``x`` is randomly sampled per image from the\n interval ``[0.0, 10.0]``.\n\n >>> aug = iaa.AdditivePoissonNoise(lam=5.0, per_channel=True)\n\n Adds poisson noise sampled from ``Poisson(5.0)`` to images,\n where the values are different per pixel *and* channel (e.g. a\n different one for red, green and blue channels for the same pixel).\n\n >>> aug = iaa.AdditivePoissonNoise(lam=(0.0, 10.0), per_channel=True)\n\n Adds poisson noise sampled from ``Poisson(x)`` to images,\n with ``x`` being sampled from ``uniform(0.0, 10.0)`` per image, pixel and channel.\n This is the *recommended* configuration.\n\n >>> aug = iaa.AdditivePoissonNoise(lam=2, per_channel=0.5)\n\n Adds poisson noise sampled from the distribution ``Poisson(2)`` to images,\n where the values are sometimes (50 percent of all cases) the same\n per pixel for all channels and sometimes different (other 50 percent).\n\n \"\"\"\n lam2 = iap.handle_continuous_param(lam, \"lam\", value_range=(0, None), tuple_to_uniform=True,\n list_to_choice=True)\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return AddElementwise(iap.RandomSign(iap.Poisson(lam=lam2)), per_channel=per_channel, name=name,\n deterministic=deterministic, random_state=random_state)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Dropout(p=0, per_channel=False, name=None, deterministic=False, random_state=None):\n if ia.is_single_number(p):\n p2 = iap.Binomial(1 - p)\n elif ia.is_iterable(p):\n ia.do_assert(len(p) == 2)\n ia.do_assert(p[0] < p[1])\n ia.do_assert(0 <= p[0] <= 1.0)\n ia.do_assert(0 <= p[1] <= 1.0)\n p2 = iap.Binomial(iap.Uniform(1 - p[1], 1 - p[0]))\n elif isinstance(p, iap.StochasticParameter):\n p2 = p\n else:\n raise Exception(\"Expected p to be float or int or StochasticParameter, got %s.\" % (type(p),))\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return MultiplyElementwise(p2, per_channel=per_channel, name=name, deterministic=deterministic,\n random_state=random_state)", "response": "Augmenter that drops a certain fraction of pixels in the current language."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CoarseDropout(p=0, size_px=None, size_percent=None, per_channel=False, min_size=4, name=None, deterministic=False,\n random_state=None):\n \"\"\"\n Augmenter that sets rectangular areas within images to zero.\n\n In contrast to Dropout, these areas can have larger sizes.\n (E.g. you might end up with three large black rectangles in an image.)\n Note that the current implementation leads to correlated sizes,\n so when there is one large area that is dropped, there is a high likelihood\n that all other dropped areas are also large.\n\n This method is implemented by generating the dropout mask at a\n lower resolution (than the image has) and then upsampling the mask\n before dropping the pixels.\n\n dtype support::\n\n See ``imgaug.augmenters.arithmetic.MultiplyElementwise``.\n\n Parameters\n ----------\n p : float or tuple of float or imgaug.parameters.StochasticParameter, optional\n The probability of any pixel being dropped (i.e. set to zero).\n\n * If a float, then that value will be used for all pixels. A value\n of 1.0 would mean, that all pixels will be dropped. A value of\n 0.0 would lead to no pixels being dropped.\n * If a tuple ``(a, b)``, then a value p will be sampled from the\n range ``a <= p <= b`` per image and be used as the pixel's dropout\n probability.\n * If a StochasticParameter, then this parameter will be used to\n determine per pixel whether it should be dropped (sampled value\n of 0) or shouldn't (sampled value of 1).\n\n size_px : int or tuple of int or imgaug.parameters.StochasticParameter, optional\n The size of the lower resolution image from which to sample the dropout\n mask in absolute pixel dimensions.\n\n * If an integer, then that size will be used for both height and\n width. E.g. a value of 3 would lead to a ``3x3`` mask, which is then\n upsampled to ``HxW``, where ``H`` is the image size and W the image width.\n * If a tuple ``(a, b)``, then two values ``M``, ``N`` will be sampled from the\n range ``[a..b]`` and the mask will be generated at size ``MxN``, then\n upsampled to ``HxW``.\n * If a StochasticParameter, then this parameter will be used to\n determine the sizes. It is expected to be discrete.\n\n size_percent : float or tuple of float or imgaug.parameters.StochasticParameter, optional\n The size of the lower resolution image from which to sample the dropout\n mask *in percent* of the input image.\n\n * If a float, then that value will be used as the percentage of the\n height and width (relative to the original size). E.g. for value\n p, the mask will be sampled from ``(p*H)x(p*W)`` and later upsampled\n to ``HxW``.\n * If a tuple ``(a, b)``, then two values ``m``, ``n`` will be sampled from the\n interval ``(a, b)`` and used as the percentages, i.e the mask size\n will be ``(m*H)x(n*W)``.\n * If a StochasticParameter, then this parameter will be used to\n sample the percentage values. It is expected to be continuous.\n\n per_channel : bool or float, optional\n Whether to use the same value (is dropped / is not dropped)\n for all channels of a pixel (False) or to sample a new value for each\n channel (True).\n If this value is a float ``p``, then for ``p`` percent of all images\n `per_channel` will be treated as True, otherwise as False.\n\n min_size : int, optional\n Minimum size of the low resolution mask, both width and height. If\n `size_percent` or `size_px` leads to a lower value than this, `min_size`\n will be used instead. This should never have a value of less than 2,\n otherwise one may end up with a ``1x1`` low resolution mask, leading easily\n to the whole image being dropped.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Examples\n --------\n >>> aug = iaa.CoarseDropout(0.02, size_percent=0.5)\n\n drops 2 percent of all pixels on an lower-resolution image that has\n 50 percent of the original image's size, leading to dropped areas that\n have roughly 2x2 pixels size.\n\n\n >>> aug = iaa.CoarseDropout((0.0, 0.05), size_percent=(0.05, 0.5))\n\n generates a dropout mask at 5 to 50 percent of image's size. In that mask,\n 0 to 5 percent of all pixels are dropped (random per image).\n\n >>> aug = iaa.CoarseDropout((0.0, 0.05), size_px=(2, 16))\n\n same as previous example, but the lower resolution image has 2 to 16 pixels\n size.\n\n >>> aug = iaa.CoarseDropout(0.02, size_percent=0.5, per_channel=True)\n\n drops 2 percent of all pixels at 50 percent resolution (2x2 sizes)\n in a channel-wise fashion, i.e. it is unlikely\n for any pixel to have all channels set to zero (black pixels).\n\n >>> aug = iaa.CoarseDropout(0.02, size_percent=0.5, per_channel=0.5)\n\n same as previous example, but the `per_channel` feature is only active\n for 50 percent of all images.\n\n \"\"\"\n if ia.is_single_number(p):\n p2 = iap.Binomial(1 - p)\n elif ia.is_iterable(p):\n ia.do_assert(len(p) == 2)\n ia.do_assert(p[0] < p[1])\n ia.do_assert(0 <= p[0] <= 1.0)\n ia.do_assert(0 <= p[1] <= 1.0)\n p2 = iap.Binomial(iap.Uniform(1 - p[1], 1 - p[0]))\n elif isinstance(p, iap.StochasticParameter):\n p2 = p\n else:\n raise Exception(\"Expected p to be float or int or StochasticParameter, got %s.\" % (type(p),))\n\n if size_px is not None:\n p3 = iap.FromLowerResolution(other_param=p2, size_px=size_px, min_size=min_size)\n elif size_percent is not None:\n p3 = iap.FromLowerResolution(other_param=p2, size_percent=size_percent, min_size=min_size)\n else:\n raise Exception(\"Either size_px or size_percent must be set.\")\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return MultiplyElementwise(p3, per_channel=per_channel, name=name, deterministic=deterministic,\n random_state=random_state)", "response": "Augmenter that drops the image in a non - zero area."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ImpulseNoise(p=0, name=None, deterministic=False, random_state=None):\n return SaltAndPepper(p=p, per_channel=True, name=name, deterministic=deterministic, random_state=random_state)", "response": "Augmenter to apply impulse noise to an image."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SaltAndPepper(p=0, per_channel=False, name=None, deterministic=False, random_state=None):\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return ReplaceElementwise(\n mask=p,\n replacement=iap.Beta(0.5, 0.5) * 255,\n per_channel=per_channel,\n name=name,\n deterministic=deterministic,\n random_state=random_state\n )", "response": "Augment an image with salt and pepper noise."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a new empty tree with pepper noise.", "response": "def Pepper(p=0, per_channel=False, name=None, deterministic=False, random_state=None):\n \"\"\"\n Adds pepper noise to an image, i.e. black-ish pixels.\n\n This is similar to dropout, but slower and the black pixels are not uniformly black.\n\n dtype support::\n\n See ``imgaug.augmenters.arithmetic.ReplaceElementwise``.\n\n Parameters\n ----------\n p : float or tuple of float or list of float or imgaug.parameters.StochasticParameter, optional\n Probability of changing a pixel to pepper noise.\n\n * If a float, then that value will be used for all images as the\n probability.\n * If a tuple ``(a, b)``, then a probability will be sampled per image\n from the range ``a <= x <= b``.\n * If a list, then a random value will be sampled from that list\n per image.\n * If a StochasticParameter, then this parameter will be used as\n the *mask*, i.e. it is expected to contain values between\n 0.0 and 1.0, where 1.0 means that pepper is to be added\n at that location.\n\n per_channel : bool or float, optional\n Whether to use the same value for all channels (False)\n or to sample a new value for each channel (True).\n If this value is a float ``p``, then for ``p`` percent of all images\n `per_channel` will be treated as True, otherwise as False.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Examples\n --------\n >>> aug = iaa.Pepper(0.05)\n\n Replaces 5 percent of all pixels with pepper.\n\n \"\"\"\n\n replacement01 = iap.ForceSign(\n iap.Beta(0.5, 0.5) - 0.5,\n positive=False,\n mode=\"invert\"\n ) + 0.5\n replacement = replacement01 * 255\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return ReplaceElementwise(\n mask=p,\n replacement=replacement,\n per_channel=per_channel,\n name=name,\n deterministic=deterministic,\n random_state=random_state\n )"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd coarse pepper noise to an image, i.e. rectangles that contain noisy black-ish pixels. dtype support:: See ``imgaug.augmenters.arithmetic.ReplaceElementwise``. Parameters ---------- p : float or tuple of float or list of float or imgaug.parameters.StochasticParameter, optional Probability of changing a pixel to pepper noise. * If a float, then that value will be used for all images as the probability. * If a tuple ``(a, b)``, then a probability will be sampled per image from the range ``a <= x <= b.`` * If a list, then a random value will be sampled from that list per image. * If a StochasticParameter, then this parameter will be used as the *mask*, i.e. it is expected to contain values between 0.0 and 1.0, where 1.0 means that pepper is to be added at that location. size_px : int or tuple of int or imgaug.parameters.StochasticParameter, optional The size of the lower resolution image from which to sample the noise mask in absolute pixel dimensions. * If an integer, then that size will be used for both height and width. E.g. a value of 3 would lead to a ``3x3`` mask, which is then upsampled to ``HxW``, where ``H`` is the image size and W the image width. * If a tuple ``(a, b)``, then two values ``M``, ``N`` will be sampled from the range ``[a..b]`` and the mask will be generated at size ``MxN``, then upsampled to ``HxW``. * If a StochasticParameter, then this parameter will be used to determine the sizes. It is expected to be discrete. size_percent : float or tuple of float or imgaug.parameters.StochasticParameter, optional The size of the lower resolution image from which to sample the noise mask *in percent* of the input image. * If a float, then that value will be used as the percentage of the height and width (relative to the original size). E.g. for value p, the mask will be sampled from ``(p*H)x(p*W)`` and later upsampled to ``HxW``. * If a tuple ``(a, b)``, then two values ``m``, ``n`` will be sampled from the interval ``(a, b)`` and used as the percentages, i.e the mask size will be ``(m*H)x(n*W)``. * If a StochasticParameter, then this parameter will be used to sample the percentage values. It is expected to be continuous. per_channel : bool or float, optional Whether to use the same value (is dropped / is not dropped) for all channels of a pixel (False) or to sample a new value for each channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel` will be treated as True, otherwise as False. min_size : int, optional Minimum size of the low resolution mask, both width and height. If `size_percent` or `size_px` leads to a lower value than this, `min_size` will be used instead. This should never have a value of less than 2, otherwise one may end up with a 1x1 low resolution mask, leading easily to the whole image being replaced. name : None or str, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. deterministic : bool, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. random_state : None or int or numpy.random.RandomState, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. Examples -------- >>> aug = iaa.CoarsePepper(0.05, size_percent=(0.01, 0.1)) Replaces 5 percent of all pixels with pepper in an image that has 1 to 10 percent of the input image size, then upscales the results to the input image size, leading to large rectangular areas being replaced.", "response": "def CoarsePepper(p=0, size_px=None, size_percent=None, per_channel=False, min_size=4, name=None, deterministic=False,\n random_state=None):\n \"\"\"\n Adds coarse pepper noise to an image, i.e. rectangles that contain noisy black-ish pixels.\n\n dtype support::\n\n See ``imgaug.augmenters.arithmetic.ReplaceElementwise``.\n\n Parameters\n ----------\n p : float or tuple of float or list of float or imgaug.parameters.StochasticParameter, optional\n Probability of changing a pixel to pepper noise.\n\n * If a float, then that value will be used for all images as the\n probability.\n * If a tuple ``(a, b)``, then a probability will be sampled per image\n from the range ``a <= x <= b.``\n * If a list, then a random value will be sampled from that list\n per image.\n * If a StochasticParameter, then this parameter will be used as\n the *mask*, i.e. it is expected to contain values between\n 0.0 and 1.0, where 1.0 means that pepper is to be added\n at that location.\n\n size_px : int or tuple of int or imgaug.parameters.StochasticParameter, optional\n The size of the lower resolution image from which to sample the noise\n mask in absolute pixel dimensions.\n\n * If an integer, then that size will be used for both height and\n width. E.g. a value of 3 would lead to a ``3x3`` mask, which is then\n upsampled to ``HxW``, where ``H`` is the image size and W the image width.\n * If a tuple ``(a, b)``, then two values ``M``, ``N`` will be sampled from the\n range ``[a..b]`` and the mask will be generated at size ``MxN``, then\n upsampled to ``HxW``.\n * If a StochasticParameter, then this parameter will be used to\n determine the sizes. It is expected to be discrete.\n\n size_percent : float or tuple of float or imgaug.parameters.StochasticParameter, optional\n The size of the lower resolution image from which to sample the noise\n mask *in percent* of the input image.\n\n * If a float, then that value will be used as the percentage of the\n height and width (relative to the original size). E.g. for value\n p, the mask will be sampled from ``(p*H)x(p*W)`` and later upsampled\n to ``HxW``.\n * If a tuple ``(a, b)``, then two values ``m``, ``n`` will be sampled from the\n interval ``(a, b)`` and used as the percentages, i.e the mask size\n will be ``(m*H)x(n*W)``.\n * If a StochasticParameter, then this parameter will be used to\n sample the percentage values. It is expected to be continuous.\n\n per_channel : bool or float, optional\n Whether to use the same value (is dropped / is not dropped)\n for all channels of a pixel (False) or to sample a new value for each\n channel (True).\n If this value is a float ``p``, then for ``p`` percent of all images\n `per_channel` will be treated as True, otherwise as False.\n\n min_size : int, optional\n Minimum size of the low resolution mask, both width and height. If\n `size_percent` or `size_px` leads to a lower value than this, `min_size`\n will be used instead. This should never have a value of less than 2,\n otherwise one may end up with a 1x1 low resolution mask, leading easily\n to the whole image being replaced.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Examples\n --------\n >>> aug = iaa.CoarsePepper(0.05, size_percent=(0.01, 0.1))\n\n Replaces 5 percent of all pixels with pepper in an image that has\n 1 to 10 percent of the input image size, then upscales the results\n to the input image size, leading to large rectangular areas being replaced.\n\n \"\"\"\n mask = iap.handle_probability_param(p, \"p\", tuple_to_uniform=True, list_to_choice=True)\n\n if size_px is not None:\n mask_low = iap.FromLowerResolution(other_param=mask, size_px=size_px, min_size=min_size)\n elif size_percent is not None:\n mask_low = iap.FromLowerResolution(other_param=mask, size_percent=size_percent, min_size=min_size)\n else:\n raise Exception(\"Either size_px or size_percent must be set.\")\n\n replacement01 = iap.ForceSign(\n iap.Beta(0.5, 0.5) - 0.5,\n positive=False,\n mode=\"invert\"\n ) + 0.5\n replacement = replacement01 * 255\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return ReplaceElementwise(\n mask=mask_low,\n replacement=replacement,\n per_channel=per_channel,\n name=name,\n deterministic=deterministic,\n random_state=random_state\n )"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck whether a variable is a float.", "response": "def is_single_float(val):\n \"\"\"\n Checks whether a variable is a float.\n\n Parameters\n ----------\n val\n The variable to check.\n\n Returns\n -------\n bool\n True if the variable is a float. Otherwise False.\n\n \"\"\"\n return isinstance(val, numbers.Real) and not is_single_integer(val) and not isinstance(val, bool)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef is_integer_array(val):\n return is_np_array(val) and issubclass(val.dtype.type, np.integer)", "response": "Checks whether a variable is a numpy integer array."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef is_float_array(val):\n return is_np_array(val) and issubclass(val.dtype.type, np.floating)", "response": "Checks whether a variable is a numpy float array."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck whether a variable is a callable.", "response": "def is_callable(val):\n \"\"\"\n Checks whether a variable is a callable, e.g. a function.\n\n Parameters\n ----------\n val\n The variable to check.\n\n Returns\n -------\n bool\n True if the variable is a callable. Otherwise False.\n\n \"\"\"\n # python 3.x with x <= 2 does not support callable(), apparently\n if sys.version_info[0] == 3 and sys.version_info[1] <= 2:\n return hasattr(val, '__call__')\n else:\n return callable(val)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef flatten(nested_iterable):\n # don't just check if something is iterable here, because then strings\n # and arrays will be split into their characters and components\n if not isinstance(nested_iterable, (list, tuple)):\n yield nested_iterable\n else:\n for i in nested_iterable:\n if isinstance(i, (list, tuple)):\n for j in flatten(i):\n yield j\n else:\n yield i", "response": "A generator that yields all the items in the nested_iterable."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef new_random_state(seed=None, fully_random=False):\n if seed is None:\n if not fully_random:\n # sample manually a seed instead of just RandomState(),\n # because the latter one\n # is way slower.\n seed = CURRENT_RANDOM_STATE.randint(SEED_MIN_VALUE, SEED_MAX_VALUE, 1)[0]\n return np.random.RandomState(seed)", "response": "Returns a new random state."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a copy of a random state.", "response": "def copy_random_state(random_state, force_copy=False):\n \"\"\"\n Creates a copy of a random state.\n\n Parameters\n ----------\n random_state : numpy.random.RandomState\n The random state to copy.\n\n force_copy : bool, optional\n If True, this function will always create a copy of every random\n state. If False, it will not copy numpy's default random state,\n but all other random states.\n\n Returns\n -------\n rs_copy : numpy.random.RandomState\n The copied random state.\n\n \"\"\"\n if random_state == np.random and not force_copy:\n return random_state\n else:\n rs_copy = dummy_random_state()\n orig_state = random_state.get_state()\n rs_copy.set_state(orig_state)\n return rs_copy"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates N new random states based on an existing random state or seed.", "response": "def derive_random_states(random_state, n=1):\n \"\"\"\n Create N new random states based on an existing random state or seed.\n\n Parameters\n ----------\n random_state : numpy.random.RandomState\n Random state or seed from which to derive new random states.\n\n n : int, optional\n Number of random states to derive.\n\n Returns\n -------\n list of numpy.random.RandomState\n Derived random states.\n\n \"\"\"\n seed_ = random_state.randint(SEED_MIN_VALUE, SEED_MAX_VALUE, 1)[0]\n return [new_random_state(seed_+i) for i in sm.xrange(n)]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nnormalize the extract parameter.", "response": "def _quokka_normalize_extract(extract):\n \"\"\"\n Generate a normalized rectangle to be extract from the standard quokka image.\n\n Parameters\n ----------\n extract : 'square' or tuple of number or imgaug.BoundingBox or imgaug.BoundingBoxesOnImage\n Unnormalized representation of the image subarea to be extracted.\n\n * If string ``square``, then a squared area ``(x: 0 to max 643, y: 0 to max 643)``\n will be extracted from the image.\n * If a tuple, then expected to contain four numbers denoting ``x1``, ``y1``, ``x2``\n and ``y2``.\n * If a BoundingBox, then that bounding box's area will be extracted from the image.\n * If a BoundingBoxesOnImage, then expected to contain exactly one bounding box\n and a shape matching the full image dimensions (i.e. (643, 960, *)). Then the\n one bounding box will be used similar to BoundingBox.\n\n Returns\n -------\n bb : imgaug.BoundingBox\n Normalized representation of the area to extract from the standard quokka image.\n\n \"\"\"\n # TODO get rid of this deferred import\n from imgaug.augmentables.bbs import BoundingBox, BoundingBoxesOnImage\n\n if extract == \"square\":\n bb = BoundingBox(x1=0, y1=0, x2=643, y2=643)\n elif isinstance(extract, tuple) and len(extract) == 4:\n bb = BoundingBox(x1=extract[0], y1=extract[1], x2=extract[2], y2=extract[3])\n elif isinstance(extract, BoundingBox):\n bb = extract\n elif isinstance(extract, BoundingBoxesOnImage):\n do_assert(len(extract.bounding_boxes) == 1)\n do_assert(extract.shape[0:2] == (643, 960))\n bb = extract.bounding_boxes[0]\n else:\n raise Exception(\n \"Expected 'square' or tuple of four entries or BoundingBox or BoundingBoxesOnImage \"\n + \"for parameter 'extract', got %s.\" % (type(extract),)\n )\n return bb"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _compute_resized_shape(from_shape, to_shape):\n if is_np_array(from_shape):\n from_shape = from_shape.shape\n if is_np_array(to_shape):\n to_shape = to_shape.shape\n\n to_shape_computed = list(from_shape)\n\n if to_shape is None:\n pass\n elif isinstance(to_shape, tuple):\n do_assert(len(from_shape) in [2, 3])\n do_assert(len(to_shape) in [2, 3])\n\n if len(from_shape) == 3 and len(to_shape) == 3:\n do_assert(from_shape[2] == to_shape[2])\n elif len(to_shape) == 3:\n to_shape_computed.append(to_shape[2])\n\n do_assert(all([v is None or is_single_number(v) for v in to_shape[0:2]]),\n \"Expected the first two entries in to_shape to be None or numbers, \"\n + \"got types %s.\" % (str([type(v) for v in to_shape[0:2]]),))\n\n for i, from_shape_i in enumerate(from_shape[0:2]):\n if to_shape[i] is None:\n to_shape_computed[i] = from_shape_i\n elif is_single_integer(to_shape[i]):\n to_shape_computed[i] = to_shape[i]\n else: # float\n to_shape_computed[i] = int(np.round(from_shape_i * to_shape[i]))\n elif is_single_integer(to_shape) or is_single_float(to_shape):\n to_shape_computed = _compute_resized_shape(from_shape, (to_shape, to_shape))\n else:\n raise Exception(\"Expected to_shape to be None or ndarray or tuple of floats or tuple of ints or single int \"\n + \"or single float, got %s.\" % (type(to_shape),))\n\n return tuple(to_shape_computed)", "response": "Compute the intended new shape of an image - like array after resizing."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns an image of a quokka image.", "response": "def quokka(size=None, extract=None):\n \"\"\"\n Returns an image of a quokka as a numpy array.\n\n Parameters\n ----------\n size : None or float or tuple of int, optional\n Size of the output image. Input into :func:`imgaug.imgaug.imresize_single_image`.\n Usually expected to be a tuple ``(H, W)``, where ``H`` is the desired height\n and ``W`` is the width. If None, then the image will not be resized.\n\n extract : None or 'square' or tuple of number or imgaug.BoundingBox or imgaug.BoundingBoxesOnImage\n Subarea of the quokka image to extract:\n\n * If None, then the whole image will be used.\n * If string ``square``, then a squared area ``(x: 0 to max 643, y: 0 to max 643)`` will\n be extracted from the image.\n * If a tuple, then expected to contain four numbers denoting ``x1``, ``y1``, ``x2``\n and ``y2``.\n * If a BoundingBox, then that bounding box's area will be extracted from the image.\n * If a BoundingBoxesOnImage, then expected to contain exactly one bounding box\n and a shape matching the full image dimensions (i.e. ``(643, 960, *)``). Then the\n one bounding box will be used similar to BoundingBox.\n\n Returns\n -------\n img : (H,W,3) ndarray\n The image array of dtype uint8.\n\n \"\"\"\n img = imageio.imread(QUOKKA_FP, pilmode=\"RGB\")\n if extract is not None:\n bb = _quokka_normalize_extract(extract)\n img = bb.extract_from_image(img)\n if size is not None:\n shape_resized = _compute_resized_shape(img.shape, size)\n img = imresize_single_image(img, shape_resized[0:2])\n return img"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef quokka_heatmap(size=None, extract=None):\n # TODO get rid of this deferred import\n from imgaug.augmentables.heatmaps import HeatmapsOnImage\n\n img = imageio.imread(QUOKKA_DEPTH_MAP_HALFRES_FP, pilmode=\"RGB\")\n img = imresize_single_image(img, (643, 960), interpolation=\"cubic\")\n\n if extract is not None:\n bb = _quokka_normalize_extract(extract)\n img = bb.extract_from_image(img)\n if size is None:\n size = img.shape[0:2]\n\n shape_resized = _compute_resized_shape(img.shape, size)\n img = imresize_single_image(img, shape_resized[0:2])\n img_0to1 = img[..., 0] # depth map was saved as 3-channel RGB\n img_0to1 = img_0to1.astype(np.float32) / 255.0\n img_0to1 = 1 - img_0to1 # depth map was saved as 0 being furthest away\n\n return HeatmapsOnImage(img_0to1, shape=img_0to1.shape[0:2] + (3,))", "response": "Returns a depth map for the standard example quokka image."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a segmentation map for the standard example quokka image.", "response": "def quokka_segmentation_map(size=None, extract=None):\n \"\"\"\n Returns a segmentation map for the standard example quokka image.\n\n Parameters\n ----------\n size : None or float or tuple of int, optional\n See :func:`imgaug.quokka`.\n\n extract : None or 'square' or tuple of number or imgaug.BoundingBox or imgaug.BoundingBoxesOnImage\n See :func:`imgaug.quokka`.\n\n Returns\n -------\n result : imgaug.SegmentationMapOnImage\n Segmentation map object.\n\n \"\"\"\n # TODO get rid of this deferred import\n from imgaug.augmentables.segmaps import SegmentationMapOnImage\n\n with open(QUOKKA_ANNOTATIONS_FP, \"r\") as f:\n json_dict = json.load(f)\n\n xx = []\n yy = []\n for kp_dict in json_dict[\"polygons\"][0][\"keypoints\"]:\n x = kp_dict[\"x\"]\n y = kp_dict[\"y\"]\n xx.append(x)\n yy.append(y)\n\n img_seg = np.zeros((643, 960, 1), dtype=np.float32)\n rr, cc = skimage.draw.polygon(np.array(yy), np.array(xx), shape=img_seg.shape)\n img_seg[rr, cc] = 1.0\n\n if extract is not None:\n bb = _quokka_normalize_extract(extract)\n img_seg = bb.extract_from_image(img_seg)\n\n segmap = SegmentationMapOnImage(img_seg, shape=img_seg.shape[0:2] + (3,))\n\n if size is not None:\n shape_resized = _compute_resized_shape(img_seg.shape, size)\n segmap = segmap.resize(shape_resized[0:2])\n segmap.shape = tuple(shape_resized[0:2]) + (3,)\n\n return segmap"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef quokka_keypoints(size=None, extract=None):\n # TODO get rid of this deferred import\n from imgaug.augmentables.kps import Keypoint, KeypointsOnImage\n\n left, top = 0, 0\n if extract is not None:\n bb_extract = _quokka_normalize_extract(extract)\n left = bb_extract.x1\n top = bb_extract.y1\n with open(QUOKKA_ANNOTATIONS_FP, \"r\") as f:\n json_dict = json.load(f)\n keypoints = []\n for kp_dict in json_dict[\"keypoints\"]:\n keypoints.append(Keypoint(x=kp_dict[\"x\"] - left, y=kp_dict[\"y\"] - top))\n if extract is not None:\n shape = (bb_extract.height, bb_extract.width, 3)\n else:\n shape = (643, 960, 3)\n kpsoi = KeypointsOnImage(keypoints, shape=shape)\n if size is not None:\n shape_resized = _compute_resized_shape(shape, size)\n kpsoi = kpsoi.on(shape_resized)\n return kpsoi", "response": "Returns example keypoints on the quokka image."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef quokka_bounding_boxes(size=None, extract=None):\n # TODO get rid of this deferred import\n from imgaug.augmentables.bbs import BoundingBox, BoundingBoxesOnImage\n\n left, top = 0, 0\n if extract is not None:\n bb_extract = _quokka_normalize_extract(extract)\n left = bb_extract.x1\n top = bb_extract.y1\n with open(QUOKKA_ANNOTATIONS_FP, \"r\") as f:\n json_dict = json.load(f)\n bbs = []\n for bb_dict in json_dict[\"bounding_boxes\"]:\n bbs.append(\n BoundingBox(\n x1=bb_dict[\"x1\"] - left,\n y1=bb_dict[\"y1\"] - top,\n x2=bb_dict[\"x2\"] - left,\n y2=bb_dict[\"y2\"] - top\n )\n )\n if extract is not None:\n shape = (bb_extract.height, bb_extract.width, 3)\n else:\n shape = (643, 960, 3)\n bbsoi = BoundingBoxesOnImage(bbs, shape=shape)\n if size is not None:\n shape_resized = _compute_resized_shape(shape, size)\n bbsoi = bbsoi.on(shape_resized)\n return bbsoi", "response": "Returns a list of bounding boxes that cover the quokka image."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning example polygons on the quokka image.", "response": "def quokka_polygons(size=None, extract=None):\n \"\"\"\n Returns example polygons on the standard example quokke image.\n\n The result contains one polygon, covering the quokka's outline.\n\n Parameters\n ----------\n size : None or float or tuple of int or tuple of float, optional\n Size of the output image on which the polygons are placed. If None,\n then the polygons are not projected to any new size (positions on the\n original image are used). Floats lead to relative size changes, ints\n to absolute sizes in pixels.\n\n extract : None or 'square' or tuple of number or imgaug.BoundingBox or \\\n imgaug.BoundingBoxesOnImage\n Subarea to extract from the image. See :func:`imgaug.quokka`.\n\n Returns\n -------\n psoi : imgaug.PolygonsOnImage\n Example polygons on the quokka image.\n\n \"\"\"\n # TODO get rid of this deferred import\n from imgaug.augmentables.polys import Polygon, PolygonsOnImage\n\n left, top = 0, 0\n if extract is not None:\n bb_extract = _quokka_normalize_extract(extract)\n left = bb_extract.x1\n top = bb_extract.y1\n with open(QUOKKA_ANNOTATIONS_FP, \"r\") as f:\n json_dict = json.load(f)\n polygons = []\n for poly_json in json_dict[\"polygons\"]:\n polygons.append(\n Polygon([(point[\"x\"] - left, point[\"y\"] - top)\n for point in poly_json[\"keypoints\"]])\n )\n if extract is not None:\n shape = (bb_extract.height, bb_extract.width, 3)\n else:\n shape = (643, 960, 3)\n psoi = PolygonsOnImage(polygons, shape=shape)\n if size is not None:\n shape_resized = _compute_resized_shape(shape, size)\n psoi = psoi.on(shape_resized)\n return psoi"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the angle in radians between vectors v1 and v2.", "response": "def angle_between_vectors(v1, v2):\n \"\"\"\n Returns the angle in radians between vectors `v1` and `v2`.\n\n From http://stackoverflow.com/questions/2827393/angles-between-two-n-dimensional-vectors-in-python\n\n Parameters\n ----------\n v1 : (N,) ndarray\n First vector.\n\n v2 : (N,) ndarray\n Second vector.\n\n Returns\n -------\n out : float\n Angle in radians.\n\n Examples\n --------\n >>> angle_between_vectors(np.float32([1, 0, 0]), np.float32([0, 1, 0]))\n 1.570796...\n\n >>> angle_between_vectors(np.float32([1, 0, 0]), np.float32([1, 0, 0]))\n 0.0\n\n >>> angle_between_vectors(np.float32([1, 0, 0]), np.float32([-1, 0, 0]))\n 3.141592...\n\n \"\"\"\n l1 = np.linalg.norm(v1)\n l2 = np.linalg.norm(v2)\n v1_u = (v1 / l1) if l1 > 0 else np.float32(v1) * 0\n v2_u = (v2 / l2) if l2 > 0 else np.float32(v2) * 0\n return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0))"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the intersection point of two lines.", "response": "def compute_line_intersection_point(x1, y1, x2, y2, x3, y3, x4, y4):\n \"\"\"\n Compute the intersection point of two lines.\n\n Taken from https://stackoverflow.com/a/20679579 .\n\n Parameters\n ----------\n x1 : number\n x coordinate of the first point on line 1. (The lines extends beyond this point.)\n\n y1 : number\n y coordinate of the first point on line 1. (The lines extends beyond this point.)\n\n x2 : number\n x coordinate of the second point on line 1. (The lines extends beyond this point.)\n\n y2 : number\n y coordinate of the second point on line 1. (The lines extends beyond this point.)\n\n x3 : number\n x coordinate of the first point on line 2. (The lines extends beyond this point.)\n\n y3 : number\n y coordinate of the first point on line 2. (The lines extends beyond this point.)\n\n x4 : number\n x coordinate of the second point on line 2. (The lines extends beyond this point.)\n\n y4 : number\n y coordinate of the second point on line 2. (The lines extends beyond this point.)\n\n Returns\n -------\n tuple of number or bool\n The coordinate of the intersection point as a tuple ``(x, y)``.\n If the lines are parallel (no intersection point or an infinite number of them), the result is False.\n\n \"\"\"\n def _make_line(p1, p2):\n A = (p1[1] - p2[1])\n B = (p2[0] - p1[0])\n C = (p1[0]*p2[1] - p2[0]*p1[1])\n return A, B, -C\n\n L1 = _make_line((x1, y1), (x2, y2))\n L2 = _make_line((x3, y3), (x4, y4))\n\n D = L1[0] * L2[1] - L1[1] * L2[0]\n Dx = L1[2] * L2[1] - L1[1] * L2[2]\n Dy = L1[0] * L2[2] - L1[2] * L2[0]\n if D != 0:\n x = Dx / D\n y = Dy / D\n return x, y\n else:\n return False"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndrawing text on an image.", "response": "def draw_text(img, y, x, text, color=(0, 255, 0), size=25):\n \"\"\"\n Draw text on an image.\n\n This uses by default DejaVuSans as its font, which is included in this library.\n\n dtype support::\n\n * ``uint8``: yes; fully tested\n * ``uint16``: no\n * ``uint32``: no\n * ``uint64``: no\n * ``int8``: no\n * ``int16``: no\n * ``int32``: no\n * ``int64``: no\n * ``float16``: no\n * ``float32``: yes; not tested\n * ``float64``: no\n * ``float128``: no\n * ``bool``: no\n\n TODO check if other dtypes could be enabled\n\n Parameters\n ----------\n img : (H,W,3) ndarray\n The image array to draw text on.\n Expected to be of dtype uint8 or float32 (value range 0.0 to 255.0).\n\n y : int\n x-coordinate of the top left corner of the text.\n\n x : int\n y- coordinate of the top left corner of the text.\n\n text : str\n The text to draw.\n\n color : iterable of int, optional\n Color of the text to draw. For RGB-images this is expected to be an RGB color.\n\n size : int, optional\n Font size of the text to draw.\n\n Returns\n -------\n img_np : (H,W,3) ndarray\n Input image with text drawn on it.\n\n \"\"\"\n do_assert(img.dtype in [np.uint8, np.float32])\n\n input_dtype = img.dtype\n if img.dtype == np.float32:\n img = img.astype(np.uint8)\n\n img = PIL_Image.fromarray(img)\n font = PIL_ImageFont.truetype(DEFAULT_FONT_FP, size)\n context = PIL_ImageDraw.Draw(img)\n context.text((x, y), text, fill=tuple(color), font=font)\n img_np = np.asarray(img)\n\n # PIL/asarray returns read only array\n if not img_np.flags[\"WRITEABLE\"]:\n try:\n # this seems to no longer work with np 1.16 (or was pillow updated?)\n img_np.setflags(write=True)\n except ValueError as ex:\n if \"cannot set WRITEABLE flag to True of this array\" in str(ex):\n img_np = np.copy(img_np)\n\n if img_np.dtype != input_dtype:\n img_np = img_np.astype(input_dtype)\n\n return img_np"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef imresize_single_image(image, sizes, interpolation=None):\n grayscale = False\n if image.ndim == 2:\n grayscale = True\n image = image[:, :, np.newaxis]\n do_assert(len(image.shape) == 3, image.shape)\n rs = imresize_many_images(image[np.newaxis, :, :, :], sizes, interpolation=interpolation)\n if grayscale:\n return np.squeeze(rs[0, :, :, 0])\n else:\n return rs[0, ...]", "response": "Resizes a single image."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pad(arr, top=0, right=0, bottom=0, left=0, mode=\"constant\", cval=0):\n do_assert(arr.ndim in [2, 3])\n do_assert(top >= 0)\n do_assert(right >= 0)\n do_assert(bottom >= 0)\n do_assert(left >= 0)\n if top > 0 or right > 0 or bottom > 0 or left > 0:\n mapping_mode_np_to_cv2 = {\n \"constant\": cv2.BORDER_CONSTANT,\n \"edge\": cv2.BORDER_REPLICATE,\n \"linear_ramp\": None,\n \"maximum\": None,\n \"mean\": None,\n \"median\": None,\n \"minimum\": None,\n \"reflect\": cv2.BORDER_REFLECT_101,\n \"symmetric\": cv2.BORDER_REFLECT,\n \"wrap\": None,\n cv2.BORDER_CONSTANT: cv2.BORDER_CONSTANT,\n cv2.BORDER_REPLICATE: cv2.BORDER_REPLICATE,\n cv2.BORDER_REFLECT_101: cv2.BORDER_REFLECT_101,\n cv2.BORDER_REFLECT: cv2.BORDER_REFLECT\n }\n bad_mode_cv2 = mapping_mode_np_to_cv2.get(mode, None) is None\n\n # these datatypes all simply generate a \"TypeError: src data type = X is not supported\" error\n bad_datatype_cv2 = arr.dtype.name in [\"uint32\", \"uint64\", \"int64\", \"float16\", \"float128\", \"bool\"]\n\n if not bad_datatype_cv2 and not bad_mode_cv2:\n cval = float(cval) if arr.dtype.kind == \"f\" else int(cval) # results in TypeError otherwise for np inputs\n\n if arr.ndim == 2 or arr.shape[2] <= 4:\n # without this, only the first channel is padded with the cval, all following channels with 0\n if arr.ndim == 3:\n cval = tuple([cval] * arr.shape[2])\n\n arr_pad = cv2.copyMakeBorder(arr, top=top, bottom=bottom, left=left, right=right,\n borderType=mapping_mode_np_to_cv2[mode], value=cval)\n if arr.ndim == 3 and arr_pad.ndim == 2:\n arr_pad = arr_pad[..., np.newaxis]\n else:\n result = []\n channel_start_idx = 0\n while channel_start_idx < arr.shape[2]:\n arr_c = arr[..., channel_start_idx:channel_start_idx+4]\n cval_c = tuple([cval] * arr_c.shape[2])\n arr_pad_c = cv2.copyMakeBorder(arr_c, top=top, bottom=bottom, left=left, right=right,\n borderType=mapping_mode_np_to_cv2[mode], value=cval_c)\n arr_pad_c = np.atleast_3d(arr_pad_c)\n result.append(arr_pad_c)\n channel_start_idx += 4\n arr_pad = np.concatenate(result, axis=2)\n else:\n paddings_np = [(top, bottom), (left, right)] # paddings for 2d case\n if arr.ndim == 3:\n paddings_np.append((0, 0)) # add paddings for 3d case\n\n if mode == \"constant\":\n arr_pad = np.pad(arr, paddings_np, mode=mode, constant_values=cval)\n elif mode == \"linear_ramp\":\n arr_pad = np.pad(arr, paddings_np, mode=mode, end_values=cval)\n else:\n arr_pad = np.pad(arr, paddings_np, mode=mode)\n\n return arr_pad\n return np.copy(arr)", "response": "Pad an image - like array on its top right bottom left side."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the amount of pixels by which an array has to be padded to fulfill an aspect ratio.", "response": "def compute_paddings_for_aspect_ratio(arr, aspect_ratio):\n \"\"\"\n Compute the amount of pixels by which an array has to be padded to fulfill an aspect ratio.\n\n The aspect ratio is given as width/height.\n Depending on which dimension is smaller (height or width), only the corresponding\n sides (left/right or top/bottom) will be padded. In each case, both of the sides will\n be padded equally.\n\n Parameters\n ----------\n arr : (H,W) ndarray or (H,W,C) ndarray\n Image-like array for which to compute pad amounts.\n\n aspect_ratio : float\n Target aspect ratio, given as width/height. E.g. 2.0 denotes the image having twice\n as much width as height.\n\n Returns\n -------\n result : tuple of int\n Required paddign amounts to reach the target aspect ratio, given as a tuple\n of the form ``(top, right, bottom, left)``.\n\n \"\"\"\n do_assert(arr.ndim in [2, 3])\n do_assert(aspect_ratio > 0)\n height, width = arr.shape[0:2]\n do_assert(height > 0)\n aspect_ratio_current = width / height\n\n pad_top = 0\n pad_right = 0\n pad_bottom = 0\n pad_left = 0\n\n if aspect_ratio_current < aspect_ratio:\n # vertical image, height > width\n diff = (aspect_ratio * height) - width\n pad_right = int(np.ceil(diff / 2))\n pad_left = int(np.floor(diff / 2))\n elif aspect_ratio_current > aspect_ratio:\n # horizontal image, width > height\n diff = ((1/aspect_ratio) * width) - height\n pad_top = int(np.floor(diff / 2))\n pad_bottom = int(np.ceil(diff / 2))\n\n return pad_top, pad_right, pad_bottom, pad_left"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pad_to_aspect_ratio(arr, aspect_ratio, mode=\"constant\", cval=0, return_pad_amounts=False):\n pad_top, pad_right, pad_bottom, pad_left = compute_paddings_for_aspect_ratio(arr, aspect_ratio)\n arr_padded = pad(\n arr,\n top=pad_top,\n right=pad_right,\n bottom=pad_bottom,\n left=pad_left,\n mode=mode,\n cval=cval\n )\n\n if return_pad_amounts:\n return arr_padded, (pad_top, pad_right, pad_bottom, pad_left)\n else:\n return arr_padded", "response": "Pad an image - like array to match a target aspect ratio."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pool(arr, block_size, func, cval=0, preserve_dtype=True):\n # TODO find better way to avoid circular import\n from . import dtypes as iadt\n iadt.gate_dtypes(arr,\n allowed=[\"bool\", \"uint8\", \"uint16\", \"uint32\", \"int8\", \"int16\", \"int32\",\n \"float16\", \"float32\", \"float64\", \"float128\"],\n disallowed=[\"uint64\", \"uint128\", \"uint256\", \"int64\", \"int128\", \"int256\",\n \"float256\"],\n augmenter=None)\n\n do_assert(arr.ndim in [2, 3])\n is_valid_int = is_single_integer(block_size) and block_size >= 1\n is_valid_tuple = is_iterable(block_size) and len(block_size) in [2, 3] \\\n and [is_single_integer(val) and val >= 1 for val in block_size]\n do_assert(is_valid_int or is_valid_tuple)\n\n if is_single_integer(block_size):\n block_size = [block_size, block_size]\n if len(block_size) < arr.ndim:\n block_size = list(block_size) + [1]\n\n input_dtype = arr.dtype\n arr_reduced = skimage.measure.block_reduce(arr, tuple(block_size), func, cval=cval)\n if preserve_dtype and arr_reduced.dtype.type != input_dtype:\n arr_reduced = arr_reduced.astype(input_dtype)\n return arr_reduced", "response": "Wrapper for the pooling function for the array - like object."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef avg_pool(arr, block_size, cval=0, preserve_dtype=True):\n return pool(arr, block_size, np.average, cval=cval, preserve_dtype=preserve_dtype)", "response": "Resize an array using average pooling."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef draw_grid(images, rows=None, cols=None):\n nb_images = len(images)\n do_assert(nb_images > 0)\n\n if is_np_array(images):\n do_assert(images.ndim == 4)\n else:\n do_assert(is_iterable(images) and is_np_array(images[0]) and images[0].ndim == 3)\n dts = [image.dtype.name for image in images]\n nb_dtypes = len(set(dts))\n do_assert(nb_dtypes == 1, (\"All images provided to draw_grid() must have the same dtype, \"\n + \"found %d dtypes (%s)\") % (nb_dtypes, \", \".join(dts)))\n\n cell_height = max([image.shape[0] for image in images])\n cell_width = max([image.shape[1] for image in images])\n channels = set([image.shape[2] for image in images])\n do_assert(\n len(channels) == 1,\n \"All images are expected to have the same number of channels, \"\n + \"but got channel set %s with length %d instead.\" % (str(channels), len(channels))\n )\n nb_channels = list(channels)[0]\n if rows is None and cols is None:\n rows = cols = int(math.ceil(math.sqrt(nb_images)))\n elif rows is not None:\n cols = int(math.ceil(nb_images / rows))\n elif cols is not None:\n rows = int(math.ceil(nb_images / cols))\n do_assert(rows * cols >= nb_images)\n\n width = cell_width * cols\n height = cell_height * rows\n dt = images.dtype if is_np_array(images) else images[0].dtype\n grid = np.zeros((height, width, nb_channels), dtype=dt)\n cell_idx = 0\n for row_idx in sm.xrange(rows):\n for col_idx in sm.xrange(cols):\n if cell_idx < nb_images:\n image = images[cell_idx]\n cell_y1 = cell_height * row_idx\n cell_y2 = cell_y1 + image.shape[0]\n cell_x1 = cell_width * col_idx\n cell_x2 = cell_x1 + image.shape[1]\n grid[cell_y1:cell_y2, cell_x1:cell_x2, :] = image\n cell_idx += 1\n\n return grid", "response": "Generates a new image grid from a list of images."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef show_grid(images, rows=None, cols=None):\n grid = draw_grid(images, rows=rows, cols=cols)\n imshow(grid)", "response": "Converts the input images to a grid image and shows it in a new window."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndisplay an image in a window.", "response": "def imshow(image, backend=IMSHOW_BACKEND_DEFAULT):\n \"\"\"\n Shows an image in a window.\n\n dtype support::\n\n * ``uint8``: yes; not tested\n * ``uint16``: ?\n * ``uint32``: ?\n * ``uint64``: ?\n * ``int8``: ?\n * ``int16``: ?\n * ``int32``: ?\n * ``int64``: ?\n * ``float16``: ?\n * ``float32``: ?\n * ``float64``: ?\n * ``float128``: ?\n * ``bool``: ?\n\n Parameters\n ----------\n image : (H,W,3) ndarray\n Image to show.\n\n backend : {'matplotlib', 'cv2'}, optional\n Library to use to show the image. May be either matplotlib or OpenCV ('cv2').\n OpenCV tends to be faster, but apparently causes more technical issues.\n\n \"\"\"\n do_assert(backend in [\"matplotlib\", \"cv2\"], \"Expected backend 'matplotlib' or 'cv2', got %s.\" % (backend,))\n\n if backend == \"cv2\":\n image_bgr = image\n if image.ndim == 3 and image.shape[2] in [3, 4]:\n image_bgr = image[..., 0:3][..., ::-1]\n\n win_name = \"imgaug-default-window\"\n cv2.namedWindow(win_name, cv2.WINDOW_NORMAL)\n cv2.imshow(win_name, image_bgr)\n cv2.waitKey(0)\n cv2.destroyWindow(win_name)\n else:\n # import only when necessary (faster startup; optional dependency; less fragile -- see issue #225)\n import matplotlib.pyplot as plt\n\n dpi = 96\n h, w = image.shape[0] / dpi, image.shape[1] / dpi\n w = max(w, 6) # if the figure is too narrow, the footer may appear and make the fig suddenly wider (ugly)\n fig, ax = plt.subplots(figsize=(w, h), dpi=dpi)\n fig.canvas.set_window_title(\"imgaug.imshow(%s)\" % (image.shape,))\n ax.imshow(image, cmap=\"gray\") # cmap is only activate for grayscale images\n plt.show()"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating a non - silent deprecation warning with stacktrace.", "response": "def warn_deprecated(msg, stacklevel=2):\n \"\"\"Generate a non-silent deprecation warning with stacktrace.\n\n The used warning is ``imgaug.imgaug.DeprecationWarning``.\n\n Parameters\n ----------\n msg : str\n The message of the warning.\n\n stacklevel : int, optional\n How many steps above this function to \"jump\" in the stacktrace for\n the displayed file and line number of the error message.\n Usually 2.\n\n \"\"\"\n import warnings\n warnings.warn(msg,\n category=DeprecationWarning,\n stacklevel=stacklevel)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef is_activated(self, images, augmenter, parents, default):\n if self.activator is None:\n return default\n else:\n return self.activator(images, augmenter, parents, default)", "response": "Returns whether the image is activated."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn whether an augmenter may call its children to propagate the an image.", "response": "def is_propagating(self, images, augmenter, parents, default):\n \"\"\"\n Returns whether an augmenter may call its children to augment an\n image. This is independent of the augmenter itself possible changing\n the image, without calling its children. (Most (all?) augmenters with\n children currently dont perform any changes themselves.)\n\n Returns\n -------\n bool\n If True, the augmenter may be propagate to its children. If False, it may not.\n\n \"\"\"\n if self.propagator is None:\n return default\n else:\n return self.propagator(images, augmenter, parents, default)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the multiprocessing. Pool instance or create it if not done yet.", "response": "def pool(self):\n \"\"\"Return the multiprocessing.Pool instance or create it if not done yet.\n\n Returns\n -------\n multiprocessing.Pool\n The multiprocessing.Pool used internally by this imgaug.multicore.Pool.\n\n \"\"\"\n if self._pool is None:\n processes = self.processes\n if processes is not None and processes < 0:\n try:\n # cpu count includes the hyperthreads, e.g. 8 for 4 cores + hyperthreading\n processes = multiprocessing.cpu_count() - abs(processes)\n processes = max(processes, 1)\n except (ImportError, NotImplementedError):\n processes = None\n\n self._pool = multiprocessing.Pool(processes,\n initializer=_Pool_initialize_worker,\n initargs=(self.augseq, self.seed),\n maxtasksperchild=self.maxtasksperchild)\n return self._pool"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nmapping a list of batches into a list of unique keys.", "response": "def map_batches(self, batches, chunksize=None):\n \"\"\"\n Augment batches.\n\n Parameters\n ----------\n batches : list of imgaug.augmentables.batches.Batch\n The batches to augment.\n\n chunksize : None or int, optional\n Rough indicator of how many tasks should be sent to each worker. Increasing this number can improve\n performance.\n\n Returns\n -------\n list of imgaug.augmentables.batches.Batch\n Augmented batches.\n\n \"\"\"\n assert isinstance(batches, list), (\"Expected to get a list as 'batches', got type %s. \"\n + \"Call imap_batches() if you use generators.\") % (type(batches),)\n return self.pool.map(_Pool_starworker, self._handle_batch_ids(batches), chunksize=chunksize)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\naugments batches asynchonously. Parameters ---------- batches : list of imgaug.augmentables.batches.Batch The batches to augment. chunksize : None or int, optional Rough indicator of how many tasks should be sent to each worker. Increasing this number can improve performance. callback : None or callable, optional Function to call upon finish. See `multiprocessing.Pool`. error_callback : None or callable, optional Function to call upon errors. See `multiprocessing.Pool`. Returns ------- multiprocessing.MapResult Asynchonous result. See `multiprocessing.Pool`.", "response": "def map_batches_async(self, batches, chunksize=None, callback=None, error_callback=None):\n \"\"\"\n Augment batches asynchonously.\n\n Parameters\n ----------\n batches : list of imgaug.augmentables.batches.Batch\n The batches to augment.\n\n chunksize : None or int, optional\n Rough indicator of how many tasks should be sent to each worker. Increasing this number can improve\n performance.\n\n callback : None or callable, optional\n Function to call upon finish. See `multiprocessing.Pool`.\n\n error_callback : None or callable, optional\n Function to call upon errors. See `multiprocessing.Pool`.\n\n Returns\n -------\n multiprocessing.MapResult\n Asynchonous result. See `multiprocessing.Pool`.\n\n \"\"\"\n assert isinstance(batches, list), (\"Expected to get a list as 'batches', got type %s. \"\n + \"Call imap_batches() if you use generators.\") % (type(batches),)\n return self.pool.map_async(_Pool_starworker, self._handle_batch_ids(batches),\n chunksize=chunksize, callback=callback, error_callback=error_callback)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef imap_batches(self, batches, chunksize=1):\n assert ia.is_generator(batches), (\"Expected to get a generator as 'batches', got type %s. \"\n + \"Call map_batches() if you use lists.\") % (type(batches),)\n # TODO change this to 'yield from' once switched to 3.3+\n gen = self.pool.imap(_Pool_starworker, self._handle_batch_ids_gen(batches), chunksize=chunksize)\n for batch in gen:\n yield batch", "response": "Yields a set of batch objects from a generator."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef imap_batches_unordered(self, batches, chunksize=1):\n assert ia.is_generator(batches), (\"Expected to get a generator as 'batches', got type %s. \"\n + \"Call map_batches() if you use lists.\") % (type(batches),)\n # TODO change this to 'yield from' once switched to 3.3+\n gen = self.pool.imap_unordered(_Pool_starworker, self._handle_batch_ids_gen(batches), chunksize=chunksize)\n for batch in gen:\n yield batch", "response": "Yields a set of batch objects from a generator of batches."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef terminate(self):\n if self._pool is not None:\n self._pool.terminate()\n self._pool.join()\n self._pool = None", "response": "Terminate the pool immediately."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef terminate(self):\n if not self.join_signal.is_set():\n self.join_signal.set()\n # give minimal time to put generated batches in queue and gracefully shut down\n time.sleep(0.01)\n\n if self.main_worker_thread.is_alive():\n self.main_worker_thread.join()\n\n if self.threaded:\n for worker in self.workers:\n if worker.is_alive():\n worker.join()\n else:\n for worker in self.workers:\n if worker.is_alive():\n worker.terminate()\n worker.join()\n\n # wait until all workers are fully terminated\n while not self.all_finished():\n time.sleep(0.001)\n\n # empty queue until at least one element can be added and place None as signal that BL finished\n if self.queue.full():\n self.queue.get()\n self.queue.put(pickle.dumps(None, protocol=-1))\n time.sleep(0.01)\n\n # clean the queue, this reportedly prevents hanging threads\n while True:\n try:\n self._queue_internal.get(timeout=0.005)\n except QueueEmpty:\n break\n\n if not self._queue_internal._closed:\n self._queue_internal.close()\n if not self.queue._closed:\n self.queue.close()\n self._queue_internal.join_thread()\n self.queue.join_thread()\n time.sleep(0.025)", "response": "Terminate all workers and clean the queue."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_batch(self):\n if self.all_finished():\n return None\n\n batch_str = self.queue_result.get()\n batch = pickle.loads(batch_str)\n if batch is not None:\n return batch\n else:\n self.nb_workers_finished += 1\n if self.nb_workers_finished >= self.nb_workers:\n try:\n self.queue_source.get(timeout=0.001) # remove the None from the source queue\n except QueueEmpty:\n pass\n return None\n else:\n return self.get_batch()", "response": "Returns a batch from the queue of augmented batches."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _augment_images_worker(self, augseq, queue_source, queue_result, seedval):\n np.random.seed(seedval)\n random.seed(seedval)\n augseq.reseed(seedval)\n ia.seed(seedval)\n\n loader_finished = False\n\n while not loader_finished:\n # wait for a new batch in the source queue and load it\n try:\n batch_str = queue_source.get(timeout=0.1)\n batch = pickle.loads(batch_str)\n if batch is None:\n loader_finished = True\n # put it back in so that other workers know that the loading queue is finished\n queue_source.put(pickle.dumps(None, protocol=-1))\n else:\n batch_aug = augseq.augment_batch(batch)\n\n # send augmented batch to output queue\n batch_str = pickle.dumps(batch_aug, protocol=-1)\n queue_result.put(batch_str)\n except QueueEmpty:\n time.sleep(0.01)\n\n queue_result.put(pickle.dumps(None, protocol=-1))\n time.sleep(0.01)", "response": "Augment endlessly images in the source queue and send the augmented images to the output queue."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef terminate(self):\n for worker in self.workers:\n if worker.is_alive():\n worker.terminate()\n self.nb_workers_finished = len(self.workers)\n\n if not self.queue_result._closed:\n self.queue_result.close()\n time.sleep(0.01)", "response": "Terminates all background processes immediately."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting this unnormalized batch to an instance of Batch.", "response": "def to_normalized_batch(self):\n \"\"\"Convert this unnormalized batch to an instance of Batch.\n\n As this method is intended to be called before augmentation, it\n assumes that none of the ``*_aug`` attributes is yet set.\n It will produce an AssertionError otherwise.\n\n The newly created Batch's ``*_unaug`` attributes will match the ones\n in this batch, just in normalized form.\n\n Returns\n -------\n imgaug.augmentables.batches.Batch\n The batch, with ``*_unaug`` attributes being normalized.\n\n \"\"\"\n assert all([\n attr is None for attr_name, attr in self.__dict__.items()\n if attr_name.endswith(\"_aug\")]), \\\n \"Expected UnnormalizedBatch to not contain any augmented data \" \\\n \"before normalization, but at least one '*_aug' attribute was \" \\\n \"already set.\"\n\n images_unaug = nlib.normalize_images(self.images_unaug)\n shapes = None\n if images_unaug is not None:\n shapes = [image.shape for image in images_unaug]\n\n return Batch(\n images=images_unaug,\n heatmaps=nlib.normalize_heatmaps(\n self.heatmaps_unaug, shapes),\n segmentation_maps=nlib.normalize_segmentation_maps(\n self.segmentation_maps_unaug, shapes),\n keypoints=nlib.normalize_keypoints(\n self.keypoints_unaug, shapes),\n bounding_boxes=nlib.normalize_bounding_boxes(\n self.bounding_boxes_unaug, shapes),\n polygons=nlib.normalize_polygons(\n self.polygons_unaug, shapes),\n line_strings=nlib.normalize_line_strings(\n self.line_strings_unaug, shapes),\n data=self.data\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fill_from_augmented_normalized_batch(self, batch_aug_norm):\n # we take here the .data from the normalized batch instead of from\n # self for the rare case where one has decided to somehow change it\n # during augmentation\n batch = UnnormalizedBatch(\n images=self.images_unaug,\n heatmaps=self.heatmaps_unaug,\n segmentation_maps=self.segmentation_maps_unaug,\n keypoints=self.keypoints_unaug,\n bounding_boxes=self.bounding_boxes_unaug,\n polygons=self.polygons_unaug,\n line_strings=self.line_strings_unaug,\n data=batch_aug_norm.data\n )\n\n batch.images_aug = nlib.invert_normalize_images(\n batch_aug_norm.images_aug, self.images_unaug)\n batch.heatmaps_aug = nlib.invert_normalize_heatmaps(\n batch_aug_norm.heatmaps_aug, self.heatmaps_unaug)\n batch.segmentation_maps_aug = nlib.invert_normalize_segmentation_maps(\n batch_aug_norm.segmentation_maps_aug, self.segmentation_maps_unaug)\n batch.keypoints_aug = nlib.invert_normalize_keypoints(\n batch_aug_norm.keypoints_aug, self.keypoints_unaug)\n batch.bounding_boxes_aug = nlib.invert_normalize_bounding_boxes(\n batch_aug_norm.bounding_boxes_aug, self.bounding_boxes_unaug)\n batch.polygons_aug = nlib.invert_normalize_polygons(\n batch_aug_norm.polygons_aug, self.polygons_unaug)\n batch.line_strings_aug = nlib.invert_normalize_line_strings(\n batch_aug_norm.line_strings_aug, self.line_strings_unaug)\n\n return batch", "response": "Fill this batch with augmented values."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Positive(other_param, mode=\"invert\", reroll_count_max=2):\n return ForceSign(\n other_param=other_param,\n positive=True,\n mode=mode,\n reroll_count_max=reroll_count_max\n )", "response": "Returns a new randomized version of the other parameter s results to positive values."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a new randomized version of the other parameter s results with negative values.", "response": "def Negative(other_param, mode=\"invert\", reroll_count_max=2):\n \"\"\"\n Converts another parameter's results to negative values.\n\n Parameters\n ----------\n other_param : imgaug.parameters.StochasticParameter\n Other parameter which's sampled values are to be\n modified.\n\n mode : {'invert', 'reroll'}, optional\n How to change the signs. Valid values are ``invert`` and ``reroll``.\n ``invert`` means that wrong signs are simply flipped.\n ``reroll`` means that all samples with wrong signs are sampled again,\n optionally many times, until they randomly end up having the correct\n sign.\n\n reroll_count_max : int, optional\n If `mode` is set to ``reroll``, this determines how often values may\n be rerolled before giving up and simply flipping the sign (as in\n ``mode=\"invert\"``). This shouldn't be set too high, as rerolling is\n expensive.\n\n Examples\n --------\n >>> param = Negative(Normal(0, 1), mode=\"reroll\")\n\n Generates a normal distribution that has only negative values.\n\n \"\"\"\n return ForceSign(\n other_param=other_param,\n positive=False,\n mode=mode,\n reroll_count_max=reroll_count_max\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_for_noise(other_param, threshold=(-10, 10), activated=True):\n return Sigmoid(other_param, threshold, activated, mul=20, add=-10)", "response": "Creates a Sigmoid that is adjusted to be used with noise parameters."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the area of the polygon.", "response": "def area(self):\n \"\"\"\n Estimate the area of the polygon.\n\n Returns\n -------\n number\n Area of the polygon.\n\n \"\"\"\n if len(self.exterior) < 3:\n raise Exception(\"Cannot compute the polygon's area because it contains less than three points.\")\n poly = self.to_shapely_polygon()\n return poly.area"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nproject the polygon onto an image with different shape.", "response": "def project(self, from_shape, to_shape):\n \"\"\"\n Project the polygon onto an image with different shape.\n\n The relative coordinates of all points remain the same.\n E.g. a point at (x=20, y=20) on an image (width=100, height=200) will be\n projected on a new image (width=200, height=100) to (x=40, y=10).\n\n This is intended for cases where the original image is resized.\n It cannot be used for more complex changes (e.g. padding, cropping).\n\n Parameters\n ----------\n from_shape : tuple of int\n Shape of the original image. (Before resize.)\n\n to_shape : tuple of int\n Shape of the new image. (After resize.)\n\n Returns\n -------\n imgaug.Polygon\n Polygon object with new coordinates.\n\n \"\"\"\n if from_shape[0:2] == to_shape[0:2]:\n return self.copy()\n ls_proj = self.to_line_string(closed=False).project(\n from_shape, to_shape)\n return self.copy(exterior=ls_proj.coords)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find_closest_point_index(self, x, y, return_distance=False):\n ia.do_assert(len(self.exterior) > 0)\n distances = []\n for x2, y2 in self.exterior:\n d = (x2 - x) ** 2 + (y2 - y) ** 2\n distances.append(d)\n distances = np.sqrt(distances)\n closest_idx = np.argmin(distances)\n if return_distance:\n return closest_idx, distances[closest_idx]\n return closest_idx", "response": "Find the index of the point within the exterior that is closest to the given coordinates."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef is_fully_within_image(self, image):\n return not self.is_out_of_image(image, fully=True, partly=True)", "response": "Return True if the polygon is fully inside the image area."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef is_partly_within_image(self, image):\n return not self.is_out_of_image(image, fully=True, partly=False)", "response": "Return True if the polygon is at least partially inside the image area."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef is_out_of_image(self, image, fully=True, partly=False):\n # TODO this is inconsistent with line strings, which return a default\n # value in these cases\n if len(self.exterior) == 0:\n raise Exception(\"Cannot determine whether the polygon is inside the image, because it contains no points.\")\n ls = self.to_line_string()\n return ls.is_out_of_image(image, fully=fully, partly=partly)", "response": "Return True if the polygon is inside the image area."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nclipping the image of the current image into a list of shapely. Polygons that are part of the image are clipped to fall within the image plane.", "response": "def clip_out_of_image(self, image):\n \"\"\"\n Cut off all parts of the polygon that are outside of the image.\n\n This operation may lead to new points being created.\n As a single polygon may be split into multiple new polygons, the result\n is always a list, which may contain more than one output polygon.\n\n This operation will return an empty list if the polygon is completely\n outside of the image plane.\n\n Parameters\n ----------\n image : (H,W,...) ndarray or tuple of int\n Image dimensions to use for the clipping of the polygon.\n If an ndarray, its shape will be used.\n If a tuple, it is assumed to represent the image shape and must\n contain at least two integers.\n\n Returns\n -------\n list of imgaug.Polygon\n Polygon, clipped to fall within the image dimensions.\n Returned as a list, because the clipping can split the polygon into\n multiple parts. The list may also be empty, if the polygon was\n fully outside of the image plane.\n\n \"\"\"\n # load shapely lazily, which makes the dependency more optional\n import shapely.geometry\n\n # if fully out of image, clip everything away, nothing remaining\n if self.is_out_of_image(image, fully=True, partly=False):\n return []\n\n h, w = image.shape[0:2] if ia.is_np_array(image) else image[0:2]\n poly_shapely = self.to_shapely_polygon()\n poly_image = shapely.geometry.Polygon([(0, 0), (w, 0), (w, h), (0, h)])\n multipoly_inter_shapely = poly_shapely.intersection(poly_image)\n if not isinstance(multipoly_inter_shapely, shapely.geometry.MultiPolygon):\n ia.do_assert(isinstance(multipoly_inter_shapely, shapely.geometry.Polygon))\n multipoly_inter_shapely = shapely.geometry.MultiPolygon([multipoly_inter_shapely])\n\n polygons = []\n for poly_inter_shapely in multipoly_inter_shapely.geoms:\n polygons.append(Polygon.from_shapely(poly_inter_shapely, label=self.label))\n\n # shapely changes the order of points, we try here to preserve it as\n # much as possible\n polygons_reordered = []\n for polygon in polygons:\n found = False\n for x, y in self.exterior:\n closest_idx, dist = polygon.find_closest_point_index(x=x, y=y, return_distance=True)\n if dist < 1e-6:\n polygon_reordered = polygon.change_first_point_by_index(closest_idx)\n polygons_reordered.append(polygon_reordered)\n found = True\n break\n ia.do_assert(found) # could only not find closest points if new polys are empty\n\n return polygons_reordered"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nshifts the polygon from one or more image sides i. e. move it on the x - axis.", "response": "def shift(self, top=None, right=None, bottom=None, left=None):\n \"\"\"\n Shift the polygon from one or more image sides, i.e. move it on the x/y-axis.\n\n Parameters\n ----------\n top : None or int, optional\n Amount of pixels by which to shift the polygon from the top.\n\n right : None or int, optional\n Amount of pixels by which to shift the polygon from the right.\n\n bottom : None or int, optional\n Amount of pixels by which to shift the polygon from the bottom.\n\n left : None or int, optional\n Amount of pixels by which to shift the polygon from the left.\n\n Returns\n -------\n imgaug.Polygon\n Shifted polygon.\n\n \"\"\"\n ls_shifted = self.to_line_string(closed=False).shift(\n top=top, right=right, bottom=bottom, left=left)\n return self.copy(exterior=ls_shifted.coords)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef draw_on_image(self,\n image,\n color=(0, 255, 0), color_face=None,\n color_lines=None, color_points=None,\n alpha=1.0, alpha_face=None,\n alpha_lines=None, alpha_points=None,\n size=1, size_lines=None, size_points=None,\n raise_if_out_of_image=False):\n \"\"\"\n Draw the polygon on an image.\n\n Parameters\n ----------\n image : (H,W,C) ndarray\n The image onto which to draw the polygon. Usually expected to be\n of dtype ``uint8``, though other dtypes are also handled.\n\n color : iterable of int, optional\n The color to use for the whole polygon.\n Must correspond to the channel layout of the image. Usually RGB.\n The values for `color_face`, `color_lines` and `color_points`\n will be derived from this color if they are set to ``None``.\n This argument has no effect if `color_face`, `color_lines`\n and `color_points` are all set anything other than ``None``.\n\n color_face : None or iterable of int, optional\n The color to use for the inner polygon area (excluding perimeter).\n Must correspond to the channel layout of the image. Usually RGB.\n If this is ``None``, it will be derived from ``color * 1.0``.\n\n color_lines : None or iterable of int, optional\n The color to use for the line (aka perimeter/border) of the polygon.\n Must correspond to the channel layout of the image. Usually RGB.\n If this is ``None``, it will be derived from ``color * 0.5``.\n\n color_points : None or iterable of int, optional\n The color to use for the corner points of the polygon.\n Must correspond to the channel layout of the image. Usually RGB.\n If this is ``None``, it will be derived from ``color * 0.5``.\n\n alpha : float, optional\n The opacity of the whole polygon, where ``1.0`` denotes a completely\n visible polygon and ``0.0`` an invisible one.\n The values for `alpha_face`, `alpha_lines` and `alpha_points`\n will be derived from this alpha value if they are set to ``None``.\n This argument has no effect if `alpha_face`, `alpha_lines`\n and `alpha_points` are all set anything other than ``None``.\n\n alpha_face : None or number, optional\n The opacity of the polygon's inner area (excluding the perimeter),\n where ``1.0`` denotes a completely visible inner area and ``0.0``\n an invisible one.\n If this is ``None``, it will be derived from ``alpha * 0.5``.\n\n alpha_lines : None or number, optional\n The opacity of the polygon's line (aka perimeter/border),\n where ``1.0`` denotes a completely visible line and ``0.0`` an\n invisible one.\n If this is ``None``, it will be derived from ``alpha * 1.0``.\n\n alpha_points : None or number, optional\n The opacity of the polygon's corner points, where ``1.0`` denotes\n completely visible corners and ``0.0`` invisible ones.\n If this is ``None``, it will be derived from ``alpha * 1.0``.\n\n size : int, optional\n Size of the polygon.\n The sizes of the line and points are derived from this value,\n unless they are set.\n\n size_lines : None or int, optional\n Thickness of the polygon's line (aka perimeter/border).\n If ``None``, this value is derived from `size`.\n\n size_points : int, optional\n Size of the points in pixels.\n If ``None``, this value is derived from ``3 * size``.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the polygon is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n result : (H,W,C) ndarray\n Image with polygon drawn on it. Result dtype is the same as the input dtype.\n\n \"\"\"\n assert color is not None\n assert alpha is not None\n assert size is not None\n\n color_face = color_face if color_face is not None else np.array(color)\n color_lines = color_lines if color_lines is not None else np.array(color) * 0.5\n color_points = color_points if color_points is not None else np.array(color) * 0.5\n\n alpha_face = alpha_face if alpha_face is not None else alpha * 0.5\n alpha_lines = alpha_lines if alpha_lines is not None else alpha\n alpha_points = alpha_points if alpha_points is not None else alpha\n\n size_lines = size_lines if size_lines is not None else size\n size_points = size_points if size_points is not None else size * 3\n\n if image.ndim == 2:\n assert ia.is_single_number(color_face), (\n \"Got a 2D image. Expected then 'color_face' to be a single \"\n \"number, but got %s.\" % (str(color_face),))\n color_face = [color_face]\n elif image.ndim == 3 and ia.is_single_number(color_face):\n color_face = [color_face] * image.shape[-1]\n\n if alpha_face < 0.01:\n alpha_face = 0\n elif alpha_face > 0.99:\n alpha_face = 1\n\n if raise_if_out_of_image and self.is_out_of_image(image):\n raise Exception(\"Cannot draw polygon %s on image with shape %s.\" % (\n str(self), image.shape\n ))\n\n # TODO np.clip to image plane if is_fully_within_image(), similar to how it is done for bounding boxes\n\n # TODO improve efficiency by only drawing in rectangle that covers poly instead of drawing in the whole image\n # TODO for a rectangular polygon, the face coordinates include the top/left boundary but not the right/bottom\n # boundary. This may be unintuitive when not drawing the boundary. Maybe somehow remove the boundary\n # coordinates from the face coordinates after generating both?\n input_dtype = image.dtype\n result = image.astype(np.float32)\n rr, cc = skimage.draw.polygon(self.yy_int, self.xx_int, shape=image.shape)\n if len(rr) > 0:\n if alpha_face == 1:\n result[rr, cc] = np.float32(color_face)\n elif alpha_face == 0:\n pass\n else:\n result[rr, cc] = (\n (1 - alpha_face) * result[rr, cc, :]\n + alpha_face * np.float32(color_face)\n )\n\n ls_open = self.to_line_string(closed=False)\n ls_closed = self.to_line_string(closed=True)\n result = ls_closed.draw_lines_on_image(\n result, color=color_lines, alpha=alpha_lines,\n size=size_lines, raise_if_out_of_image=raise_if_out_of_image)\n result = ls_open.draw_points_on_image(\n result, color=color_points, alpha=alpha_points,\n size=size_points, raise_if_out_of_image=raise_if_out_of_image)\n\n if input_dtype.type == np.uint8:\n result = np.clip(np.round(result), 0, 255).astype(input_dtype) # TODO make clipping more flexible\n else:\n result = result.astype(input_dtype)\n\n return result", "response": "Draw the polygon on an image."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nextract the image pixels within the polygon. This function will zero-pad the image if the polygon is partially/fully outside of the image. Parameters ---------- image : (H,W) ndarray or (H,W,C) ndarray The image from which to extract the pixels within the polygon. Returns ------- result : (H',W') ndarray or (H',W',C) ndarray Pixels within the polygon. Zero-padded if the polygon is partially/fully outside of the image.", "response": "def extract_from_image(self, image):\n \"\"\"\n Extract the image pixels within the polygon.\n\n This function will zero-pad the image if the polygon is partially/fully outside of\n the image.\n\n Parameters\n ----------\n image : (H,W) ndarray or (H,W,C) ndarray\n The image from which to extract the pixels within the polygon.\n\n Returns\n -------\n result : (H',W') ndarray or (H',W',C) ndarray\n Pixels within the polygon. Zero-padded if the polygon is partially/fully\n outside of the image.\n\n \"\"\"\n ia.do_assert(image.ndim in [2, 3])\n if len(self.exterior) <= 2:\n raise Exception(\"Polygon must be made up of at least 3 points to extract its area from an image.\")\n\n bb = self.to_bounding_box()\n bb_area = bb.extract_from_image(image)\n if self.is_out_of_image(image, fully=True, partly=False):\n return bb_area\n\n xx = self.xx_int\n yy = self.yy_int\n xx_mask = xx - np.min(xx)\n yy_mask = yy - np.min(yy)\n height_mask = np.max(yy_mask)\n width_mask = np.max(xx_mask)\n\n rr_face, cc_face = skimage.draw.polygon(yy_mask, xx_mask, shape=(height_mask, width_mask))\n\n mask = np.zeros((height_mask, width_mask), dtype=np.bool)\n mask[rr_face, cc_face] = True\n\n if image.ndim == 3:\n mask = np.tile(mask[:, :, np.newaxis], (1, 1, image.shape[2]))\n\n return bb_area * mask"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchanging the first point of the exterior to the given point based on its coordinates.", "response": "def change_first_point_by_coords(self, x, y, max_distance=1e-4,\n raise_if_too_far_away=True):\n \"\"\"\n Set the first point of the exterior to the given point based on its coordinates.\n\n If multiple points are found, the closest one will be picked.\n If no matching points are found, an exception is raised.\n\n Note: This method does *not* work in-place.\n\n Parameters\n ----------\n x : number\n X-coordinate of the point.\n\n y : number\n Y-coordinate of the point.\n\n max_distance : None or number, optional\n Maximum distance past which possible matches are ignored.\n If ``None`` the distance limit is deactivated.\n\n raise_if_too_far_away : bool, optional\n Whether to raise an exception if the closest found point is too\n far away (``True``) or simply return an unchanged copy if this\n object (``False``).\n\n Returns\n -------\n imgaug.Polygon\n Copy of this polygon with the new point order.\n\n \"\"\"\n if len(self.exterior) == 0:\n raise Exception(\"Cannot reorder polygon points, because it contains no points.\")\n\n closest_idx, closest_dist = self.find_closest_point_index(x=x, y=y, return_distance=True)\n if max_distance is not None and closest_dist > max_distance:\n if not raise_if_too_far_away:\n return self.deepcopy()\n\n closest_point = self.exterior[closest_idx, :]\n raise Exception(\n \"Closest found point (%.9f, %.9f) exceeds max_distance of %.9f exceeded\" % (\n closest_point[0], closest_point[1], closest_dist)\n )\n return self.change_first_point_by_index(closest_idx)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef change_first_point_by_index(self, point_idx):\n ia.do_assert(0 <= point_idx < len(self.exterior))\n if point_idx == 0:\n return self.deepcopy()\n exterior = np.concatenate(\n (self.exterior[point_idx:, :], self.exterior[:point_idx, :]),\n axis=0\n )\n return self.deepcopy(exterior=exterior)", "response": "Change the first point of the exterior to the given point based on its index."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_shapely_polygon(self):\n # load shapely lazily, which makes the dependency more optional\n import shapely.geometry\n\n return shapely.geometry.Polygon([(point[0], point[1]) for point in self.exterior])", "response": "Convert this polygon to a Shapely polygon."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts this polygon to a Shapely LineString object.", "response": "def to_shapely_line_string(self, closed=False, interpolate=0):\n \"\"\"\n Convert this polygon to a Shapely LineString object.\n\n Parameters\n ----------\n closed : bool, optional\n Whether to return the line string with the last point being identical to the first point.\n\n interpolate : int, optional\n Number of points to interpolate between any pair of two consecutive points. These points are added\n to the final line string.\n\n Returns\n -------\n shapely.geometry.LineString\n The Shapely LineString matching the polygon's exterior.\n\n \"\"\"\n return _convert_points_to_shapely_line_string(self.exterior, closed=closed, interpolate=interpolate)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert this polygon to a bounding box tightly containing the whole polygon.", "response": "def to_bounding_box(self):\n \"\"\"\n Convert this polygon to a bounding box tightly containing the whole polygon.\n\n Returns\n -------\n imgaug.BoundingBox\n Tight bounding box around the polygon.\n\n \"\"\"\n # TODO get rid of this deferred import\n from imgaug.augmentables.bbs import BoundingBox\n\n xx = self.xx\n yy = self.yy\n return BoundingBox(x1=min(xx), x2=max(xx), y1=min(yy), y2=max(yy), label=self.label)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_keypoints(self):\n # TODO get rid of this deferred import\n from imgaug.augmentables.kps import Keypoint\n\n return [Keypoint(x=point[0], y=point[1]) for point in self.exterior]", "response": "Convert this polygon s exterior to a list of imgaug. Keypoint instances."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_line_string(self, closed=True):\n from imgaug.augmentables.lines import LineString\n if not closed or len(self.exterior) <= 1:\n return LineString(self.exterior, label=self.label)\n return LineString(\n np.concatenate([self.exterior, self.exterior[0:1, :]], axis=0),\n label=self.label)", "response": "Converts this polygon s exterior to a line string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a new polygon from a shapely polygon.", "response": "def from_shapely(polygon_shapely, label=None):\n \"\"\"\n Create a polygon from a Shapely polygon.\n\n Note: This will remove any holes in the Shapely polygon.\n\n Parameters\n ----------\n polygon_shapely : shapely.geometry.Polygon\n The shapely polygon.\n\n label : None or str, optional\n The label of the new polygon.\n\n Returns\n -------\n imgaug.Polygon\n A polygon with the same exterior as the Shapely polygon.\n\n \"\"\"\n # load shapely lazily, which makes the dependency more optional\n import shapely.geometry\n\n ia.do_assert(isinstance(polygon_shapely, shapely.geometry.Polygon))\n # polygon_shapely.exterior can be None if the polygon was instantiated without points\n if polygon_shapely.exterior is None or len(polygon_shapely.exterior.coords) == 0:\n return Polygon([], label=label)\n exterior = np.float32([[x, y] for (x, y) in polygon_shapely.exterior.coords])\n return Polygon(exterior, label=label)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a string that represents the almost identical polygon.", "response": "def exterior_almost_equals(self, other, max_distance=1e-6, points_per_edge=8):\n \"\"\"\n Estimate if this and other polygon's exterior are almost identical.\n\n The two exteriors can have different numbers of points, but any point\n randomly sampled on the exterior of one polygon should be close to the\n closest point on the exterior of the other polygon.\n\n Note that this method works approximately. One can come up with\n polygons with fairly different shapes that will still be estimated as\n equal by this method. In practice however this should be unlikely to be\n the case. The probability for something like that goes down as the\n interpolation parameter is increased.\n\n Parameters\n ----------\n other : imgaug.Polygon or (N,2) ndarray or list of tuple\n The other polygon with which to compare the exterior.\n If this is an ndarray, it is assumed to represent an exterior.\n It must then have dtype ``float32`` and shape ``(N,2)`` with the\n second dimension denoting xy-coordinates.\n If this is a list of tuples, it is assumed to represent an exterior.\n Each tuple then must contain exactly two numbers, denoting\n xy-coordinates.\n\n max_distance : number, optional\n The maximum euclidean distance between a point on one polygon and\n the closest point on the other polygon. If the distance is exceeded\n for any such pair, the two exteriors are not viewed as equal. The\n points are other the points contained in the polygon's exterior\n ndarray or interpolated points between these.\n\n points_per_edge : int, optional\n How many points to interpolate on each edge.\n\n Returns\n -------\n bool\n Whether the two polygon's exteriors can be viewed as equal\n (approximate test).\n\n \"\"\"\n if isinstance(other, list):\n other = Polygon(np.float32(other))\n elif ia.is_np_array(other):\n other = Polygon(other)\n else:\n assert isinstance(other, Polygon)\n other = other\n\n return self.to_line_string(closed=True).coords_almost_equals(\n other.to_line_string(closed=True),\n max_distance=max_distance,\n points_per_edge=points_per_edge\n )"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef almost_equals(self, other, max_distance=1e-6, points_per_edge=8):\n if not isinstance(other, Polygon):\n return False\n if self.label is not None or other.label is not None:\n if self.label is None:\n return False\n if other.label is None:\n return False\n if self.label != other.label:\n return False\n return self.exterior_almost_equals(\n other, max_distance=max_distance, points_per_edge=points_per_edge)", "response": "Return True if this polygon s and other s geometry and labels are similar."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef copy(self, exterior=None, label=None):\n return self.deepcopy(exterior=exterior, label=label)", "response": "Create a shallow copy of the polygon object."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef deepcopy(self, exterior=None, label=None):\n return Polygon(\n exterior=np.copy(self.exterior) if exterior is None else exterior,\n label=self.label if label is None else label\n )", "response": "Create a deep copy of the polygon."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef on(self, image):\n shape = normalize_shape(image)\n if shape[0:2] == self.shape[0:2]:\n return self.deepcopy()\n polygons = [poly.project(self.shape, shape) for poly in self.polygons]\n # TODO use deepcopy() here\n return PolygonsOnImage(polygons, shape)", "response": "Project polygons from one image to a new one."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef draw_on_image(self,\n image,\n color=(0, 255, 0), color_face=None,\n color_lines=None, color_points=None,\n alpha=1.0, alpha_face=None,\n alpha_lines=None, alpha_points=None,\n size=1, size_lines=None, size_points=None,\n raise_if_out_of_image=False):\n \"\"\"\n Draw all polygons onto a given image.\n\n Parameters\n ----------\n image : (H,W,C) ndarray\n The image onto which to draw the bounding boxes.\n This image should usually have the same shape as set in\n ``PolygonsOnImage.shape``.\n\n color : iterable of int, optional\n The color to use for the whole polygons.\n Must correspond to the channel layout of the image. Usually RGB.\n The values for `color_face`, `color_lines` and `color_points`\n will be derived from this color if they are set to ``None``.\n This argument has no effect if `color_face`, `color_lines`\n and `color_points` are all set anything other than ``None``.\n\n color_face : None or iterable of int, optional\n The color to use for the inner polygon areas (excluding perimeters).\n Must correspond to the channel layout of the image. Usually RGB.\n If this is ``None``, it will be derived from ``color * 1.0``.\n\n color_lines : None or iterable of int, optional\n The color to use for the lines (aka perimeters/borders) of the\n polygons. Must correspond to the channel layout of the image.\n Usually RGB. If this is ``None``, it will be derived\n from ``color * 0.5``.\n\n color_points : None or iterable of int, optional\n The color to use for the corner points of the polygons.\n Must correspond to the channel layout of the image. Usually RGB.\n If this is ``None``, it will be derived from ``color * 0.5``.\n\n alpha : float, optional\n The opacity of the whole polygons, where ``1.0`` denotes\n completely visible polygons and ``0.0`` invisible ones.\n The values for `alpha_face`, `alpha_lines` and `alpha_points`\n will be derived from this alpha value if they are set to ``None``.\n This argument has no effect if `alpha_face`, `alpha_lines`\n and `alpha_points` are all set anything other than ``None``.\n\n alpha_face : None or number, optional\n The opacity of the polygon's inner areas (excluding the perimeters),\n where ``1.0`` denotes completely visible inner areas and ``0.0``\n invisible ones.\n If this is ``None``, it will be derived from ``alpha * 0.5``.\n\n alpha_lines : None or number, optional\n The opacity of the polygon's lines (aka perimeters/borders),\n where ``1.0`` denotes completely visible perimeters and ``0.0``\n invisible ones.\n If this is ``None``, it will be derived from ``alpha * 1.0``.\n\n alpha_points : None or number, optional\n The opacity of the polygon's corner points, where ``1.0`` denotes\n completely visible corners and ``0.0`` invisible ones.\n Currently this is an on/off choice, i.e. only ``0.0`` or ``1.0``\n are allowed.\n If this is ``None``, it will be derived from ``alpha * 1.0``.\n\n size : int, optional\n Size of the polygons.\n The sizes of the line and points are derived from this value,\n unless they are set.\n\n size_lines : None or int, optional\n Thickness of the polygon lines (aka perimeter/border).\n If ``None``, this value is derived from `size`.\n\n size_points : int, optional\n The size of all corner points. If set to ``C``, each corner point\n will be drawn as a square of size ``C x C``.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if any polygon is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n image : (H,W,C) ndarray\n Image with drawn polygons.\n\n \"\"\"\n for poly in self.polygons:\n image = poly.draw_on_image(\n image,\n color=color,\n color_face=color_face,\n color_lines=color_lines,\n color_points=color_points,\n alpha=alpha,\n alpha_face=alpha_face,\n alpha_lines=alpha_lines,\n alpha_points=alpha_points,\n size=size,\n size_lines=size_lines,\n size_points=size_points,\n raise_if_out_of_image=raise_if_out_of_image\n )\n return image", "response": "Draws all polygons onto a given image."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a reduced set of polygons that are fully or partially outside of the image.", "response": "def remove_out_of_image(self, fully=True, partly=False):\n \"\"\"\n Remove all polygons that are fully or partially outside of the image.\n\n Parameters\n ----------\n fully : bool, optional\n Whether to remove polygons that are fully outside of the image.\n\n partly : bool, optional\n Whether to remove polygons that are partially outside of the image.\n\n Returns\n -------\n imgaug.PolygonsOnImage\n Reduced set of polygons, with those that were fully/partially\n outside of the image removed.\n\n \"\"\"\n polys_clean = [\n poly for poly in self.polygons\n if not poly.is_out_of_image(self.shape, fully=fully, partly=partly)\n ]\n # TODO use deepcopy() here\n return PolygonsOnImage(polys_clean, shape=self.shape)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef clip_out_of_image(self):\n polys_cut = [\n poly.clip_out_of_image(self.shape)\n for poly\n in self.polygons\n if poly.is_partly_within_image(self.shape)\n ]\n polys_cut_flat = [poly for poly_lst in polys_cut for poly in poly_lst]\n # TODO use deepcopy() here\n return PolygonsOnImage(polys_cut_flat, shape=self.shape)", "response": "Clip off all parts from all polygons that are outside of the image."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef shift(self, top=None, right=None, bottom=None, left=None):\n polys_new = [\n poly.shift(top=top, right=right, bottom=bottom, left=left)\n for poly\n in self.polygons\n ]\n return PolygonsOnImage(polys_new, shape=self.shape)", "response": "Shift all polygons from one or more image sides i. e. move them on the x - axis."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a deep copy of the PolygonsOnImage object. Returns ------- imgaug. PolygonsOnImage Deep copy.", "response": "def deepcopy(self):\n \"\"\"\n Create a deep copy of the PolygonsOnImage object.\n\n Returns\n -------\n imgaug.PolygonsOnImage\n Deep copy.\n\n \"\"\"\n # Manual copy is far faster than deepcopy for PolygonsOnImage,\n # so use manual copy here too\n polys = [poly.deepcopy() for poly in self.polygons]\n return PolygonsOnImage(polys, tuple(self.shape))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef from_shapely(geometry, label=None):\n # load shapely lazily, which makes the dependency more optional\n import shapely.geometry\n\n if isinstance(geometry, shapely.geometry.MultiPolygon):\n return MultiPolygon([Polygon.from_shapely(poly, label=label) for poly in geometry.geoms])\n elif isinstance(geometry, shapely.geometry.Polygon):\n return MultiPolygon([Polygon.from_shapely(geometry, label=label)])\n elif isinstance(geometry, shapely.geometry.collection.GeometryCollection):\n ia.do_assert(all([isinstance(poly, shapely.geometry.Polygon) for poly in geometry.geoms]))\n return MultiPolygon([Polygon.from_shapely(poly, label=label) for poly in geometry.geoms])\n else:\n raise Exception(\"Unknown datatype '%s'. Expected shapely.geometry.Polygon or \"\n \"shapely.geometry.MultiPolygon or \"\n \"shapely.geometry.collections.GeometryCollection.\" % (type(geometry),))", "response": "Create a MultiPolygon from a Shapely MultiPolygon or Shapely GeometryCollection."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Pad(px=None, percent=None, pad_mode=\"constant\", pad_cval=0, keep_size=True, sample_independently=True,\n name=None, deterministic=False, random_state=None):\n \"\"\"\n Augmenter that pads images, i.e. adds columns/rows to them.\n\n dtype support::\n\n See ``imgaug.augmenters.size.CropAndPad``.\n\n Parameters\n ----------\n px : None or int or imgaug.parameters.StochasticParameter or tuple, optional\n The number of pixels to pad on each side of the image.\n Either this or the parameter `percent` may be set, not both at the same\n time.\n\n * If None, then pixel-based padding will not be used.\n * If int, then that exact number of pixels will always be padded.\n * If StochasticParameter, then that parameter will be used for each\n image. Four samples will be drawn per image (top, right, bottom,\n left).\n * If a tuple of two ints with values a and b, then each side will\n be padded by a random amount in the range ``a <= x <= b``.\n ``x`` is sampled per image side.\n * If a tuple of four entries, then the entries represent top, right,\n bottom, left. Each entry may be a single integer (always pad by\n exactly that value), a tuple of two ints ``a`` and ``b`` (pad by\n an amount ``a <= x <= b``), a list of ints (pad by a random value\n that is contained in the list) or a StochasticParameter (sample\n the amount to pad from that parameter).\n\n percent : None or int or float or imgaug.parameters.StochasticParameter \\\n or tuple, optional\n The number of pixels to pad on each side of the image given\n *in percent* of the image height/width.\n E.g. if this is set to 0.1, the augmenter will always add 10 percent\n of the image's height to the top, 10 percent of the width to the right,\n 10 percent of the height at the bottom and 10 percent of the width to\n the left. Either this or the parameter `px` may be set, not both at the\n same time.\n\n * If None, then percent-based padding will not be used.\n * If int, then expected to be 0 (no padding).\n * If float, then that percentage will always be padded.\n * If StochasticParameter, then that parameter will be used for each\n image. Four samples will be drawn per image (top, right, bottom,\n left).\n * If a tuple of two floats with values a and b, then each side will\n be padded by a random percentage in the range ``a <= x <= b``.\n ``x`` is sampled per image side.\n * If a tuple of four entries, then the entries represent top, right,\n bottom, left. Each entry may be a single float (always pad by\n exactly that percent value), a tuple of two floats ``a`` and ``b``\n (pad by a percentage ``a <= x <= b``), a list of floats (pad by a\n random value that is contained in the list) or a\n StochasticParameter (sample the percentage to pad from that\n parameter).\n\n pad_mode : imgaug.ALL or str or list of str or \\\n imgaug.parameters.StochasticParameter, optional\n Padding mode to use. The available modes match the numpy padding modes,\n i.e. ``constant``, ``edge``, ``linear_ramp``, ``maximum``, ``median``,\n ``minimum``, ``reflect``, ``symmetric``, ``wrap``. The modes\n ``constant`` and ``linear_ramp`` use extra values, which are provided\n by ``pad_cval`` when necessary. See :func:`imgaug.imgaug.pad` for\n more details.\n\n * If ``imgaug.ALL``, then a random mode from all available modes\n will be sampled per image.\n * If a string, it will be used as the pad mode for all images.\n * If a list of strings, a random one of these will be sampled per\n image and used as the mode.\n * If StochasticParameter, a random mode will be sampled from this\n parameter per image.\n\n pad_cval : number or tuple of number list of number or \\\n imgaug.parameters.StochasticParameter, optional\n The constant value to use if the pad mode is ``constant`` or the end\n value to use if the mode is ``linear_ramp``.\n See :func:`imgaug.imgaug.pad` for more details.\n\n * If number, then that value will be used.\n * If a tuple of two numbers and at least one of them is a float,\n then a random number will be sampled from the continuous range\n ``a <= x <= b`` and used as the value. If both numbers are\n integers, the range is discrete.\n * If a list of number, then a random value will be chosen from the\n elements of the list and used as the value.\n * If StochasticParameter, a random value will be sampled from that\n parameter per image.\n\n keep_size : bool, optional\n After padding, the result image will usually have a different\n height/width compared to the original input image. If this parameter is\n set to True, then the padded image will be resized to the input image's\n size, i.e. the augmenter's output shape is always identical to the\n input shape.\n\n sample_independently : bool, optional\n If False AND the values for `px`/`percent` result in exactly one\n probability distribution for the amount to pad, only one single value\n will be sampled from that probability distribution and used for all\n sides. I.e. the pad amount then is the same for all sides.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Examples\n --------\n >>> aug = iaa.Pad(px=(0, 10))\n\n pads each side by a random value from the range 0px to 10px (the value\n is sampled per side). The added rows/columns are filled with black pixels.\n\n >>> aug = iaa.Pad(px=(0, 10), sample_independently=False)\n\n samples one value v from the discrete range ``[0..10]`` and pads all sides\n by ``v`` pixels.\n\n >>> aug = iaa.Pad(px=(0, 10), keep_size=False)\n\n pads each side by a random value from the range 0px to 10px (the value\n is sampled per side). After padding, the images are NOT resized to their\n original size (i.e. the images may end up having different heights/widths).\n\n >>> aug = iaa.Pad(px=((0, 10), (0, 5), (0, 10), (0, 5)))\n\n pads the top and bottom by a random value from the range 0px to 10px\n and the left and right by a random value in the range 0px to 5px.\n\n >>> aug = iaa.Pad(percent=(0, 0.1))\n\n pads each side by a random value from the range 0 percent to\n 10 percent. (Percent with respect to the side's size, e.g. for the\n top side it uses the image's height.)\n\n >>> aug = iaa.Pad(percent=([0.05, 0.1], [0.05, 0.1], [0.05, 0.1], [0.05, 0.1]))\n\n pads each side by either 5 percent or 10 percent.\n\n >>> aug = iaa.Pad(px=(0, 10), pad_mode=\"edge\")\n\n pads each side by a random value from the range 0px to 10px (the values\n are sampled per side). The padding uses the ``edge`` mode from numpy's\n pad function.\n\n >>> aug = iaa.Pad(px=(0, 10), pad_mode=[\"constant\", \"edge\"])\n\n pads each side by a random value from the range 0px to 10px (the values\n are sampled per side). The padding uses randomly either the ``constant``\n or ``edge`` mode from numpy's pad function.\n\n >>> aug = iaa.Pad(px=(0, 10), pad_mode=ia.ALL, pad_cval=(0, 255))\n\n pads each side by a random value from the range 0px to 10px (the values\n are sampled per side). It uses a random mode for numpy's pad function.\n If the mode is ``constant`` or ``linear_ramp``, it samples a random value\n ``v`` from the range ``[0, 255]`` and uses that as the constant\n value (``mode=constant``) or end value (``mode=linear_ramp``).\n\n \"\"\"\n\n def recursive_validate(v):\n if v is None:\n return v\n elif ia.is_single_number(v):\n ia.do_assert(v >= 0)\n return v\n elif isinstance(v, iap.StochasticParameter):\n return v\n elif isinstance(v, tuple):\n return tuple([recursive_validate(v_) for v_ in v])\n elif isinstance(v, list):\n return [recursive_validate(v_) for v_ in v]\n else:\n raise Exception(\"Expected None or int or float or StochasticParameter or list or tuple, got %s.\" % (\n type(v),))\n\n px = recursive_validate(px)\n percent = recursive_validate(percent)\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n aug = CropAndPad(\n px=px, percent=percent,\n pad_mode=pad_mode, pad_cval=pad_cval,\n keep_size=keep_size, sample_independently=sample_independently,\n name=name, deterministic=deterministic, random_state=random_state\n )\n return aug", "response": "Augmenter that pads images with the specified number of pixels and percents."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef isect_segments__naive(segments):\n isect = []\n\n # order points left -> right\n if Real is float:\n segments = [\n (s[0], s[1]) if s[0][X] <= s[1][X] else\n (s[1], s[0])\n for s in segments]\n else:\n segments = [\n (\n (Real(s[0][0]), Real(s[0][1])),\n (Real(s[1][0]), Real(s[1][1])),\n ) if (s[0] <= s[1]) else\n (\n (Real(s[1][0]), Real(s[1][1])),\n (Real(s[0][0]), Real(s[0][1])),\n )\n for s in segments]\n\n n = len(segments)\n\n for i in range(n):\n a0, a1 = segments[i]\n for j in range(i + 1, n):\n b0, b1 = segments[j]\n if a0 not in (b0, b1) and a1 not in (b0, b1):\n ix = isect_seg_seg_v2_point(a0, a1, b0, b1)\n if ix is not None:\n # USE_IGNORE_SEGMENT_ENDINGS handled already\n isect.append(ix)\n\n return isect", "response": "A function that returns a list of the isects that are not innaive order."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef isect_polygon__naive(points):\n isect = []\n\n n = len(points)\n\n if Real is float:\n pass\n else:\n points = [(Real(p[0]), Real(p[1])) for p in points]\n\n\n for i in range(n):\n a0, a1 = points[i], points[(i + 1) % n]\n for j in range(i + 1, n):\n b0, b1 = points[j], points[(j + 1) % n]\n if a0 not in (b0, b1) and a1 not in (b0, b1):\n ix = isect_seg_seg_v2_point(a0, a1, b0, b1)\n if ix is not None:\n\n if USE_IGNORE_SEGMENT_ENDINGS:\n if ((len_squared_v2v2(ix, a0) < NUM_EPS_SQ or\n len_squared_v2v2(ix, a1) < NUM_EPS_SQ) and\n (len_squared_v2v2(ix, b0) < NUM_EPS_SQ or\n len_squared_v2v2(ix, b1) < NUM_EPS_SQ)):\n continue\n\n isect.append(ix)\n\n return isect", "response": "A naive version of isect_polygon."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a list of unordered intersection points.", "response": "def get_intersections(self):\n \"\"\"\n Return a list of unordered intersection points.\n \"\"\"\n if Real is float:\n return list(self.intersections.keys())\n else:\n return [(float(p[0]), float(p[1])) for p in self.intersections.keys()]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of unordered intersection ( point segment pairs where segments may contain 2 or more values.", "response": "def get_intersections_with_segments(self):\n \"\"\"\n Return a list of unordered intersection '(point, segment)' pairs,\n where segments may contain 2 or more values.\n \"\"\"\n if Real is float:\n return [\n (p, [event.segment for event in event_set])\n for p, event_set in self.intersections.items()\n ]\n else:\n return [\n (\n (float(p[0]), float(p[1])),\n [((float(event.segment[0][0]), float(event.segment[0][1])),\n (float(event.segment[1][0]), float(event.segment[1][1])))\n for event in event_set],\n )\n for p, event_set in self.intersections.items()\n ]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef poll(self):\n assert(len(self.events_scan) != 0)\n p, events_current = self.events_scan.pop_min()\n return p, events_current", "response": "Get and remove the first item from this queue."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nremoving all items from T.", "response": "def clear(self):\n \"\"\"T.clear() -> None. Remove all items from T.\"\"\"\n def _clear(node):\n if node is not None:\n _clear(node.left)\n _clear(node.right)\n node.free()\n _clear(self._root)\n self._count = 0\n self._root = None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef pop_item(self):\n if self.is_empty():\n raise KeyError(\"pop_item(): tree is empty\")\n node = self._root\n while True:\n if node.left is not None:\n node = node.left\n elif node.right is not None:\n node = node.right\n else:\n break\n key = node.key\n value = node.value\n self.remove(key)\n return key, value", "response": "T. pop_item - Remove and return some key - value pair as a\n 2 - tuple ; but raise KeyError if T is empty."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef min_item(self):\n if self.is_empty():\n raise ValueError(\"Tree is empty\")\n node = self._root\n while node.left is not None:\n node = node.left\n return node.key, node.value", "response": "Get item with min key of tree. Raises ValueError if tree is empty."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef max_item(self):\n if self.is_empty():\n raise ValueError(\"Tree is empty\")\n node = self._root\n while node.right is not None:\n node = node.right\n return node.key, node.value", "response": "Get item with max key of tree. Raises ValueError if tree is empty."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef succ_item(self, key, default=_sentinel):\n # removed graingets version, because it was little slower on CPython and much slower on pypy\n # this version runs about 4x faster with pypy than the Cython version\n # Note: Code sharing of succ_item() and ceiling_item() is possible, but has always a speed penalty.\n node = self._root\n succ_node = None\n while node is not None:\n cmp = self._cmp(self._cmp_data, key, node.key)\n if cmp == 0:\n break\n elif cmp < 0:\n if (succ_node is None) or self._cmp(self._cmp_data, node.key, succ_node.key) < 0:\n succ_node = node\n node = node.left\n else:\n node = node.right\n\n if node is None: # stay at dead end\n if default is _sentinel:\n raise KeyError(str(key))\n return default\n # found node of key\n if node.right is not None:\n # find smallest node of right subtree\n node = node.right\n while node.left is not None:\n node = node.left\n if succ_node is None:\n succ_node = node\n elif self._cmp(self._cmp_data, node.key, succ_node.key) < 0:\n succ_node = node\n elif succ_node is None: # given key is biggest in tree\n if default is _sentinel:\n raise KeyError(str(key))\n return default\n return succ_node.key, succ_node.value", "response": "Get successor ( k v ) pair of key. raises KeyError if key is max key."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget predecessor key value pair of key.", "response": "def prev_item(self, key, default=_sentinel):\n \"\"\"Get predecessor (k,v) pair of key, raises KeyError if key is min key\n or key does not exist. optimized for pypy.\n \"\"\"\n # removed graingets version, because it was little slower on CPython and much slower on pypy\n # this version runs about 4x faster with pypy than the Cython version\n # Note: Code sharing of prev_item() and floor_item() is possible, but has always a speed penalty.\n node = self._root\n prev_node = None\n\n while node is not None:\n cmp = self._cmp(self._cmp_data, key, node.key)\n if cmp == 0:\n break\n elif cmp < 0:\n node = node.left\n else:\n if (prev_node is None) or self._cmp(self._cmp_data, prev_node.key, node.key) < 0:\n prev_node = node\n node = node.right\n\n if node is None: # stay at dead end (None)\n if default is _sentinel:\n raise KeyError(str(key))\n return default\n # found node of key\n if node.left is not None:\n # find biggest node of left subtree\n node = node.left\n while node.right is not None:\n node = node.right\n if prev_node is None:\n prev_node = node\n elif self._cmp(self._cmp_data, prev_node.key, node.key) < 0:\n prev_node = node\n elif prev_node is None: # given key is smallest in tree\n if default is _sentinel:\n raise KeyError(str(key))\n return default\n return prev_node.key, prev_node.value"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting default value for key", "response": "def set_default(self, key, default=None):\n \"\"\"T.set_default(k[,d]) -> T.get(k,d), also set T[k]=d if k not in T\"\"\"\n try:\n return self.get_value(key)\n except KeyError:\n self.insert(key, default)\n return default"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the previous key.", "response": "def prev_key(self, key, default=_sentinel):\n \"\"\"Get predecessor to key, raises KeyError if key is min key\n or key does not exist.\n \"\"\"\n item = self.prev_item(key, default)\n return default if item is default else item[0]"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets successor to key raises KeyError if key is max key", "response": "def succ_key(self, key, default=_sentinel):\n \"\"\"Get successor to key, raises KeyError if key is max key\n or key does not exist.\n \"\"\"\n item = self.succ_item(key, default)\n return default if item is default else item[0]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\niterate over the items of the associated tree.", "response": "def iter_items(self, start_key=None, end_key=None, reverse=False):\n \"\"\"Iterates over the (key, value) items of the associated tree,\n in ascending order if reverse is True, iterate in descending order,\n reverse defaults to False\"\"\"\n # optimized iterator (reduced method calls) - faster on CPython but slower on pypy\n\n if self.is_empty():\n return []\n if reverse:\n return self._iter_items_backward(start_key, end_key)\n else:\n return self._iter_items_forward(start_key, end_key)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninserts new entry into tree.", "response": "def insert(self, key, value):\n \"\"\"T.insert(key, value) <==> T[key] = value, insert key, value into tree.\"\"\"\n if self._root is None: # Empty tree case\n self._root = self._new_node(key, value)\n self._root.red = False # make root black\n return\n\n head = Node() # False tree root\n grand_parent = None\n grand_grand_parent = head\n parent = None # parent\n direction = 0\n last = 0\n\n # Set up helpers\n grand_grand_parent.right = self._root\n node = grand_grand_parent.right\n # Search down the tree\n while True:\n if node is None: # Insert new node at the bottom\n node = self._new_node(key, value)\n parent[direction] = node\n elif RBTree.is_red(node.left) and RBTree.is_red(node.right): # Color flip\n node.red = True\n node.left.red = False\n node.right.red = False\n\n # Fix red violation\n if RBTree.is_red(node) and RBTree.is_red(parent):\n direction2 = 1 if grand_grand_parent.right is grand_parent else 0\n if node is parent[last]:\n grand_grand_parent[direction2] = RBTree.jsw_single(grand_parent, 1 - last)\n else:\n grand_grand_parent[direction2] = RBTree.jsw_double(grand_parent, 1 - last)\n\n # Stop if found\n if self._cmp(self._cmp_data, key, node.key) == 0:\n node.value = value # set new value for key\n break\n\n last = direction\n direction = 0 if (self._cmp(self._cmp_data, key, node.key) < 0) else 1\n # Update helpers\n if grand_parent is not None:\n grand_grand_parent = grand_parent\n grand_parent = parent\n parent = node\n node = node[direction]\n\n self._root = head.right # Update root\n self._root.red = False"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef remove(self, key):\n if self._root is None:\n raise KeyError(str(key))\n head = Node() # False tree root\n node = head\n node.right = self._root\n parent = None\n grand_parent = None\n found = None # Found item\n direction = 1\n\n # Search and push a red down\n while node[direction] is not None:\n last = direction\n\n # Update helpers\n grand_parent = parent\n parent = node\n node = node[direction]\n\n direction = 1 if (self._cmp(self._cmp_data, node.key, key) < 0) else 0\n\n # Save found node\n if self._cmp(self._cmp_data, key, node.key) == 0:\n found = node\n\n # Push the red node down\n if not RBTree.is_red(node) and not RBTree.is_red(node[direction]):\n if RBTree.is_red(node[1 - direction]):\n parent[last] = RBTree.jsw_single(node, direction)\n parent = parent[last]\n elif not RBTree.is_red(node[1 - direction]):\n sibling = parent[1 - last]\n if sibling is not None:\n if (not RBTree.is_red(sibling[1 - last])) and (not RBTree.is_red(sibling[last])):\n # Color flip\n parent.red = False\n sibling.red = True\n node.red = True\n else:\n direction2 = 1 if grand_parent.right is parent else 0\n if RBTree.is_red(sibling[last]):\n grand_parent[direction2] = RBTree.jsw_double(parent, last)\n elif RBTree.is_red(sibling[1-last]):\n grand_parent[direction2] = RBTree.jsw_single(parent, last)\n # Ensure correct coloring\n grand_parent[direction2].red = True\n node.red = True\n grand_parent[direction2].left.red = False\n grand_parent[direction2].right.red = False\n\n # Replace and remove if found\n if found is not None:\n found.key = node.key\n found.value = node.value\n parent[int(parent.right is node)] = node[int(node.left is None)]\n node.free()\n self._count -= 1\n\n # Update root and make it black\n self._root = head.right\n if self._root is not None:\n self._root.red = False\n if not found:\n raise KeyError(str(key))", "response": "Remove item from tree."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef noise2d(self, x, y):\n # Place input coordinates onto grid.\n stretch_offset = (x + y) * STRETCH_CONSTANT_2D\n xs = x + stretch_offset\n ys = y + stretch_offset\n\n # Floor to get grid coordinates of rhombus (stretched square) super-cell origin.\n xsb = floor(xs)\n ysb = floor(ys)\n\n # Skew out to get actual coordinates of rhombus origin. We'll need these later.\n squish_offset = (xsb + ysb) * SQUISH_CONSTANT_2D\n xb = xsb + squish_offset\n yb = ysb + squish_offset\n\n # Compute grid coordinates relative to rhombus origin.\n xins = xs - xsb\n yins = ys - ysb\n\n # Sum those together to get a value that determines which region we're in.\n in_sum = xins + yins\n\n # Positions relative to origin point.\n dx0 = x - xb\n dy0 = y - yb\n\n value = 0\n\n # Contribution (1,0)\n dx1 = dx0 - 1 - SQUISH_CONSTANT_2D\n dy1 = dy0 - 0 - SQUISH_CONSTANT_2D\n attn1 = 2 - dx1 * dx1 - dy1 * dy1\n extrapolate = self._extrapolate2d\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 1, ysb + 0, dx1, dy1)\n\n # Contribution (0,1)\n dx2 = dx0 - 0 - SQUISH_CONSTANT_2D\n dy2 = dy0 - 1 - SQUISH_CONSTANT_2D\n attn2 = 2 - dx2 * dx2 - dy2 * dy2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 0, ysb + 1, dx2, dy2)\n\n if in_sum <= 1: # We're inside the triangle (2-Simplex) at (0,0)\n zins = 1 - in_sum\n if zins > xins or zins > yins: # (0,0) is one of the closest two triangular vertices\n if xins > yins:\n xsv_ext = xsb + 1\n ysv_ext = ysb - 1\n dx_ext = dx0 - 1\n dy_ext = dy0 + 1\n else:\n xsv_ext = xsb - 1\n ysv_ext = ysb + 1\n dx_ext = dx0 + 1\n dy_ext = dy0 - 1\n else: # (1,0) and (0,1) are the closest two vertices.\n xsv_ext = xsb + 1\n ysv_ext = ysb + 1\n dx_ext = dx0 - 1 - 2 * SQUISH_CONSTANT_2D\n dy_ext = dy0 - 1 - 2 * SQUISH_CONSTANT_2D\n else: # We're inside the triangle (2-Simplex) at (1,1)\n zins = 2 - in_sum\n if zins < xins or zins < yins: # (0,0) is one of the closest two triangular vertices\n if xins > yins:\n xsv_ext = xsb + 2\n ysv_ext = ysb + 0\n dx_ext = dx0 - 2 - 2 * SQUISH_CONSTANT_2D\n dy_ext = dy0 + 0 - 2 * SQUISH_CONSTANT_2D\n else:\n xsv_ext = xsb + 0\n ysv_ext = ysb + 2\n dx_ext = dx0 + 0 - 2 * SQUISH_CONSTANT_2D\n dy_ext = dy0 - 2 - 2 * SQUISH_CONSTANT_2D\n else: # (1,0) and (0,1) are the closest two vertices.\n dx_ext = dx0\n dy_ext = dy0\n xsv_ext = xsb\n ysv_ext = ysb\n xsb += 1\n ysb += 1\n dx0 = dx0 - 1 - 2 * SQUISH_CONSTANT_2D\n dy0 = dy0 - 1 - 2 * SQUISH_CONSTANT_2D\n\n # Contribution (0,0) or (1,1)\n attn0 = 2 - dx0 * dx0 - dy0 * dy0\n if attn0 > 0:\n attn0 *= attn0\n value += attn0 * attn0 * extrapolate(xsb, ysb, dx0, dy0)\n\n # Extra Vertex\n attn_ext = 2 - dx_ext * dx_ext - dy_ext * dy_ext\n if attn_ext > 0:\n attn_ext *= attn_ext\n value += attn_ext * attn_ext * extrapolate(xsv_ext, ysv_ext, dx_ext, dy_ext)\n\n return value / NORM_CONSTANT_2D", "response": "Generate 2D OpenSimplex noise from X Y coordinates."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef noise3d(self, x, y, z):\n # Place input coordinates on simplectic honeycomb.\n stretch_offset = (x + y + z) * STRETCH_CONSTANT_3D\n xs = x + stretch_offset\n ys = y + stretch_offset\n zs = z + stretch_offset\n\n # Floor to get simplectic honeycomb coordinates of rhombohedron (stretched cube) super-cell origin.\n xsb = floor(xs)\n ysb = floor(ys)\n zsb = floor(zs)\n\n # Skew out to get actual coordinates of rhombohedron origin. We'll need these later.\n squish_offset = (xsb + ysb + zsb) * SQUISH_CONSTANT_3D\n xb = xsb + squish_offset\n yb = ysb + squish_offset\n zb = zsb + squish_offset\n\n # Compute simplectic honeycomb coordinates relative to rhombohedral origin.\n xins = xs - xsb\n yins = ys - ysb\n zins = zs - zsb\n\n # Sum those together to get a value that determines which region we're in.\n in_sum = xins + yins + zins\n\n # Positions relative to origin point.\n dx0 = x - xb\n dy0 = y - yb\n dz0 = z - zb\n\n value = 0\n extrapolate = self._extrapolate3d\n if in_sum <= 1: # We're inside the tetrahedron (3-Simplex) at (0,0,0)\n\n # Determine which two of (0,0,1), (0,1,0), (1,0,0) are closest.\n a_point = 0x01\n a_score = xins\n b_point = 0x02\n b_score = yins\n if a_score >= b_score and zins > b_score:\n b_score = zins\n b_point = 0x04\n elif a_score < b_score and zins > a_score:\n a_score = zins\n a_point = 0x04\n\n # Now we determine the two lattice points not part of the tetrahedron that may contribute.\n # This depends on the closest two tetrahedral vertices, including (0,0,0)\n wins = 1 - in_sum\n if wins > a_score or wins > b_score: # (0,0,0) is one of the closest two tetrahedral vertices.\n c = b_point if (b_score > a_score) else a_point # Our other closest vertex is the closest out of a and b.\n\n if (c & 0x01) == 0:\n xsv_ext0 = xsb - 1\n xsv_ext1 = xsb\n dx_ext0 = dx0 + 1\n dx_ext1 = dx0\n else:\n xsv_ext0 = xsv_ext1 = xsb + 1\n dx_ext0 = dx_ext1 = dx0 - 1\n\n if (c & 0x02) == 0:\n ysv_ext0 = ysv_ext1 = ysb\n dy_ext0 = dy_ext1 = dy0\n if (c & 0x01) == 0:\n ysv_ext1 -= 1\n dy_ext1 += 1\n else:\n ysv_ext0 -= 1\n dy_ext0 += 1\n else:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy_ext1 = dy0 - 1\n\n if (c & 0x04) == 0:\n zsv_ext0 = zsb\n zsv_ext1 = zsb - 1\n dz_ext0 = dz0\n dz_ext1 = dz0 + 1\n else:\n zsv_ext0 = zsv_ext1 = zsb + 1\n dz_ext0 = dz_ext1 = dz0 - 1\n else: # (0,0,0) is not one of the closest two tetrahedral vertices.\n c = (a_point | b_point) # Our two extra vertices are determined by the closest two.\n\n if (c & 0x01) == 0:\n xsv_ext0 = xsb\n xsv_ext1 = xsb - 1\n dx_ext0 = dx0 - 2 * SQUISH_CONSTANT_3D\n dx_ext1 = dx0 + 1 - SQUISH_CONSTANT_3D\n else:\n xsv_ext0 = xsv_ext1 = xsb + 1\n dx_ext0 = dx0 - 1 - 2 * SQUISH_CONSTANT_3D\n dx_ext1 = dx0 - 1 - SQUISH_CONSTANT_3D\n\n if (c & 0x02) == 0:\n ysv_ext0 = ysb\n ysv_ext1 = ysb - 1\n dy_ext0 = dy0 - 2 * SQUISH_CONSTANT_3D\n dy_ext1 = dy0 + 1 - SQUISH_CONSTANT_3D\n else:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy0 - 1 - 2 * SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 1 - SQUISH_CONSTANT_3D\n\n if (c & 0x04) == 0:\n zsv_ext0 = zsb\n zsv_ext1 = zsb - 1\n dz_ext0 = dz0 - 2 * SQUISH_CONSTANT_3D\n dz_ext1 = dz0 + 1 - SQUISH_CONSTANT_3D\n else:\n zsv_ext0 = zsv_ext1 = zsb + 1\n dz_ext0 = dz0 - 1 - 2 * SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 1 - SQUISH_CONSTANT_3D\n\n # Contribution (0,0,0)\n attn0 = 2 - dx0 * dx0 - dy0 * dy0 - dz0 * dz0\n if attn0 > 0:\n attn0 *= attn0\n value += attn0 * attn0 * extrapolate(xsb + 0, ysb + 0, zsb + 0, dx0, dy0, dz0)\n\n # Contribution (1,0,0)\n dx1 = dx0 - 1 - SQUISH_CONSTANT_3D\n dy1 = dy0 - 0 - SQUISH_CONSTANT_3D\n dz1 = dz0 - 0 - SQUISH_CONSTANT_3D\n attn1 = 2 - dx1 * dx1 - dy1 * dy1 - dz1 * dz1\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 1, ysb + 0, zsb + 0, dx1, dy1, dz1)\n\n # Contribution (0,1,0)\n dx2 = dx0 - 0 - SQUISH_CONSTANT_3D\n dy2 = dy0 - 1 - SQUISH_CONSTANT_3D\n dz2 = dz1\n attn2 = 2 - dx2 * dx2 - dy2 * dy2 - dz2 * dz2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 0, ysb + 1, zsb + 0, dx2, dy2, dz2)\n\n # Contribution (0,0,1)\n dx3 = dx2\n dy3 = dy1\n dz3 = dz0 - 1 - SQUISH_CONSTANT_3D\n attn3 = 2 - dx3 * dx3 - dy3 * dy3 - dz3 * dz3\n if attn3 > 0:\n attn3 *= attn3\n value += attn3 * attn3 * extrapolate(xsb + 0, ysb + 0, zsb + 1, dx3, dy3, dz3)\n elif in_sum >= 2: # We're inside the tetrahedron (3-Simplex) at (1,1,1)\n\n # Determine which two tetrahedral vertices are the closest, out of (1,1,0), (1,0,1), (0,1,1) but not (1,1,1).\n a_point = 0x06\n a_score = xins\n b_point = 0x05\n b_score = yins\n if a_score <= b_score and zins < b_score:\n b_score = zins\n b_point = 0x03\n elif a_score > b_score and zins < a_score:\n a_score = zins\n a_point = 0x03\n\n # Now we determine the two lattice points not part of the tetrahedron that may contribute.\n # This depends on the closest two tetrahedral vertices, including (1,1,1)\n wins = 3 - in_sum\n if wins < a_score or wins < b_score: # (1,1,1) is one of the closest two tetrahedral vertices.\n c = b_point if (b_score < a_score) else a_point # Our other closest vertex is the closest out of a and b.\n\n if (c & 0x01) != 0:\n xsv_ext0 = xsb + 2\n xsv_ext1 = xsb + 1\n dx_ext0 = dx0 - 2 - 3 * SQUISH_CONSTANT_3D\n dx_ext1 = dx0 - 1 - 3 * SQUISH_CONSTANT_3D\n else:\n xsv_ext0 = xsv_ext1 = xsb\n dx_ext0 = dx_ext1 = dx0 - 3 * SQUISH_CONSTANT_3D\n\n if (c & 0x02) != 0:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy_ext1 = dy0 - 1 - 3 * SQUISH_CONSTANT_3D\n if (c & 0x01) != 0:\n ysv_ext1 += 1\n dy_ext1 -= 1\n else:\n ysv_ext0 += 1\n dy_ext0 -= 1\n else:\n ysv_ext0 = ysv_ext1 = ysb\n dy_ext0 = dy_ext1 = dy0 - 3 * SQUISH_CONSTANT_3D\n\n if (c & 0x04) != 0:\n zsv_ext0 = zsb + 1\n zsv_ext1 = zsb + 2\n dz_ext0 = dz0 - 1 - 3 * SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 2 - 3 * SQUISH_CONSTANT_3D\n else:\n zsv_ext0 = zsv_ext1 = zsb\n dz_ext0 = dz_ext1 = dz0 - 3 * SQUISH_CONSTANT_3D\n else: # (1,1,1) is not one of the closest two tetrahedral vertices.\n c = (a_point & b_point) # Our two extra vertices are determined by the closest two.\n\n if (c & 0x01) != 0:\n xsv_ext0 = xsb + 1\n xsv_ext1 = xsb + 2\n dx_ext0 = dx0 - 1 - SQUISH_CONSTANT_3D\n dx_ext1 = dx0 - 2 - 2 * SQUISH_CONSTANT_3D\n else:\n xsv_ext0 = xsv_ext1 = xsb\n dx_ext0 = dx0 - SQUISH_CONSTANT_3D\n dx_ext1 = dx0 - 2 * SQUISH_CONSTANT_3D\n\n if (c & 0x02) != 0:\n ysv_ext0 = ysb + 1\n ysv_ext1 = ysb + 2\n dy_ext0 = dy0 - 1 - SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 2 - 2 * SQUISH_CONSTANT_3D\n else:\n ysv_ext0 = ysv_ext1 = ysb\n dy_ext0 = dy0 - SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 2 * SQUISH_CONSTANT_3D\n\n if (c & 0x04) != 0:\n zsv_ext0 = zsb + 1\n zsv_ext1 = zsb + 2\n dz_ext0 = dz0 - 1 - SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 2 - 2 * SQUISH_CONSTANT_3D\n else:\n zsv_ext0 = zsv_ext1 = zsb\n dz_ext0 = dz0 - SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 2 * SQUISH_CONSTANT_3D\n\n # Contribution (1,1,0)\n dx3 = dx0 - 1 - 2 * SQUISH_CONSTANT_3D\n dy3 = dy0 - 1 - 2 * SQUISH_CONSTANT_3D\n dz3 = dz0 - 0 - 2 * SQUISH_CONSTANT_3D\n attn3 = 2 - dx3 * dx3 - dy3 * dy3 - dz3 * dz3\n if attn3 > 0:\n attn3 *= attn3\n value += attn3 * attn3 * extrapolate(xsb + 1, ysb + 1, zsb + 0, dx3, dy3, dz3)\n\n # Contribution (1,0,1)\n dx2 = dx3\n dy2 = dy0 - 0 - 2 * SQUISH_CONSTANT_3D\n dz2 = dz0 - 1 - 2 * SQUISH_CONSTANT_3D\n attn2 = 2 - dx2 * dx2 - dy2 * dy2 - dz2 * dz2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 1, ysb + 0, zsb + 1, dx2, dy2, dz2)\n\n # Contribution (0,1,1)\n dx1 = dx0 - 0 - 2 * SQUISH_CONSTANT_3D\n dy1 = dy3\n dz1 = dz2\n attn1 = 2 - dx1 * dx1 - dy1 * dy1 - dz1 * dz1\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 0, ysb + 1, zsb + 1, dx1, dy1, dz1)\n\n # Contribution (1,1,1)\n dx0 = dx0 - 1 - 3 * SQUISH_CONSTANT_3D\n dy0 = dy0 - 1 - 3 * SQUISH_CONSTANT_3D\n dz0 = dz0 - 1 - 3 * SQUISH_CONSTANT_3D\n attn0 = 2 - dx0 * dx0 - dy0 * dy0 - dz0 * dz0\n if attn0 > 0:\n attn0 *= attn0\n value += attn0 * attn0 * extrapolate(xsb + 1, ysb + 1, zsb + 1, dx0, dy0, dz0)\n else: # We're inside the octahedron (Rectified 3-Simplex) in between.\n # Decide between point (0,0,1) and (1,1,0) as closest\n p1 = xins + yins\n if p1 > 1:\n a_score = p1 - 1\n a_point = 0x03\n a_is_further_side = True\n else:\n a_score = 1 - p1\n a_point = 0x04\n a_is_further_side = False\n\n # Decide between point (0,1,0) and (1,0,1) as closest\n p2 = xins + zins\n if p2 > 1:\n b_score = p2 - 1\n b_point = 0x05\n b_is_further_side = True\n else:\n b_score = 1 - p2\n b_point = 0x02\n b_is_further_side = False\n\n # The closest out of the two (1,0,0) and (0,1,1) will replace the furthest out of the two decided above, if closer.\n p3 = yins + zins\n if p3 > 1:\n score = p3 - 1\n if a_score <= b_score and a_score < score:\n a_point = 0x06\n a_is_further_side = True\n elif a_score > b_score and b_score < score:\n b_point = 0x06\n b_is_further_side = True\n else:\n score = 1 - p3\n if a_score <= b_score and a_score < score:\n a_point = 0x01\n a_is_further_side = False\n elif a_score > b_score and b_score < score:\n b_point = 0x01\n b_is_further_side = False\n\n # Where each of the two closest points are determines how the extra two vertices are calculated.\n if a_is_further_side == b_is_further_side:\n if a_is_further_side: # Both closest points on (1,1,1) side\n\n # One of the two extra points is (1,1,1)\n dx_ext0 = dx0 - 1 - 3 * SQUISH_CONSTANT_3D\n dy_ext0 = dy0 - 1 - 3 * SQUISH_CONSTANT_3D\n dz_ext0 = dz0 - 1 - 3 * SQUISH_CONSTANT_3D\n xsv_ext0 = xsb + 1\n ysv_ext0 = ysb + 1\n zsv_ext0 = zsb + 1\n\n # Other extra point is based on the shared axis.\n c = (a_point & b_point)\n if (c & 0x01) != 0:\n dx_ext1 = dx0 - 2 - 2 * SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 2 * SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 2 * SQUISH_CONSTANT_3D\n xsv_ext1 = xsb + 2\n ysv_ext1 = ysb\n zsv_ext1 = zsb\n elif (c & 0x02) != 0:\n dx_ext1 = dx0 - 2 * SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 2 - 2 * SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 2 * SQUISH_CONSTANT_3D\n xsv_ext1 = xsb\n ysv_ext1 = ysb + 2\n zsv_ext1 = zsb\n else:\n dx_ext1 = dx0 - 2 * SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 2 * SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 2 - 2 * SQUISH_CONSTANT_3D\n xsv_ext1 = xsb\n ysv_ext1 = ysb\n zsv_ext1 = zsb + 2\n else:# Both closest points on (0,0,0) side\n\n # One of the two extra points is (0,0,0)\n dx_ext0 = dx0\n dy_ext0 = dy0\n dz_ext0 = dz0\n xsv_ext0 = xsb\n ysv_ext0 = ysb\n zsv_ext0 = zsb\n\n # Other extra point is based on the omitted axis.\n c = (a_point | b_point)\n if (c & 0x01) == 0:\n dx_ext1 = dx0 + 1 - SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 1 - SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 1 - SQUISH_CONSTANT_3D\n xsv_ext1 = xsb - 1\n ysv_ext1 = ysb + 1\n zsv_ext1 = zsb + 1\n elif (c & 0x02) == 0:\n dx_ext1 = dx0 - 1 - SQUISH_CONSTANT_3D\n dy_ext1 = dy0 + 1 - SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 1 - SQUISH_CONSTANT_3D\n xsv_ext1 = xsb + 1\n ysv_ext1 = ysb - 1\n zsv_ext1 = zsb + 1\n else:\n dx_ext1 = dx0 - 1 - SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 1 - SQUISH_CONSTANT_3D\n dz_ext1 = dz0 + 1 - SQUISH_CONSTANT_3D\n xsv_ext1 = xsb + 1\n ysv_ext1 = ysb + 1\n zsv_ext1 = zsb - 1\n else: # One point on (0,0,0) side, one point on (1,1,1) side\n if a_is_further_side:\n c1 = a_point\n c2 = b_point\n else:\n c1 = b_point\n c2 = a_point\n\n # One contribution is a _permutation of (1,1,-1)\n if (c1 & 0x01) == 0:\n dx_ext0 = dx0 + 1 - SQUISH_CONSTANT_3D\n dy_ext0 = dy0 - 1 - SQUISH_CONSTANT_3D\n dz_ext0 = dz0 - 1 - SQUISH_CONSTANT_3D\n xsv_ext0 = xsb - 1\n ysv_ext0 = ysb + 1\n zsv_ext0 = zsb + 1\n elif (c1 & 0x02) == 0:\n dx_ext0 = dx0 - 1 - SQUISH_CONSTANT_3D\n dy_ext0 = dy0 + 1 - SQUISH_CONSTANT_3D\n dz_ext0 = dz0 - 1 - SQUISH_CONSTANT_3D\n xsv_ext0 = xsb + 1\n ysv_ext0 = ysb - 1\n zsv_ext0 = zsb + 1\n else:\n dx_ext0 = dx0 - 1 - SQUISH_CONSTANT_3D\n dy_ext0 = dy0 - 1 - SQUISH_CONSTANT_3D\n dz_ext0 = dz0 + 1 - SQUISH_CONSTANT_3D\n xsv_ext0 = xsb + 1\n ysv_ext0 = ysb + 1\n zsv_ext0 = zsb - 1\n\n # One contribution is a _permutation of (0,0,2)\n dx_ext1 = dx0 - 2 * SQUISH_CONSTANT_3D\n dy_ext1 = dy0 - 2 * SQUISH_CONSTANT_3D\n dz_ext1 = dz0 - 2 * SQUISH_CONSTANT_3D\n xsv_ext1 = xsb\n ysv_ext1 = ysb\n zsv_ext1 = zsb\n if (c2 & 0x01) != 0:\n dx_ext1 -= 2\n xsv_ext1 += 2\n elif (c2 & 0x02) != 0:\n dy_ext1 -= 2\n ysv_ext1 += 2\n else:\n dz_ext1 -= 2\n zsv_ext1 += 2\n\n # Contribution (1,0,0)\n dx1 = dx0 - 1 - SQUISH_CONSTANT_3D\n dy1 = dy0 - 0 - SQUISH_CONSTANT_3D\n dz1 = dz0 - 0 - SQUISH_CONSTANT_3D\n attn1 = 2 - dx1 * dx1 - dy1 * dy1 - dz1 * dz1\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 1, ysb + 0, zsb + 0, dx1, dy1, dz1)\n\n # Contribution (0,1,0)\n dx2 = dx0 - 0 - SQUISH_CONSTANT_3D\n dy2 = dy0 - 1 - SQUISH_CONSTANT_3D\n dz2 = dz1\n attn2 = 2 - dx2 * dx2 - dy2 * dy2 - dz2 * dz2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 0, ysb + 1, zsb + 0, dx2, dy2, dz2)\n\n # Contribution (0,0,1)\n dx3 = dx2\n dy3 = dy1\n dz3 = dz0 - 1 - SQUISH_CONSTANT_3D\n attn3 = 2 - dx3 * dx3 - dy3 * dy3 - dz3 * dz3\n if attn3 > 0:\n attn3 *= attn3\n value += attn3 * attn3 * extrapolate(xsb + 0, ysb + 0, zsb + 1, dx3, dy3, dz3)\n\n # Contribution (1,1,0)\n dx4 = dx0 - 1 - 2 * SQUISH_CONSTANT_3D\n dy4 = dy0 - 1 - 2 * SQUISH_CONSTANT_3D\n dz4 = dz0 - 0 - 2 * SQUISH_CONSTANT_3D\n attn4 = 2 - dx4 * dx4 - dy4 * dy4 - dz4 * dz4\n if attn4 > 0:\n attn4 *= attn4\n value += attn4 * attn4 * extrapolate(xsb + 1, ysb + 1, zsb + 0, dx4, dy4, dz4)\n\n # Contribution (1,0,1)\n dx5 = dx4\n dy5 = dy0 - 0 - 2 * SQUISH_CONSTANT_3D\n dz5 = dz0 - 1 - 2 * SQUISH_CONSTANT_3D\n attn5 = 2 - dx5 * dx5 - dy5 * dy5 - dz5 * dz5\n if attn5 > 0:\n attn5 *= attn5\n value += attn5 * attn5 * extrapolate(xsb + 1, ysb + 0, zsb + 1, dx5, dy5, dz5)\n\n # Contribution (0,1,1)\n dx6 = dx0 - 0 - 2 * SQUISH_CONSTANT_3D\n dy6 = dy4\n dz6 = dz5\n attn6 = 2 - dx6 * dx6 - dy6 * dy6 - dz6 * dz6\n if attn6 > 0:\n attn6 *= attn6\n value += attn6 * attn6 * extrapolate(xsb + 0, ysb + 1, zsb + 1, dx6, dy6, dz6)\n\n # First extra vertex\n attn_ext0 = 2 - dx_ext0 * dx_ext0 - dy_ext0 * dy_ext0 - dz_ext0 * dz_ext0\n if attn_ext0 > 0:\n attn_ext0 *= attn_ext0\n value += attn_ext0 * attn_ext0 * extrapolate(xsv_ext0, ysv_ext0, zsv_ext0, dx_ext0, dy_ext0, dz_ext0)\n\n # Second extra vertex\n attn_ext1 = 2 - dx_ext1 * dx_ext1 - dy_ext1 * dy_ext1 - dz_ext1 * dz_ext1\n if attn_ext1 > 0:\n attn_ext1 *= attn_ext1\n value += attn_ext1 * attn_ext1 * extrapolate(xsv_ext1, ysv_ext1, zsv_ext1, dx_ext1, dy_ext1, dz_ext1)\n\n return value / NORM_CONSTANT_3D", "response": "Generate 3D OpenSimplex noise from X Y Z coordinates."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef noise4d(self, x, y, z, w):\n # Place input coordinates on simplectic honeycomb.\n stretch_offset = (x + y + z + w) * STRETCH_CONSTANT_4D\n xs = x + stretch_offset\n ys = y + stretch_offset\n zs = z + stretch_offset\n ws = w + stretch_offset\n\n # Floor to get simplectic honeycomb coordinates of rhombo-hypercube super-cell origin.\n xsb = floor(xs)\n ysb = floor(ys)\n zsb = floor(zs)\n wsb = floor(ws)\n\n # Skew out to get actual coordinates of stretched rhombo-hypercube origin. We'll need these later.\n squish_offset = (xsb + ysb + zsb + wsb) * SQUISH_CONSTANT_4D\n xb = xsb + squish_offset\n yb = ysb + squish_offset\n zb = zsb + squish_offset\n wb = wsb + squish_offset\n\n # Compute simplectic honeycomb coordinates relative to rhombo-hypercube origin.\n xins = xs - xsb\n yins = ys - ysb\n zins = zs - zsb\n wins = ws - wsb\n\n # Sum those together to get a value that determines which region we're in.\n in_sum = xins + yins + zins + wins\n\n # Positions relative to origin po.\n dx0 = x - xb\n dy0 = y - yb\n dz0 = z - zb\n dw0 = w - wb\n\n value = 0\n extrapolate = self._extrapolate4d\n if in_sum <= 1: # We're inside the pentachoron (4-Simplex) at (0,0,0,0)\n\n # Determine which two of (0,0,0,1), (0,0,1,0), (0,1,0,0), (1,0,0,0) are closest.\n a_po = 0x01\n a_score = xins\n b_po = 0x02\n b_score = yins\n if a_score >= b_score and zins > b_score:\n b_score = zins\n b_po = 0x04\n elif a_score < b_score and zins > a_score:\n a_score = zins\n a_po = 0x04\n\n if a_score >= b_score and wins > b_score:\n b_score = wins\n b_po = 0x08\n elif a_score < b_score and wins > a_score:\n a_score = wins\n a_po = 0x08\n\n # Now we determine the three lattice pos not part of the pentachoron that may contribute.\n # This depends on the closest two pentachoron vertices, including (0,0,0,0)\n uins = 1 - in_sum\n if uins > a_score or uins > b_score: # (0,0,0,0) is one of the closest two pentachoron vertices.\n c = b_po if (b_score > a_score) else a_po # Our other closest vertex is the closest out of a and b.\n if (c & 0x01) == 0:\n xsv_ext0 = xsb - 1\n xsv_ext1 = xsv_ext2 = xsb\n dx_ext0 = dx0 + 1\n dx_ext1 = dx_ext2 = dx0\n else:\n xsv_ext0 = xsv_ext1 = xsv_ext2 = xsb + 1\n dx_ext0 = dx_ext1 = dx_ext2 = dx0 - 1\n\n if (c & 0x02) == 0:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb\n dy_ext0 = dy_ext1 = dy_ext2 = dy0\n if (c & 0x01) == 0x01:\n ysv_ext0 -= 1\n dy_ext0 += 1\n else:\n ysv_ext1 -= 1\n dy_ext1 += 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb + 1\n dy_ext0 = dy_ext1 = dy_ext2 = dy0 - 1\n\n if (c & 0x04) == 0:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb\n dz_ext0 = dz_ext1 = dz_ext2 = dz0\n if (c & 0x03) != 0:\n if (c & 0x03) == 0x03:\n zsv_ext0 -= 1\n dz_ext0 += 1\n else:\n zsv_ext1 -= 1\n dz_ext1 += 1\n\n else:\n zsv_ext2 -= 1\n dz_ext2 += 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb + 1\n dz_ext0 = dz_ext1 = dz_ext2 = dz0 - 1\n\n\n if (c & 0x08) == 0:\n wsv_ext0 = wsv_ext1 = wsb\n wsv_ext2 = wsb - 1\n dw_ext0 = dw_ext1 = dw0\n dw_ext2 = dw0 + 1\n else:\n wsv_ext0 = wsv_ext1 = wsv_ext2 = wsb + 1\n dw_ext0 = dw_ext1 = dw_ext2 = dw0 - 1\n\n else: # (0,0,0,0) is not one of the closest two pentachoron vertices.\n c = (a_po | b_po) # Our three extra vertices are determined by the closest two.\n\n if (c & 0x01) == 0:\n xsv_ext0 = xsv_ext2 = xsb\n xsv_ext1 = xsb - 1\n dx_ext0 = dx0 - 2 * SQUISH_CONSTANT_4D\n dx_ext1 = dx0 + 1 - SQUISH_CONSTANT_4D\n dx_ext2 = dx0 - SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsv_ext2 = xsb + 1\n dx_ext0 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dx_ext1 = dx_ext2 = dx0 - 1 - SQUISH_CONSTANT_4D\n\n if (c & 0x02) == 0:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb\n dy_ext0 = dy0 - 2 * SQUISH_CONSTANT_4D\n dy_ext1 = dy_ext2 = dy0 - SQUISH_CONSTANT_4D\n if (c & 0x01) == 0x01:\n ysv_ext1 -= 1\n dy_ext1 += 1\n else:\n ysv_ext2 -= 1\n dy_ext2 += 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb + 1\n dy_ext0 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy_ext1 = dy_ext2 = dy0 - 1 - SQUISH_CONSTANT_4D\n\n if (c & 0x04) == 0:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb\n dz_ext0 = dz0 - 2 * SQUISH_CONSTANT_4D\n dz_ext1 = dz_ext2 = dz0 - SQUISH_CONSTANT_4D\n if (c & 0x03) == 0x03:\n zsv_ext1 -= 1\n dz_ext1 += 1\n else:\n zsv_ext2 -= 1\n dz_ext2 += 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb + 1\n dz_ext0 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz_ext1 = dz_ext2 = dz0 - 1 - SQUISH_CONSTANT_4D\n\n\n if (c & 0x08) == 0:\n wsv_ext0 = wsv_ext1 = wsb\n wsv_ext2 = wsb - 1\n dw_ext0 = dw0 - 2 * SQUISH_CONSTANT_4D\n dw_ext1 = dw0 - SQUISH_CONSTANT_4D\n dw_ext2 = dw0 + 1 - SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsv_ext2 = wsb + 1\n dw_ext0 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw_ext1 = dw_ext2 = dw0 - 1 - SQUISH_CONSTANT_4D\n\n # Contribution (0,0,0,0)\n attn0 = 2 - dx0 * dx0 - dy0 * dy0 - dz0 * dz0 - dw0 * dw0\n if attn0 > 0:\n attn0 *= attn0\n value += attn0 * attn0 * extrapolate(xsb + 0, ysb + 0, zsb + 0, wsb + 0, dx0, dy0, dz0, dw0)\n\n # Contribution (1,0,0,0)\n dx1 = dx0 - 1 - SQUISH_CONSTANT_4D\n dy1 = dy0 - 0 - SQUISH_CONSTANT_4D\n dz1 = dz0 - 0 - SQUISH_CONSTANT_4D\n dw1 = dw0 - 0 - SQUISH_CONSTANT_4D\n attn1 = 2 - dx1 * dx1 - dy1 * dy1 - dz1 * dz1 - dw1 * dw1\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 1, ysb + 0, zsb + 0, wsb + 0, dx1, dy1, dz1, dw1)\n\n # Contribution (0,1,0,0)\n dx2 = dx0 - 0 - SQUISH_CONSTANT_4D\n dy2 = dy0 - 1 - SQUISH_CONSTANT_4D\n dz2 = dz1\n dw2 = dw1\n attn2 = 2 - dx2 * dx2 - dy2 * dy2 - dz2 * dz2 - dw2 * dw2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 0, ysb + 1, zsb + 0, wsb + 0, dx2, dy2, dz2, dw2)\n\n # Contribution (0,0,1,0)\n dx3 = dx2\n dy3 = dy1\n dz3 = dz0 - 1 - SQUISH_CONSTANT_4D\n dw3 = dw1\n attn3 = 2 - dx3 * dx3 - dy3 * dy3 - dz3 * dz3 - dw3 * dw3\n if attn3 > 0:\n attn3 *= attn3\n value += attn3 * attn3 * extrapolate(xsb + 0, ysb + 0, zsb + 1, wsb + 0, dx3, dy3, dz3, dw3)\n\n # Contribution (0,0,0,1)\n dx4 = dx2\n dy4 = dy1\n dz4 = dz1\n dw4 = dw0 - 1 - SQUISH_CONSTANT_4D\n attn4 = 2 - dx4 * dx4 - dy4 * dy4 - dz4 * dz4 - dw4 * dw4\n if attn4 > 0:\n attn4 *= attn4\n value += attn4 * attn4 * extrapolate(xsb + 0, ysb + 0, zsb + 0, wsb + 1, dx4, dy4, dz4, dw4)\n\n elif in_sum >= 3: # We're inside the pentachoron (4-Simplex) at (1,1,1,1)\n # Determine which two of (1,1,1,0), (1,1,0,1), (1,0,1,1), (0,1,1,1) are closest.\n a_po = 0x0E\n a_score = xins\n b_po = 0x0D\n b_score = yins\n if a_score <= b_score and zins < b_score:\n b_score = zins\n b_po = 0x0B\n elif a_score > b_score and zins < a_score:\n a_score = zins\n a_po = 0x0B\n\n if a_score <= b_score and wins < b_score:\n b_score = wins\n b_po = 0x07\n elif a_score > b_score and wins < a_score:\n a_score = wins\n a_po = 0x07\n\n # Now we determine the three lattice pos not part of the pentachoron that may contribute.\n # This depends on the closest two pentachoron vertices, including (0,0,0,0)\n uins = 4 - in_sum\n if uins < a_score or uins < b_score: # (1,1,1,1) is one of the closest two pentachoron vertices.\n c = b_po if (b_score < a_score) else a_po # Our other closest vertex is the closest out of a and b.\n\n if (c & 0x01) != 0:\n xsv_ext0 = xsb + 2\n xsv_ext1 = xsv_ext2 = xsb + 1\n dx_ext0 = dx0 - 2 - 4 * SQUISH_CONSTANT_4D\n dx_ext1 = dx_ext2 = dx0 - 1 - 4 * SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsv_ext2 = xsb\n dx_ext0 = dx_ext1 = dx_ext2 = dx0 - 4 * SQUISH_CONSTANT_4D\n\n if (c & 0x02) != 0:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb + 1\n dy_ext0 = dy_ext1 = dy_ext2 = dy0 - 1 - 4 * SQUISH_CONSTANT_4D\n if (c & 0x01) != 0:\n ysv_ext1 += 1\n dy_ext1 -= 1\n else:\n ysv_ext0 += 1\n dy_ext0 -= 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb\n dy_ext0 = dy_ext1 = dy_ext2 = dy0 - 4 * SQUISH_CONSTANT_4D\n\n if (c & 0x04) != 0:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb + 1\n dz_ext0 = dz_ext1 = dz_ext2 = dz0 - 1 - 4 * SQUISH_CONSTANT_4D\n if (c & 0x03) != 0x03:\n if (c & 0x03) == 0:\n zsv_ext0 += 1\n dz_ext0 -= 1\n else:\n zsv_ext1 += 1\n dz_ext1 -= 1\n\n else:\n zsv_ext2 += 1\n dz_ext2 -= 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb\n dz_ext0 = dz_ext1 = dz_ext2 = dz0 - 4 * SQUISH_CONSTANT_4D\n\n if (c & 0x08) != 0:\n wsv_ext0 = wsv_ext1 = wsb + 1\n wsv_ext2 = wsb + 2\n dw_ext0 = dw_ext1 = dw0 - 1 - 4 * SQUISH_CONSTANT_4D\n dw_ext2 = dw0 - 2 - 4 * SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsv_ext2 = wsb\n dw_ext0 = dw_ext1 = dw_ext2 = dw0 - 4 * SQUISH_CONSTANT_4D\n\n else: # (1,1,1,1) is not one of the closest two pentachoron vertices.\n c = (a_po & b_po) # Our three extra vertices are determined by the closest two.\n\n if (c & 0x01) != 0:\n xsv_ext0 = xsv_ext2 = xsb + 1\n xsv_ext1 = xsb + 2\n dx_ext0 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dx_ext1 = dx0 - 2 - 3 * SQUISH_CONSTANT_4D\n dx_ext2 = dx0 - 1 - 3 * SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsv_ext2 = xsb\n dx_ext0 = dx0 - 2 * SQUISH_CONSTANT_4D\n dx_ext1 = dx_ext2 = dx0 - 3 * SQUISH_CONSTANT_4D\n\n if (c & 0x02) != 0:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb + 1\n dy_ext0 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy_ext1 = dy_ext2 = dy0 - 1 - 3 * SQUISH_CONSTANT_4D\n if (c & 0x01) != 0:\n ysv_ext2 += 1\n dy_ext2 -= 1\n else:\n ysv_ext1 += 1\n dy_ext1 -= 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysv_ext2 = ysb\n dy_ext0 = dy0 - 2 * SQUISH_CONSTANT_4D\n dy_ext1 = dy_ext2 = dy0 - 3 * SQUISH_CONSTANT_4D\n\n if (c & 0x04) != 0:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb + 1\n dz_ext0 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz_ext1 = dz_ext2 = dz0 - 1 - 3 * SQUISH_CONSTANT_4D\n if (c & 0x03) != 0:\n zsv_ext2 += 1\n dz_ext2 -= 1\n else:\n zsv_ext1 += 1\n dz_ext1 -= 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsv_ext2 = zsb\n dz_ext0 = dz0 - 2 * SQUISH_CONSTANT_4D\n dz_ext1 = dz_ext2 = dz0 - 3 * SQUISH_CONSTANT_4D\n\n if (c & 0x08) != 0:\n wsv_ext0 = wsv_ext1 = wsb + 1\n wsv_ext2 = wsb + 2\n dw_ext0 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw_ext1 = dw0 - 1 - 3 * SQUISH_CONSTANT_4D\n dw_ext2 = dw0 - 2 - 3 * SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsv_ext2 = wsb\n dw_ext0 = dw0 - 2 * SQUISH_CONSTANT_4D\n dw_ext1 = dw_ext2 = dw0 - 3 * SQUISH_CONSTANT_4D\n\n # Contribution (1,1,1,0)\n dx4 = dx0 - 1 - 3 * SQUISH_CONSTANT_4D\n dy4 = dy0 - 1 - 3 * SQUISH_CONSTANT_4D\n dz4 = dz0 - 1 - 3 * SQUISH_CONSTANT_4D\n dw4 = dw0 - 3 * SQUISH_CONSTANT_4D\n attn4 = 2 - dx4 * dx4 - dy4 * dy4 - dz4 * dz4 - dw4 * dw4\n if attn4 > 0:\n attn4 *= attn4\n value += attn4 * attn4 * extrapolate(xsb + 1, ysb + 1, zsb + 1, wsb + 0, dx4, dy4, dz4, dw4)\n\n # Contribution (1,1,0,1)\n dx3 = dx4\n dy3 = dy4\n dz3 = dz0 - 3 * SQUISH_CONSTANT_4D\n dw3 = dw0 - 1 - 3 * SQUISH_CONSTANT_4D\n attn3 = 2 - dx3 * dx3 - dy3 * dy3 - dz3 * dz3 - dw3 * dw3\n if attn3 > 0:\n attn3 *= attn3\n value += attn3 * attn3 * extrapolate(xsb + 1, ysb + 1, zsb + 0, wsb + 1, dx3, dy3, dz3, dw3)\n\n # Contribution (1,0,1,1)\n dx2 = dx4\n dy2 = dy0 - 3 * SQUISH_CONSTANT_4D\n dz2 = dz4\n dw2 = dw3\n attn2 = 2 - dx2 * dx2 - dy2 * dy2 - dz2 * dz2 - dw2 * dw2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 1, ysb + 0, zsb + 1, wsb + 1, dx2, dy2, dz2, dw2)\n\n # Contribution (0,1,1,1)\n dx1 = dx0 - 3 * SQUISH_CONSTANT_4D\n dz1 = dz4\n dy1 = dy4\n dw1 = dw3\n attn1 = 2 - dx1 * dx1 - dy1 * dy1 - dz1 * dz1 - dw1 * dw1\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 0, ysb + 1, zsb + 1, wsb + 1, dx1, dy1, dz1, dw1)\n\n # Contribution (1,1,1,1)\n dx0 = dx0 - 1 - 4 * SQUISH_CONSTANT_4D\n dy0 = dy0 - 1 - 4 * SQUISH_CONSTANT_4D\n dz0 = dz0 - 1 - 4 * SQUISH_CONSTANT_4D\n dw0 = dw0 - 1 - 4 * SQUISH_CONSTANT_4D\n attn0 = 2 - dx0 * dx0 - dy0 * dy0 - dz0 * dz0 - dw0 * dw0\n if attn0 > 0:\n attn0 *= attn0\n value += attn0 * attn0 * extrapolate(xsb + 1, ysb + 1, zsb + 1, wsb + 1, dx0, dy0, dz0, dw0)\n\n elif in_sum <= 2: # We're inside the first dispentachoron (Rectified 4-Simplex)\n a_is_bigger_side = True\n b_is_bigger_side = True\n\n # Decide between (1,1,0,0) and (0,0,1,1)\n if xins + yins > zins + wins:\n a_score = xins + yins\n a_po = 0x03\n else:\n a_score = zins + wins\n a_po = 0x0C\n\n # Decide between (1,0,1,0) and (0,1,0,1)\n if xins + zins > yins + wins:\n b_score = xins + zins\n b_po = 0x05\n else:\n b_score = yins + wins\n b_po = 0x0A\n\n # Closer between (1,0,0,1) and (0,1,1,0) will replace the further of a and b, if closer.\n if xins + wins > yins + zins:\n score = xins + wins\n if a_score >= b_score and score > b_score:\n b_score = score\n b_po = 0x09\n elif a_score < b_score and score > a_score:\n a_score = score\n a_po = 0x09\n\n else:\n score = yins + zins\n if a_score >= b_score and score > b_score:\n b_score = score\n b_po = 0x06\n elif a_score < b_score and score > a_score:\n a_score = score\n a_po = 0x06\n\n # Decide if (1,0,0,0) is closer.\n p1 = 2 - in_sum + xins\n if a_score >= b_score and p1 > b_score:\n b_score = p1\n b_po = 0x01\n b_is_bigger_side = False\n elif a_score < b_score and p1 > a_score:\n a_score = p1\n a_po = 0x01\n a_is_bigger_side = False\n\n # Decide if (0,1,0,0) is closer.\n p2 = 2 - in_sum + yins\n if a_score >= b_score and p2 > b_score:\n b_score = p2\n b_po = 0x02\n b_is_bigger_side = False\n elif a_score < b_score and p2 > a_score:\n a_score = p2\n a_po = 0x02\n a_is_bigger_side = False\n\n # Decide if (0,0,1,0) is closer.\n p3 = 2 - in_sum + zins\n if a_score >= b_score and p3 > b_score:\n b_score = p3\n b_po = 0x04\n b_is_bigger_side = False\n elif a_score < b_score and p3 > a_score:\n a_score = p3\n a_po = 0x04\n a_is_bigger_side = False\n\n # Decide if (0,0,0,1) is closer.\n p4 = 2 - in_sum + wins\n if a_score >= b_score and p4 > b_score:\n b_po = 0x08\n b_is_bigger_side = False\n elif a_score < b_score and p4 > a_score:\n a_po = 0x08\n a_is_bigger_side = False\n\n # Where each of the two closest pos are determines how the extra three vertices are calculated.\n if a_is_bigger_side == b_is_bigger_side:\n if a_is_bigger_side: # Both closest pos on the bigger side\n c1 = (a_po | b_po)\n c2 = (a_po & b_po)\n if (c1 & 0x01) == 0:\n xsv_ext0 = xsb\n xsv_ext1 = xsb - 1\n dx_ext0 = dx0 - 3 * SQUISH_CONSTANT_4D\n dx_ext1 = dx0 + 1 - 2 * SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsb + 1\n dx_ext0 = dx0 - 1 - 3 * SQUISH_CONSTANT_4D\n dx_ext1 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n\n if (c1 & 0x02) == 0:\n ysv_ext0 = ysb\n ysv_ext1 = ysb - 1\n dy_ext0 = dy0 - 3 * SQUISH_CONSTANT_4D\n dy_ext1 = dy0 + 1 - 2 * SQUISH_CONSTANT_4D\n else:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy0 - 1 - 3 * SQUISH_CONSTANT_4D\n dy_ext1 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n\n if (c1 & 0x04) == 0:\n zsv_ext0 = zsb\n zsv_ext1 = zsb - 1\n dz_ext0 = dz0 - 3 * SQUISH_CONSTANT_4D\n dz_ext1 = dz0 + 1 - 2 * SQUISH_CONSTANT_4D\n else:\n zsv_ext0 = zsv_ext1 = zsb + 1\n dz_ext0 = dz0 - 1 - 3 * SQUISH_CONSTANT_4D\n dz_ext1 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n\n if (c1 & 0x08) == 0:\n wsv_ext0 = wsb\n wsv_ext1 = wsb - 1\n dw_ext0 = dw0 - 3 * SQUISH_CONSTANT_4D\n dw_ext1 = dw0 + 1 - 2 * SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsb + 1\n dw_ext0 = dw0 - 1 - 3 * SQUISH_CONSTANT_4D\n dw_ext1 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n\n # One combination is a _permutation of (0,0,0,2) based on c2\n xsv_ext2 = xsb\n ysv_ext2 = ysb\n zsv_ext2 = zsb\n wsv_ext2 = wsb\n dx_ext2 = dx0 - 2 * SQUISH_CONSTANT_4D\n dy_ext2 = dy0 - 2 * SQUISH_CONSTANT_4D\n dz_ext2 = dz0 - 2 * SQUISH_CONSTANT_4D\n dw_ext2 = dw0 - 2 * SQUISH_CONSTANT_4D\n if (c2 & 0x01) != 0:\n xsv_ext2 += 2\n dx_ext2 -= 2\n elif (c2 & 0x02) != 0:\n ysv_ext2 += 2\n dy_ext2 -= 2\n elif (c2 & 0x04) != 0:\n zsv_ext2 += 2\n dz_ext2 -= 2\n else:\n wsv_ext2 += 2\n dw_ext2 -= 2\n\n else: # Both closest pos on the smaller side\n # One of the two extra pos is (0,0,0,0)\n xsv_ext2 = xsb\n ysv_ext2 = ysb\n zsv_ext2 = zsb\n wsv_ext2 = wsb\n dx_ext2 = dx0\n dy_ext2 = dy0\n dz_ext2 = dz0\n dw_ext2 = dw0\n\n # Other two pos are based on the omitted axes.\n c = (a_po | b_po)\n\n if (c & 0x01) == 0:\n xsv_ext0 = xsb - 1\n xsv_ext1 = xsb\n dx_ext0 = dx0 + 1 - SQUISH_CONSTANT_4D\n dx_ext1 = dx0 - SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsb + 1\n dx_ext0 = dx_ext1 = dx0 - 1 - SQUISH_CONSTANT_4D\n\n if (c & 0x02) == 0:\n ysv_ext0 = ysv_ext1 = ysb\n dy_ext0 = dy_ext1 = dy0 - SQUISH_CONSTANT_4D\n if (c & 0x01) == 0x01:\n ysv_ext0 -= 1\n dy_ext0 += 1\n else:\n ysv_ext1 -= 1\n dy_ext1 += 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy_ext1 = dy0 - 1 - SQUISH_CONSTANT_4D\n\n if (c & 0x04) == 0:\n zsv_ext0 = zsv_ext1 = zsb\n dz_ext0 = dz_ext1 = dz0 - SQUISH_CONSTANT_4D\n if (c & 0x03) == 0x03:\n zsv_ext0 -= 1\n dz_ext0 += 1\n else:\n zsv_ext1 -= 1\n dz_ext1 += 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsb + 1\n dz_ext0 = dz_ext1 = dz0 - 1 - SQUISH_CONSTANT_4D\n\n\n if (c & 0x08) == 0:\n wsv_ext0 = wsb\n wsv_ext1 = wsb - 1\n dw_ext0 = dw0 - SQUISH_CONSTANT_4D\n dw_ext1 = dw0 + 1 - SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsb + 1\n dw_ext0 = dw_ext1 = dw0 - 1 - SQUISH_CONSTANT_4D\n\n else: # One po on each \"side\"\n if a_is_bigger_side:\n c1 = a_po\n c2 = b_po\n else:\n c1 = b_po\n c2 = a_po\n\n # Two contributions are the bigger-sided po with each 0 replaced with -1.\n if (c1 & 0x01) == 0:\n xsv_ext0 = xsb - 1\n xsv_ext1 = xsb\n dx_ext0 = dx0 + 1 - SQUISH_CONSTANT_4D\n dx_ext1 = dx0 - SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsb + 1\n dx_ext0 = dx_ext1 = dx0 - 1 - SQUISH_CONSTANT_4D\n\n if (c1 & 0x02) == 0:\n ysv_ext0 = ysv_ext1 = ysb\n dy_ext0 = dy_ext1 = dy0 - SQUISH_CONSTANT_4D\n if (c1 & 0x01) == 0x01:\n ysv_ext0 -= 1\n dy_ext0 += 1\n else:\n ysv_ext1 -= 1\n dy_ext1 += 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy_ext1 = dy0 - 1 - SQUISH_CONSTANT_4D\n\n if (c1 & 0x04) == 0:\n zsv_ext0 = zsv_ext1 = zsb\n dz_ext0 = dz_ext1 = dz0 - SQUISH_CONSTANT_4D\n if (c1 & 0x03) == 0x03:\n zsv_ext0 -= 1\n dz_ext0 += 1\n else:\n zsv_ext1 -= 1\n dz_ext1 += 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsb + 1\n dz_ext0 = dz_ext1 = dz0 - 1 - SQUISH_CONSTANT_4D\n\n if (c1 & 0x08) == 0:\n wsv_ext0 = wsb\n wsv_ext1 = wsb - 1\n dw_ext0 = dw0 - SQUISH_CONSTANT_4D\n dw_ext1 = dw0 + 1 - SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsb + 1\n dw_ext0 = dw_ext1 = dw0 - 1 - SQUISH_CONSTANT_4D\n\n # One contribution is a _permutation of (0,0,0,2) based on the smaller-sided po\n xsv_ext2 = xsb\n ysv_ext2 = ysb\n zsv_ext2 = zsb\n wsv_ext2 = wsb\n dx_ext2 = dx0 - 2 * SQUISH_CONSTANT_4D\n dy_ext2 = dy0 - 2 * SQUISH_CONSTANT_4D\n dz_ext2 = dz0 - 2 * SQUISH_CONSTANT_4D\n dw_ext2 = dw0 - 2 * SQUISH_CONSTANT_4D\n if (c2 & 0x01) != 0:\n xsv_ext2 += 2\n dx_ext2 -= 2\n elif (c2 & 0x02) != 0:\n ysv_ext2 += 2\n dy_ext2 -= 2\n elif (c2 & 0x04) != 0:\n zsv_ext2 += 2\n dz_ext2 -= 2\n else:\n wsv_ext2 += 2\n dw_ext2 -= 2\n\n # Contribution (1,0,0,0)\n dx1 = dx0 - 1 - SQUISH_CONSTANT_4D\n dy1 = dy0 - 0 - SQUISH_CONSTANT_4D\n dz1 = dz0 - 0 - SQUISH_CONSTANT_4D\n dw1 = dw0 - 0 - SQUISH_CONSTANT_4D\n attn1 = 2 - dx1 * dx1 - dy1 * dy1 - dz1 * dz1 - dw1 * dw1\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 1, ysb + 0, zsb + 0, wsb + 0, dx1, dy1, dz1, dw1)\n\n # Contribution (0,1,0,0)\n dx2 = dx0 - 0 - SQUISH_CONSTANT_4D\n dy2 = dy0 - 1 - SQUISH_CONSTANT_4D\n dz2 = dz1\n dw2 = dw1\n attn2 = 2 - dx2 * dx2 - dy2 * dy2 - dz2 * dz2 - dw2 * dw2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 0, ysb + 1, zsb + 0, wsb + 0, dx2, dy2, dz2, dw2)\n\n # Contribution (0,0,1,0)\n dx3 = dx2\n dy3 = dy1\n dz3 = dz0 - 1 - SQUISH_CONSTANT_4D\n dw3 = dw1\n attn3 = 2 - dx3 * dx3 - dy3 * dy3 - dz3 * dz3 - dw3 * dw3\n if attn3 > 0:\n attn3 *= attn3\n value += attn3 * attn3 * extrapolate(xsb + 0, ysb + 0, zsb + 1, wsb + 0, dx3, dy3, dz3, dw3)\n\n # Contribution (0,0,0,1)\n dx4 = dx2\n dy4 = dy1\n dz4 = dz1\n dw4 = dw0 - 1 - SQUISH_CONSTANT_4D\n attn4 = 2 - dx4 * dx4 - dy4 * dy4 - dz4 * dz4 - dw4 * dw4\n if attn4 > 0:\n attn4 *= attn4\n value += attn4 * attn4 * extrapolate(xsb + 0, ysb + 0, zsb + 0, wsb + 1, dx4, dy4, dz4, dw4)\n\n # Contribution (1,1,0,0)\n dx5 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy5 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz5 = dz0 - 0 - 2 * SQUISH_CONSTANT_4D\n dw5 = dw0 - 0 - 2 * SQUISH_CONSTANT_4D\n attn5 = 2 - dx5 * dx5 - dy5 * dy5 - dz5 * dz5 - dw5 * dw5\n if attn5 > 0:\n attn5 *= attn5\n value += attn5 * attn5 * extrapolate(xsb + 1, ysb + 1, zsb + 0, wsb + 0, dx5, dy5, dz5, dw5)\n\n # Contribution (1,0,1,0)\n dx6 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy6 = dy0 - 0 - 2 * SQUISH_CONSTANT_4D\n dz6 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw6 = dw0 - 0 - 2 * SQUISH_CONSTANT_4D\n attn6 = 2 - dx6 * dx6 - dy6 * dy6 - dz6 * dz6 - dw6 * dw6\n if attn6 > 0:\n attn6 *= attn6\n value += attn6 * attn6 * extrapolate(xsb + 1, ysb + 0, zsb + 1, wsb + 0, dx6, dy6, dz6, dw6)\n\n # Contribution (1,0,0,1)\n dx7 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy7 = dy0 - 0 - 2 * SQUISH_CONSTANT_4D\n dz7 = dz0 - 0 - 2 * SQUISH_CONSTANT_4D\n dw7 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n attn7 = 2 - dx7 * dx7 - dy7 * dy7 - dz7 * dz7 - dw7 * dw7\n if attn7 > 0:\n attn7 *= attn7\n value += attn7 * attn7 * extrapolate(xsb + 1, ysb + 0, zsb + 0, wsb + 1, dx7, dy7, dz7, dw7)\n\n # Contribution (0,1,1,0)\n dx8 = dx0 - 0 - 2 * SQUISH_CONSTANT_4D\n dy8 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz8 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw8 = dw0 - 0 - 2 * SQUISH_CONSTANT_4D\n attn8 = 2 - dx8 * dx8 - dy8 * dy8 - dz8 * dz8 - dw8 * dw8\n if attn8 > 0:\n attn8 *= attn8\n value += attn8 * attn8 * extrapolate(xsb + 0, ysb + 1, zsb + 1, wsb + 0, dx8, dy8, dz8, dw8)\n\n # Contribution (0,1,0,1)\n dx9 = dx0 - 0 - 2 * SQUISH_CONSTANT_4D\n dy9 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz9 = dz0 - 0 - 2 * SQUISH_CONSTANT_4D\n dw9 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n attn9 = 2 - dx9 * dx9 - dy9 * dy9 - dz9 * dz9 - dw9 * dw9\n if attn9 > 0:\n attn9 *= attn9\n value += attn9 * attn9 * extrapolate(xsb + 0, ysb + 1, zsb + 0, wsb + 1, dx9, dy9, dz9, dw9)\n\n # Contribution (0,0,1,1)\n dx10 = dx0 - 0 - 2 * SQUISH_CONSTANT_4D\n dy10 = dy0 - 0 - 2 * SQUISH_CONSTANT_4D\n dz10 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw10 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n attn10 = 2 - dx10 * dx10 - dy10 * dy10 - dz10 * dz10 - dw10 * dw10\n if attn10 > 0:\n attn10 *= attn10\n value += attn10 * attn10 * extrapolate(xsb + 0, ysb + 0, zsb + 1, wsb + 1, dx10, dy10, dz10, dw10)\n\n else: # We're inside the second dispentachoron (Rectified 4-Simplex)\n a_is_bigger_side = True\n b_is_bigger_side = True\n\n # Decide between (0,0,1,1) and (1,1,0,0)\n if xins + yins < zins + wins:\n a_score = xins + yins\n a_po = 0x0C\n else:\n a_score = zins + wins\n a_po = 0x03\n\n # Decide between (0,1,0,1) and (1,0,1,0)\n if xins + zins < yins + wins:\n b_score = xins + zins\n b_po = 0x0A\n else:\n b_score = yins + wins\n b_po = 0x05\n\n # Closer between (0,1,1,0) and (1,0,0,1) will replace the further of a and b, if closer.\n if xins + wins < yins + zins:\n score = xins + wins\n if a_score <= b_score and score < b_score:\n b_score = score\n b_po = 0x06\n elif a_score > b_score and score < a_score:\n a_score = score\n a_po = 0x06\n\n else:\n score = yins + zins\n if a_score <= b_score and score < b_score:\n b_score = score\n b_po = 0x09\n elif a_score > b_score and score < a_score:\n a_score = score\n a_po = 0x09\n\n # Decide if (0,1,1,1) is closer.\n p1 = 3 - in_sum + xins\n if a_score <= b_score and p1 < b_score:\n b_score = p1\n b_po = 0x0E\n b_is_bigger_side = False\n elif a_score > b_score and p1 < a_score:\n a_score = p1\n a_po = 0x0E\n a_is_bigger_side = False\n\n # Decide if (1,0,1,1) is closer.\n p2 = 3 - in_sum + yins\n if a_score <= b_score and p2 < b_score:\n b_score = p2\n b_po = 0x0D\n b_is_bigger_side = False\n elif a_score > b_score and p2 < a_score:\n a_score = p2\n a_po = 0x0D\n a_is_bigger_side = False\n\n # Decide if (1,1,0,1) is closer.\n p3 = 3 - in_sum + zins\n if a_score <= b_score and p3 < b_score:\n b_score = p3\n b_po = 0x0B\n b_is_bigger_side = False\n elif a_score > b_score and p3 < a_score:\n a_score = p3\n a_po = 0x0B\n a_is_bigger_side = False\n\n # Decide if (1,1,1,0) is closer.\n p4 = 3 - in_sum + wins\n if a_score <= b_score and p4 < b_score:\n b_po = 0x07\n b_is_bigger_side = False\n elif a_score > b_score and p4 < a_score:\n a_po = 0x07\n a_is_bigger_side = False\n\n # Where each of the two closest pos are determines how the extra three vertices are calculated.\n if a_is_bigger_side == b_is_bigger_side:\n if a_is_bigger_side: # Both closest pos on the bigger side\n c1 = (a_po & b_po)\n c2 = (a_po | b_po)\n\n # Two contributions are _permutations of (0,0,0,1) and (0,0,0,2) based on c1\n xsv_ext0 = xsv_ext1 = xsb\n ysv_ext0 = ysv_ext1 = ysb\n zsv_ext0 = zsv_ext1 = zsb\n wsv_ext0 = wsv_ext1 = wsb\n dx_ext0 = dx0 - SQUISH_CONSTANT_4D\n dy_ext0 = dy0 - SQUISH_CONSTANT_4D\n dz_ext0 = dz0 - SQUISH_CONSTANT_4D\n dw_ext0 = dw0 - SQUISH_CONSTANT_4D\n dx_ext1 = dx0 - 2 * SQUISH_CONSTANT_4D\n dy_ext1 = dy0 - 2 * SQUISH_CONSTANT_4D\n dz_ext1 = dz0 - 2 * SQUISH_CONSTANT_4D\n dw_ext1 = dw0 - 2 * SQUISH_CONSTANT_4D\n if (c1 & 0x01) != 0:\n xsv_ext0 += 1\n dx_ext0 -= 1\n xsv_ext1 += 2\n dx_ext1 -= 2\n elif (c1 & 0x02) != 0:\n ysv_ext0 += 1\n dy_ext0 -= 1\n ysv_ext1 += 2\n dy_ext1 -= 2\n elif (c1 & 0x04) != 0:\n zsv_ext0 += 1\n dz_ext0 -= 1\n zsv_ext1 += 2\n dz_ext1 -= 2\n else:\n wsv_ext0 += 1\n dw_ext0 -= 1\n wsv_ext1 += 2\n dw_ext1 -= 2\n\n # One contribution is a _permutation of (1,1,1,-1) based on c2\n xsv_ext2 = xsb + 1\n ysv_ext2 = ysb + 1\n zsv_ext2 = zsb + 1\n wsv_ext2 = wsb + 1\n dx_ext2 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy_ext2 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz_ext2 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw_ext2 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n if (c2 & 0x01) == 0:\n xsv_ext2 -= 2\n dx_ext2 += 2\n elif (c2 & 0x02) == 0:\n ysv_ext2 -= 2\n dy_ext2 += 2\n elif (c2 & 0x04) == 0:\n zsv_ext2 -= 2\n dz_ext2 += 2\n else:\n wsv_ext2 -= 2\n dw_ext2 += 2\n\n else: # Both closest pos on the smaller side\n # One of the two extra pos is (1,1,1,1)\n xsv_ext2 = xsb + 1\n ysv_ext2 = ysb + 1\n zsv_ext2 = zsb + 1\n wsv_ext2 = wsb + 1\n dx_ext2 = dx0 - 1 - 4 * SQUISH_CONSTANT_4D\n dy_ext2 = dy0 - 1 - 4 * SQUISH_CONSTANT_4D\n dz_ext2 = dz0 - 1 - 4 * SQUISH_CONSTANT_4D\n dw_ext2 = dw0 - 1 - 4 * SQUISH_CONSTANT_4D\n\n # Other two pos are based on the shared axes.\n c = (a_po & b_po)\n if (c & 0x01) != 0:\n xsv_ext0 = xsb + 2\n xsv_ext1 = xsb + 1\n dx_ext0 = dx0 - 2 - 3 * SQUISH_CONSTANT_4D\n dx_ext1 = dx0 - 1 - 3 * SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsb\n dx_ext0 = dx_ext1 = dx0 - 3 * SQUISH_CONSTANT_4D\n\n if (c & 0x02) != 0:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy_ext1 = dy0 - 1 - 3 * SQUISH_CONSTANT_4D\n if (c & 0x01) == 0:\n ysv_ext0 += 1\n dy_ext0 -= 1\n else:\n ysv_ext1 += 1\n dy_ext1 -= 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysb\n dy_ext0 = dy_ext1 = dy0 - 3 * SQUISH_CONSTANT_4D\n\n if (c & 0x04) != 0:\n zsv_ext0 = zsv_ext1 = zsb + 1\n dz_ext0 = dz_ext1 = dz0 - 1 - 3 * SQUISH_CONSTANT_4D\n if (c & 0x03) == 0:\n zsv_ext0 += 1\n dz_ext0 -= 1\n else:\n zsv_ext1 += 1\n dz_ext1 -= 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsb\n dz_ext0 = dz_ext1 = dz0 - 3 * SQUISH_CONSTANT_4D\n\n\n if (c & 0x08) != 0:\n wsv_ext0 = wsb + 1\n wsv_ext1 = wsb + 2\n dw_ext0 = dw0 - 1 - 3 * SQUISH_CONSTANT_4D\n dw_ext1 = dw0 - 2 - 3 * SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsb\n dw_ext0 = dw_ext1 = dw0 - 3 * SQUISH_CONSTANT_4D\n\n else: # One po on each \"side\"\n if a_is_bigger_side:\n c1 = a_po\n c2 = b_po\n else:\n c1 = b_po\n c2 = a_po\n\n # Two contributions are the bigger-sided po with each 1 replaced with 2.\n if (c1 & 0x01) != 0:\n xsv_ext0 = xsb + 2\n xsv_ext1 = xsb + 1\n dx_ext0 = dx0 - 2 - 3 * SQUISH_CONSTANT_4D\n dx_ext1 = dx0 - 1 - 3 * SQUISH_CONSTANT_4D\n else:\n xsv_ext0 = xsv_ext1 = xsb\n dx_ext0 = dx_ext1 = dx0 - 3 * SQUISH_CONSTANT_4D\n\n if (c1 & 0x02) != 0:\n ysv_ext0 = ysv_ext1 = ysb + 1\n dy_ext0 = dy_ext1 = dy0 - 1 - 3 * SQUISH_CONSTANT_4D\n if (c1 & 0x01) == 0:\n ysv_ext0 += 1\n dy_ext0 -= 1\n else:\n ysv_ext1 += 1\n dy_ext1 -= 1\n\n else:\n ysv_ext0 = ysv_ext1 = ysb\n dy_ext0 = dy_ext1 = dy0 - 3 * SQUISH_CONSTANT_4D\n\n if (c1 & 0x04) != 0:\n zsv_ext0 = zsv_ext1 = zsb + 1\n dz_ext0 = dz_ext1 = dz0 - 1 - 3 * SQUISH_CONSTANT_4D\n if (c1 & 0x03) == 0:\n zsv_ext0 += 1\n dz_ext0 -= 1\n else:\n zsv_ext1 += 1\n dz_ext1 -= 1\n\n else:\n zsv_ext0 = zsv_ext1 = zsb\n dz_ext0 = dz_ext1 = dz0 - 3 * SQUISH_CONSTANT_4D\n\n if (c1 & 0x08) != 0:\n wsv_ext0 = wsb + 1\n wsv_ext1 = wsb + 2\n dw_ext0 = dw0 - 1 - 3 * SQUISH_CONSTANT_4D\n dw_ext1 = dw0 - 2 - 3 * SQUISH_CONSTANT_4D\n else:\n wsv_ext0 = wsv_ext1 = wsb\n dw_ext0 = dw_ext1 = dw0 - 3 * SQUISH_CONSTANT_4D\n\n # One contribution is a _permutation of (1,1,1,-1) based on the smaller-sided po\n xsv_ext2 = xsb + 1\n ysv_ext2 = ysb + 1\n zsv_ext2 = zsb + 1\n wsv_ext2 = wsb + 1\n dx_ext2 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy_ext2 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz_ext2 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw_ext2 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n if (c2 & 0x01) == 0:\n xsv_ext2 -= 2\n dx_ext2 += 2\n elif (c2 & 0x02) == 0:\n ysv_ext2 -= 2\n dy_ext2 += 2\n elif (c2 & 0x04) == 0:\n zsv_ext2 -= 2\n dz_ext2 += 2\n else:\n wsv_ext2 -= 2\n dw_ext2 += 2\n\n # Contribution (1,1,1,0)\n dx4 = dx0 - 1 - 3 * SQUISH_CONSTANT_4D\n dy4 = dy0 - 1 - 3 * SQUISH_CONSTANT_4D\n dz4 = dz0 - 1 - 3 * SQUISH_CONSTANT_4D\n dw4 = dw0 - 3 * SQUISH_CONSTANT_4D\n attn4 = 2 - dx4 * dx4 - dy4 * dy4 - dz4 * dz4 - dw4 * dw4\n if attn4 > 0:\n attn4 *= attn4\n value += attn4 * attn4 * extrapolate(xsb + 1, ysb + 1, zsb + 1, wsb + 0, dx4, dy4, dz4, dw4)\n\n # Contribution (1,1,0,1)\n dx3 = dx4\n dy3 = dy4\n dz3 = dz0 - 3 * SQUISH_CONSTANT_4D\n dw3 = dw0 - 1 - 3 * SQUISH_CONSTANT_4D\n attn3 = 2 - dx3 * dx3 - dy3 * dy3 - dz3 * dz3 - dw3 * dw3\n if attn3 > 0:\n attn3 *= attn3\n value += attn3 * attn3 * extrapolate(xsb + 1, ysb + 1, zsb + 0, wsb + 1, dx3, dy3, dz3, dw3)\n\n # Contribution (1,0,1,1)\n dx2 = dx4\n dy2 = dy0 - 3 * SQUISH_CONSTANT_4D\n dz2 = dz4\n dw2 = dw3\n attn2 = 2 - dx2 * dx2 - dy2 * dy2 - dz2 * dz2 - dw2 * dw2\n if attn2 > 0:\n attn2 *= attn2\n value += attn2 * attn2 * extrapolate(xsb + 1, ysb + 0, zsb + 1, wsb + 1, dx2, dy2, dz2, dw2)\n\n # Contribution (0,1,1,1)\n dx1 = dx0 - 3 * SQUISH_CONSTANT_4D\n dz1 = dz4\n dy1 = dy4\n dw1 = dw3\n attn1 = 2 - dx1 * dx1 - dy1 * dy1 - dz1 * dz1 - dw1 * dw1\n if attn1 > 0:\n attn1 *= attn1\n value += attn1 * attn1 * extrapolate(xsb + 0, ysb + 1, zsb + 1, wsb + 1, dx1, dy1, dz1, dw1)\n\n # Contribution (1,1,0,0)\n dx5 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy5 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz5 = dz0 - 0 - 2 * SQUISH_CONSTANT_4D\n dw5 = dw0 - 0 - 2 * SQUISH_CONSTANT_4D\n attn5 = 2 - dx5 * dx5 - dy5 * dy5 - dz5 * dz5 - dw5 * dw5\n if attn5 > 0:\n attn5 *= attn5\n value += attn5 * attn5 * extrapolate(xsb + 1, ysb + 1, zsb + 0, wsb + 0, dx5, dy5, dz5, dw5)\n\n # Contribution (1,0,1,0)\n dx6 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy6 = dy0 - 0 - 2 * SQUISH_CONSTANT_4D\n dz6 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw6 = dw0 - 0 - 2 * SQUISH_CONSTANT_4D\n attn6 = 2 - dx6 * dx6 - dy6 * dy6 - dz6 * dz6 - dw6 * dw6\n if attn6 > 0:\n attn6 *= attn6\n value += attn6 * attn6 * extrapolate(xsb + 1, ysb + 0, zsb + 1, wsb + 0, dx6, dy6, dz6, dw6)\n\n # Contribution (1,0,0,1)\n dx7 = dx0 - 1 - 2 * SQUISH_CONSTANT_4D\n dy7 = dy0 - 0 - 2 * SQUISH_CONSTANT_4D\n dz7 = dz0 - 0 - 2 * SQUISH_CONSTANT_4D\n dw7 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n attn7 = 2 - dx7 * dx7 - dy7 * dy7 - dz7 * dz7 - dw7 * dw7\n if attn7 > 0:\n attn7 *= attn7\n value += attn7 * attn7 * extrapolate(xsb + 1, ysb + 0, zsb + 0, wsb + 1, dx7, dy7, dz7, dw7)\n\n # Contribution (0,1,1,0)\n dx8 = dx0 - 0 - 2 * SQUISH_CONSTANT_4D\n dy8 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz8 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw8 = dw0 - 0 - 2 * SQUISH_CONSTANT_4D\n attn8 = 2 - dx8 * dx8 - dy8 * dy8 - dz8 * dz8 - dw8 * dw8\n if attn8 > 0:\n attn8 *= attn8\n value += attn8 * attn8 * extrapolate(xsb + 0, ysb + 1, zsb + 1, wsb + 0, dx8, dy8, dz8, dw8)\n\n # Contribution (0,1,0,1)\n dx9 = dx0 - 0 - 2 * SQUISH_CONSTANT_4D\n dy9 = dy0 - 1 - 2 * SQUISH_CONSTANT_4D\n dz9 = dz0 - 0 - 2 * SQUISH_CONSTANT_4D\n dw9 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n attn9 = 2 - dx9 * dx9 - dy9 * dy9 - dz9 * dz9 - dw9 * dw9\n if attn9 > 0:\n attn9 *= attn9\n value += attn9 * attn9 * extrapolate(xsb + 0, ysb + 1, zsb + 0, wsb + 1, dx9, dy9, dz9, dw9)\n\n # Contribution (0,0,1,1)\n dx10 = dx0 - 0 - 2 * SQUISH_CONSTANT_4D\n dy10 = dy0 - 0 - 2 * SQUISH_CONSTANT_4D\n dz10 = dz0 - 1 - 2 * SQUISH_CONSTANT_4D\n dw10 = dw0 - 1 - 2 * SQUISH_CONSTANT_4D\n attn10 = 2 - dx10 * dx10 - dy10 * dy10 - dz10 * dz10 - dw10 * dw10\n if attn10 > 0:\n attn10 *= attn10\n value += attn10 * attn10 * extrapolate(xsb + 0, ysb + 0, zsb + 1, wsb + 1, dx10, dy10, dz10, dw10)\n\n # First extra vertex\n attn_ext0 = 2 - dx_ext0 * dx_ext0 - dy_ext0 * dy_ext0 - dz_ext0 * dz_ext0 - dw_ext0 * dw_ext0\n if attn_ext0 > 0:\n attn_ext0 *= attn_ext0\n value += attn_ext0 * attn_ext0 * extrapolate(xsv_ext0, ysv_ext0, zsv_ext0, wsv_ext0, dx_ext0, dy_ext0, dz_ext0, dw_ext0)\n\n # Second extra vertex\n attn_ext1 = 2 - dx_ext1 * dx_ext1 - dy_ext1 * dy_ext1 - dz_ext1 * dz_ext1 - dw_ext1 * dw_ext1\n if attn_ext1 > 0:\n attn_ext1 *= attn_ext1\n value += attn_ext1 * attn_ext1 * extrapolate(xsv_ext1, ysv_ext1, zsv_ext1, wsv_ext1, dx_ext1, dy_ext1, dz_ext1, dw_ext1)\n\n # Third extra vertex\n attn_ext2 = 2 - dx_ext2 * dx_ext2 - dy_ext2 * dy_ext2 - dz_ext2 * dz_ext2 - dw_ext2 * dw_ext2\n if attn_ext2 > 0:\n attn_ext2 *= attn_ext2\n value += attn_ext2 * attn_ext2 * extrapolate(xsv_ext2, ysv_ext2, zsv_ext2, wsv_ext2, dx_ext2, dy_ext2, dz_ext2, dw_ext2)\n\n return value / NORM_CONSTANT_4D", "response": "Generate 4D OpenSimplex noise from X Y Z W coordinates."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadjust the contrast of the current image by scaling each pixel value to 255 and scaling each pixel value to gamma.", "response": "def adjust_contrast_gamma(arr, gamma):\n \"\"\"\n Adjust contrast by scaling each pixel value to ``255 * ((I_ij/255)**gamma)``.\n\n dtype support::\n\n * ``uint8``: yes; fully tested (1) (2) (3)\n * ``uint16``: yes; tested (2) (3)\n * ``uint32``: yes; tested (2) (3)\n * ``uint64``: yes; tested (2) (3) (4)\n * ``int8``: limited; tested (2) (3) (5)\n * ``int16``: limited; tested (2) (3) (5)\n * ``int32``: limited; tested (2) (3) (5)\n * ``int64``: limited; tested (2) (3) (4) (5)\n * ``float16``: limited; tested (5)\n * ``float32``: limited; tested (5)\n * ``float64``: limited; tested (5)\n * ``float128``: no (6)\n * ``bool``: no (7)\n\n - (1) Handled by ``cv2``. Other dtypes are handled by ``skimage``.\n - (2) Normalization is done as ``I_ij/max``, where ``max`` is the maximum value of the\n dtype, e.g. 255 for ``uint8``. The normalization is reversed afterwards,\n e.g. ``result*255`` for ``uint8``.\n - (3) Integer-like values are not rounded after applying the contrast adjustment equation\n (before inverting the normalization to 0.0-1.0 space), i.e. projection from continuous\n space to discrete happens according to floor function.\n - (4) Note that scikit-image doc says that integers are converted to ``float64`` values before\n applying the contrast normalization method. This might lead to inaccuracies for large\n 64bit integer values. Tests showed no indication of that happening though.\n - (5) Must not contain negative values. Values >=0 are fully supported.\n - (6) Leads to error in scikit-image.\n - (7) Does not make sense for contrast adjustments.\n\n Parameters\n ----------\n arr : numpy.ndarray\n Array for which to adjust the contrast. Dtype ``uint8`` is fastest.\n\n gamma : number\n Exponent for the contrast adjustment. Higher values darken the image.\n\n Returns\n -------\n numpy.ndarray\n Array with adjusted contrast.\n\n \"\"\"\n # int8 is also possible according to docs\n # https://docs.opencv.org/3.0-beta/modules/core/doc/operations_on_arrays.html#cv2.LUT , but here it seemed\n # like `d` was 0 for CV_8S, causing that to fail\n if arr.dtype.name == \"uint8\":\n min_value, _center_value, max_value = iadt.get_value_range_of_dtype(arr.dtype)\n dynamic_range = max_value - min_value\n\n value_range = np.linspace(0, 1.0, num=dynamic_range+1, dtype=np.float32)\n # 255 * ((I_ij/255)**gamma)\n # using np.float32(.) here still works when the input is a numpy array of size 1\n table = (min_value + (value_range ** np.float32(gamma)) * dynamic_range)\n arr_aug = cv2.LUT(arr, np.clip(table, min_value, max_value).astype(arr.dtype))\n if arr.ndim == 3 and arr_aug.ndim == 2:\n return arr_aug[..., np.newaxis]\n return arr_aug\n else:\n return ski_exposure.adjust_gamma(arr, gamma)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef adjust_contrast_sigmoid(arr, gain, cutoff):\n # int8 is also possible according to docs\n # https://docs.opencv.org/3.0-beta/modules/core/doc/operations_on_arrays.html#cv2.LUT , but here it seemed\n # like `d` was 0 for CV_8S, causing that to fail\n if arr.dtype.name == \"uint8\":\n min_value, _center_value, max_value = iadt.get_value_range_of_dtype(arr.dtype)\n dynamic_range = max_value - min_value\n\n value_range = np.linspace(0, 1.0, num=dynamic_range+1, dtype=np.float32)\n # 255 * 1/(1 + exp(gain*(cutoff - I_ij/255)))\n # using np.float32(.) here still works when the input is a numpy array of size 1\n gain = np.float32(gain)\n cutoff = np.float32(cutoff)\n table = min_value + dynamic_range * 1/(1 + np.exp(gain * (cutoff - value_range)))\n arr_aug = cv2.LUT(arr, np.clip(table, min_value, max_value).astype(arr.dtype))\n if arr.ndim == 3 and arr_aug.ndim == 2:\n return arr_aug[..., np.newaxis]\n return arr_aug\n else:\n return ski_exposure.adjust_sigmoid(arr, cutoff=cutoff, gain=gain)", "response": "Adjusts the contrast of the current image by scaling each pixel value to 1 - exp ( gain * cutoff - I_ij"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadjusts the contrast of the current image by scaling each pixel value to gain log_2.", "response": "def adjust_contrast_log(arr, gain):\n \"\"\"\n Adjust contrast by scaling each pixel value to ``255 * gain * log_2(1 + I_ij/255)``.\n\n dtype support::\n\n * ``uint8``: yes; fully tested (1) (2) (3)\n * ``uint16``: yes; tested (2) (3)\n * ``uint32``: yes; tested (2) (3)\n * ``uint64``: yes; tested (2) (3) (4)\n * ``int8``: limited; tested (2) (3) (5)\n * ``int16``: limited; tested (2) (3) (5)\n * ``int32``: limited; tested (2) (3) (5)\n * ``int64``: limited; tested (2) (3) (4) (5)\n * ``float16``: limited; tested (5)\n * ``float32``: limited; tested (5)\n * ``float64``: limited; tested (5)\n * ``float128``: no (6)\n * ``bool``: no (7)\n\n - (1) Handled by ``cv2``. Other dtypes are handled by ``skimage``.\n - (2) Normalization is done as ``I_ij/max``, where ``max`` is the maximum value of the\n dtype, e.g. 255 for ``uint8``. The normalization is reversed afterwards,\n e.g. ``result*255`` for ``uint8``.\n - (3) Integer-like values are not rounded after applying the contrast adjustment equation\n (before inverting the normalization to 0.0-1.0 space), i.e. projection from continuous\n space to discrete happens according to floor function.\n - (4) Note that scikit-image doc says that integers are converted to ``float64`` values before\n applying the contrast normalization method. This might lead to inaccuracies for large\n 64bit integer values. Tests showed no indication of that happening though.\n - (5) Must not contain negative values. Values >=0 are fully supported.\n - (6) Leads to error in scikit-image.\n - (7) Does not make sense for contrast adjustments.\n\n Parameters\n ----------\n arr : numpy.ndarray\n Array for which to adjust the contrast. Dtype ``uint8`` is fastest.\n\n gain : number\n Multiplier for the logarithm result. Values around 1.0 lead to a contrast-adjusted\n images. Values above 1.0 quickly lead to partially broken images due to exceeding the\n datatype's value range.\n\n Returns\n -------\n numpy.ndarray\n Array with adjusted contrast.\n\n \"\"\"\n # int8 is also possible according to docs\n # https://docs.opencv.org/3.0-beta/modules/core/doc/operations_on_arrays.html#cv2.LUT , but here it seemed\n # like `d` was 0 for CV_8S, causing that to fail\n if arr.dtype.name == \"uint8\":\n min_value, _center_value, max_value = iadt.get_value_range_of_dtype(arr.dtype)\n dynamic_range = max_value - min_value\n\n value_range = np.linspace(0, 1.0, num=dynamic_range+1, dtype=np.float32)\n # 255 * 1/(1 + exp(gain*(cutoff - I_ij/255)))\n # using np.float32(.) here still works when the input is a numpy array of size 1\n gain = np.float32(gain)\n table = min_value + dynamic_range * gain * np.log2(1 + value_range)\n arr_aug = cv2.LUT(arr, np.clip(table, min_value, max_value).astype(arr.dtype))\n if arr.ndim == 3 and arr_aug.ndim == 2:\n return arr_aug[..., np.newaxis]\n return arr_aug\n else:\n return ski_exposure.adjust_log(arr, gain=gain)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadjust the contrast of a set of images in a new node.", "response": "def adjust_contrast_linear(arr, alpha):\n \"\"\"Adjust contrast by scaling each pixel value to ``127 + alpha*(I_ij-127)``.\n\n dtype support::\n\n * ``uint8``: yes; fully tested (1) (2)\n * ``uint16``: yes; tested (2)\n * ``uint32``: yes; tested (2)\n * ``uint64``: no (3)\n * ``int8``: yes; tested (2)\n * ``int16``: yes; tested (2)\n * ``int32``: yes; tested (2)\n * ``int64``: no (2)\n * ``float16``: yes; tested (2)\n * ``float32``: yes; tested (2)\n * ``float64``: yes; tested (2)\n * ``float128``: no (2)\n * ``bool``: no (4)\n\n - (1) Handled by ``cv2``. Other dtypes are handled by raw ``numpy``.\n - (2) Only tested for reasonable alphas with up to a value of around 100.\n - (3) Conversion to ``float64`` is done during augmentation, hence ``uint64``, ``int64``,\n and ``float128`` support cannot be guaranteed.\n - (4) Does not make sense for contrast adjustments.\n\n Parameters\n ----------\n arr : numpy.ndarray\n Array for which to adjust the contrast. Dtype ``uint8`` is fastest.\n\n alpha : number\n Multiplier to linearly pronounce (>1.0), dampen (0.0 to 1.0) or invert (<0.0) the\n difference between each pixel value and the center value, e.g. ``127`` for ``uint8``.\n\n Returns\n -------\n numpy.ndarray\n Array with adjusted contrast.\n\n \"\"\"\n # int8 is also possible according to docs\n # https://docs.opencv.org/3.0-beta/modules/core/doc/operations_on_arrays.html#cv2.LUT , but here it seemed\n # like `d` was 0 for CV_8S, causing that to fail\n if arr.dtype.name == \"uint8\":\n min_value, center_value, max_value = iadt.get_value_range_of_dtype(arr.dtype)\n\n value_range = np.arange(0, 256, dtype=np.float32)\n # 127 + alpha*(I_ij-127)\n # using np.float32(.) here still works when the input is a numpy array of size 1\n alpha = np.float32(alpha)\n table = center_value + alpha * (value_range - center_value)\n arr_aug = cv2.LUT(arr, np.clip(table, min_value, max_value).astype(arr.dtype))\n if arr.ndim == 3 and arr_aug.ndim == 2:\n return arr_aug[..., np.newaxis]\n return arr_aug\n else:\n input_dtype = arr.dtype\n _min_value, center_value, _max_value = iadt.get_value_range_of_dtype(input_dtype)\n if input_dtype.kind in [\"u\", \"i\"]:\n center_value = int(center_value)\n image_aug = center_value + alpha * (arr.astype(np.float64)-center_value)\n image_aug = iadt.restore_dtypes_(image_aug, input_dtype)\n return image_aug"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadjust contrast by scaling each pixel value to ``255 * ((I_ij/255)**gamma)``. Values in the range ``gamma=(0.5, 2.0)`` seem to be sensible. dtype support:: See :func:`imgaug.augmenters.contrast.adjust_contrast_gamma`. Parameters ---------- gamma : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional Exponent for the contrast adjustment. Higher values darken the image. * If a number, then that value will be used for all images. * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image. * If a list, then a random value will be sampled from that list per image. * If a StochasticParameter, then a value will be sampled per image from that parameter. per_channel : bool or float, optional Whether to use the same value for all channels (False) or to sample a new value for each channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel` will be treated as True, otherwise as False. name : None or str, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. deterministic : bool, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. random_state : None or int or numpy.random.RandomState, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. Returns ------- _ContrastFuncWrapper Augmenter to perform gamma contrast adjustment.", "response": "def GammaContrast(gamma=1, per_channel=False, name=None, deterministic=False, random_state=None):\n \"\"\"\n Adjust contrast by scaling each pixel value to ``255 * ((I_ij/255)**gamma)``.\n\n Values in the range ``gamma=(0.5, 2.0)`` seem to be sensible.\n\n dtype support::\n\n See :func:`imgaug.augmenters.contrast.adjust_contrast_gamma`.\n\n Parameters\n ----------\n gamma : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional\n Exponent for the contrast adjustment. Higher values darken the image.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n per_channel : bool or float, optional\n Whether to use the same value for all channels (False) or to sample a new value for each\n channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel`\n will be treated as True, otherwise as False.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Returns\n -------\n _ContrastFuncWrapper\n Augmenter to perform gamma contrast adjustment.\n\n \"\"\"\n params1d = [iap.handle_continuous_param(gamma, \"gamma\", value_range=None, tuple_to_uniform=True,\n list_to_choice=True)]\n func = adjust_contrast_gamma\n return _ContrastFuncWrapper(\n func, params1d, per_channel,\n dtypes_allowed=[\"uint8\", \"uint16\", \"uint32\", \"uint64\",\n \"int8\", \"int16\", \"int32\", \"int64\",\n \"float16\", \"float32\", \"float64\"],\n dtypes_disallowed=[\"float96\", \"float128\", \"float256\", \"bool\"],\n name=name if name is not None else ia.caller_name(),\n deterministic=deterministic,\n random_state=random_state\n )"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a new structure of the sigmoid contrast for the given set of images.", "response": "def SigmoidContrast(gain=10, cutoff=0.5, per_channel=False, name=None, deterministic=False, random_state=None):\n \"\"\"\n Adjust contrast by scaling each pixel value to ``255 * 1/(1 + exp(gain*(cutoff - I_ij/255)))``.\n\n Values in the range ``gain=(5, 20)`` and ``cutoff=(0.25, 0.75)`` seem to be sensible.\n\n dtype support::\n\n See :func:`imgaug.augmenters.contrast.adjust_contrast_sigmoid`.\n\n Parameters\n ----------\n gain : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional\n Multiplier for the sigmoid function's output.\n Higher values lead to quicker changes from dark to light pixels.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n cutoff : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional\n Cutoff that shifts the sigmoid function in horizontal direction.\n Higher values mean that the switch from dark to light pixels happens later, i.e.\n the pixels will remain darker.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n per_channel : bool or float, optional\n Whether to use the same value for all channels (False) or to sample a new value for each\n channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel`\n will be treated as True, otherwise as False.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Returns\n -------\n _ContrastFuncWrapper\n Augmenter to perform sigmoid contrast adjustment.\n\n \"\"\"\n # TODO add inv parameter?\n params1d = [\n iap.handle_continuous_param(gain, \"gain\", value_range=(0, None), tuple_to_uniform=True, list_to_choice=True),\n iap.handle_continuous_param(cutoff, \"cutoff\", value_range=(0, 1.0), tuple_to_uniform=True, list_to_choice=True)\n ]\n func = adjust_contrast_sigmoid\n return _ContrastFuncWrapper(\n func, params1d, per_channel,\n dtypes_allowed=[\"uint8\", \"uint16\", \"uint32\", \"uint64\",\n \"int8\", \"int16\", \"int32\", \"int64\",\n \"float16\", \"float32\", \"float64\"],\n dtypes_disallowed=[\"float96\", \"float128\", \"float256\", \"bool\"],\n name=name if name is not None else ia.caller_name(),\n deterministic=deterministic,\n random_state=random_state\n )"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadjusting contrast by scaling each pixel value to ``255 * gain * log_2(1 + I_ij/255)``. dtype support:: See :func:`imgaug.augmenters.contrast.adjust_contrast_log`. Parameters ---------- gain : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional Multiplier for the logarithm result. Values around 1.0 lead to a contrast-adjusted images. Values above 1.0 quickly lead to partially broken images due to exceeding the datatype's value range. * If a number, then that value will be used for all images. * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image. * If a list, then a random value will be sampled from that list per image. * If a StochasticParameter, then a value will be sampled per image from that parameter. per_channel : bool or float, optional Whether to use the same value for all channels (False) or to sample a new value for each channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel` will be treated as True, otherwise as False. name : None or str, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. deterministic : bool, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. random_state : None or int or numpy.random.RandomState, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. Returns ------- _ContrastFuncWrapper Augmenter to perform logarithmic contrast adjustment.", "response": "def LogContrast(gain=1, per_channel=False, name=None, deterministic=False, random_state=None):\n \"\"\"\n Adjust contrast by scaling each pixel value to ``255 * gain * log_2(1 + I_ij/255)``.\n\n dtype support::\n\n See :func:`imgaug.augmenters.contrast.adjust_contrast_log`.\n\n Parameters\n ----------\n gain : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional\n Multiplier for the logarithm result. Values around 1.0 lead to a contrast-adjusted\n images. Values above 1.0 quickly lead to partially broken images due to exceeding the\n datatype's value range.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n per_channel : bool or float, optional\n Whether to use the same value for all channels (False) or to sample a new value for each\n channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel`\n will be treated as True, otherwise as False.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Returns\n -------\n _ContrastFuncWrapper\n Augmenter to perform logarithmic contrast adjustment.\n\n \"\"\"\n # TODO add inv parameter?\n params1d = [iap.handle_continuous_param(gain, \"gain\", value_range=(0, None), tuple_to_uniform=True,\n list_to_choice=True)]\n func = adjust_contrast_log\n return _ContrastFuncWrapper(\n func, params1d, per_channel,\n dtypes_allowed=[\"uint8\", \"uint16\", \"uint32\", \"uint64\",\n \"int8\", \"int16\", \"int32\", \"int64\",\n \"float16\", \"float32\", \"float64\"],\n dtypes_disallowed=[\"float96\", \"float128\", \"float256\", \"bool\"],\n name=name if name is not None else ia.caller_name(),\n deterministic=deterministic,\n random_state=random_state\n )"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadjusting contrast by scaling each pixel value to ``127 + alpha*(I_ij-127)``. dtype support:: See :func:`imgaug.augmenters.contrast.adjust_contrast_linear`. Parameters ---------- alpha : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional Multiplier to linearly pronounce (>1.0), dampen (0.0 to 1.0) or invert (<0.0) the difference between each pixel value and the center value, e.g. ``127`` for ``uint8``. * If a number, then that value will be used for all images. * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image. * If a list, then a random value will be sampled from that list per image. * If a StochasticParameter, then a value will be sampled per image from that parameter. per_channel : bool or float, optional Whether to use the same value for all channels (False) or to sample a new value for each channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel` will be treated as True, otherwise as False. name : None or str, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. deterministic : bool, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. random_state : None or int or numpy.random.RandomState, optional See :func:`imgaug.augmenters.meta.Augmenter.__init__`. Returns ------- _ContrastFuncWrapper Augmenter to perform contrast adjustment by linearly scaling the distance to 128.", "response": "def LinearContrast(alpha=1, per_channel=False, name=None, deterministic=False, random_state=None):\n \"\"\"Adjust contrast by scaling each pixel value to ``127 + alpha*(I_ij-127)``.\n\n dtype support::\n\n See :func:`imgaug.augmenters.contrast.adjust_contrast_linear`.\n\n Parameters\n ----------\n alpha : number or tuple of number or list of number or imgaug.parameters.StochasticParameter, optional\n Multiplier to linearly pronounce (>1.0), dampen (0.0 to 1.0) or invert (<0.0) the\n difference between each pixel value and the center value, e.g. ``127`` for ``uint8``.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the range ``[a, b]`` will be used per image.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n per_channel : bool or float, optional\n Whether to use the same value for all channels (False) or to sample a new value for each\n channel (True). If this value is a float ``p``, then for ``p`` percent of all images `per_channel`\n will be treated as True, otherwise as False.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Returns\n -------\n _ContrastFuncWrapper\n Augmenter to perform contrast adjustment by linearly scaling the distance to 128.\n\n \"\"\"\n params1d = [\n iap.handle_continuous_param(alpha, \"alpha\", value_range=None, tuple_to_uniform=True, list_to_choice=True)\n ]\n func = adjust_contrast_linear\n return _ContrastFuncWrapper(\n func, params1d, per_channel,\n dtypes_allowed=[\"uint8\", \"uint16\", \"uint32\",\n \"int8\", \"int16\", \"int32\",\n \"float16\", \"float32\", \"float64\"],\n dtypes_disallowed=[\"uint64\", \"int64\", \"float96\", \"float128\", \"float256\", \"bool\"],\n name=name if name is not None else ia.caller_name(),\n deterministic=deterministic,\n random_state=random_state\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts images to another colorspace.", "response": "def InColorspace(to_colorspace, from_colorspace=\"RGB\", children=None, name=None, deterministic=False,\n random_state=None):\n \"\"\"Convert images to another colorspace.\"\"\"\n return WithColorspace(to_colorspace, from_colorspace, children, name, deterministic, random_state)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the height of a bounding box encapsulating the line.", "response": "def height(self):\n \"\"\"Get the height of a bounding box encapsulating the line.\"\"\"\n if len(self.coords) <= 1:\n return 0\n return np.max(self.yy) - np.min(self.yy)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef width(self):\n if len(self.coords) <= 1:\n return 0\n return np.max(self.xx) - np.min(self.xx)", "response": "Get the width of a bounding box encapsulating the line."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_pointwise_inside_image_mask(self, image):\n if len(self.coords) == 0:\n return np.zeros((0,), dtype=bool)\n shape = normalize_shape(image)\n height, width = shape[0:2]\n x_within = np.logical_and(0 <= self.xx, self.xx < width)\n y_within = np.logical_and(0 <= self.yy, self.yy < height)\n return np.logical_and(x_within, y_within)", "response": "Returns a boolean array with one value per point indicating whether the point is inside of the image plane."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef compute_neighbour_distances(self):\n if len(self.coords) <= 1:\n return np.zeros((0,), dtype=np.float32)\n return np.sqrt(\n np.sum(\n (self.coords[:-1, :] - self.coords[1:, :]) ** 2,\n axis=1\n )\n )", "response": "Compute the euclidean distance between each two consecutive point and each other point."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the minimal distance between each point on self and other.", "response": "def compute_pointwise_distances(self, other, default=None):\n \"\"\"\n Compute the minimal distance between each point on self and other.\n\n Parameters\n ----------\n other : tuple of number \\\n or imgaug.augmentables.kps.Keypoint \\\n or imgaug.augmentables.LineString\n Other object to which to compute the distances.\n\n default\n Value to return if `other` contains no points.\n\n Returns\n -------\n list of float\n Distances to `other` or `default` if not distance could be computed.\n\n \"\"\"\n import shapely.geometry\n from .kps import Keypoint\n\n if isinstance(other, Keypoint):\n other = shapely.geometry.Point((other.x, other.y))\n elif isinstance(other, LineString):\n if len(other.coords) == 0:\n return default\n elif len(other.coords) == 1:\n other = shapely.geometry.Point(other.coords[0, :])\n else:\n other = shapely.geometry.LineString(other.coords)\n elif isinstance(other, tuple):\n assert len(other) == 2\n other = shapely.geometry.Point(other)\n else:\n raise ValueError(\n (\"Expected Keypoint or LineString or tuple (x,y), \"\n + \"got type %s.\") % (type(other),))\n\n return [shapely.geometry.Point(point).distance(other)\n for point in self.coords]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the minimal distance between this line string and other.", "response": "def compute_distance(self, other, default=None):\n \"\"\"\n Compute the minimal distance between the line string and `other`.\n\n Parameters\n ----------\n other : tuple of number \\\n or imgaug.augmentables.kps.Keypoint \\\n or imgaug.augmentables.LineString\n Other object to which to compute the distance.\n\n default\n Value to return if this line string or `other` contain no points.\n\n Returns\n -------\n float\n Distance to `other` or `default` if not distance could be computed.\n\n \"\"\"\n # FIXME this computes distance pointwise, does not have to be identical\n # with the actual min distance (e.g. edge center to other's point)\n distances = self.compute_pointwise_distances(other, default=[])\n if len(distances) == 0:\n return default\n return min(distances)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn True if the bounding box contains the point other.", "response": "def contains(self, other, max_distance=1e-4):\n \"\"\"\n Estimate whether the bounding box contains a point.\n\n Parameters\n ----------\n other : tuple of number or imgaug.augmentables.kps.Keypoint\n Point to check for.\n\n max_distance : float\n Maximum allowed euclidean distance between the point and the\n closest point on the line. If the threshold is exceeded, the point\n is not considered to be contained in the line.\n\n Returns\n -------\n bool\n True if the point is contained in the line string, False otherwise.\n It is contained if its distance to the line or any of its points\n is below a threshold.\n\n \"\"\"\n return self.compute_distance(other, default=np.inf) < max_distance"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef project(self, from_shape, to_shape):\n coords_proj = project_coords(self.coords, from_shape, to_shape)\n return self.copy(coords=coords_proj)", "response": "Project the line string onto a differently shaped image."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef is_fully_within_image(self, image, default=False):\n if len(self.coords) == 0:\n return default\n return np.all(self.get_pointwise_inside_image_mask(image))", "response": "Return True if the line string is fully inside the image area."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef is_partly_within_image(self, image, default=False):\n if len(self.coords) == 0:\n return default\n # check mask first to avoid costly computation of intersection points\n # whenever possible\n mask = self.get_pointwise_inside_image_mask(image)\n if np.any(mask):\n return True\n return len(self.clip_out_of_image(image)) > 0", "response": "Return True if the line string is at least partially inside the image area."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef is_out_of_image(self, image, fully=True, partly=False, default=True):\n if len(self.coords) == 0:\n return default\n\n if self.is_fully_within_image(image):\n return False\n elif self.is_partly_within_image(image):\n return partly\n else:\n return fully", "response": "Return True if the line is partially outside of the image area."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef clip_out_of_image(self, image):\n if len(self.coords) == 0:\n return []\n\n inside_image_mask = self.get_pointwise_inside_image_mask(image)\n ooi_mask = ~inside_image_mask\n\n if len(self.coords) == 1:\n if not np.any(inside_image_mask):\n return []\n return [self.copy()]\n\n if np.all(inside_image_mask):\n return [self.copy()]\n\n # top, right, bottom, left image edges\n # we subtract eps here, because intersection() works inclusively,\n # i.e. not subtracting eps would be equivalent to 0<=x<=C for C being\n # height or width\n # don't set the eps too low, otherwise points at height/width seem\n # to get rounded to height/width by shapely, which can cause problems\n # when first clipping and then calling is_fully_within_image()\n # returning false\n height, width = normalize_shape(image)[0:2]\n eps = 1e-3\n edges = [\n LineString([(0.0, 0.0), (width - eps, 0.0)]),\n LineString([(width - eps, 0.0), (width - eps, height - eps)]),\n LineString([(width - eps, height - eps), (0.0, height - eps)]),\n LineString([(0.0, height - eps), (0.0, 0.0)])\n ]\n intersections = self.find_intersections_with(edges)\n\n points = []\n gen = enumerate(zip(self.coords[:-1], self.coords[1:],\n ooi_mask[:-1], ooi_mask[1:],\n intersections))\n for i, (line_start, line_end, ooi_start, ooi_end, inter_line) in gen:\n points.append((line_start, False, ooi_start))\n for p_inter in inter_line:\n points.append((p_inter, True, False))\n\n is_last = (i == len(self.coords) - 2)\n if is_last and not ooi_end:\n points.append((line_end, False, ooi_end))\n\n lines = []\n line = []\n for i, (coord, was_added, ooi) in enumerate(points):\n # remove any point that is outside of the image,\n # also start a new line once such a point is detected\n if ooi:\n if len(line) > 0:\n lines.append(line)\n line = []\n continue\n\n if not was_added:\n # add all points that were part of the original line string\n # AND that are inside the image plane\n line.append(coord)\n else:\n is_last_point = (i == len(points)-1)\n # ooi is a numpy.bool_, hence the bool(.)\n is_next_ooi = (not is_last_point\n and bool(points[i+1][2]) is True)\n\n # Add all points that were new (i.e. intersections), so\n # long that they aren't essentially identical to other point.\n # This prevents adding overlapping intersections multiple times.\n # (E.g. when a line intersects with a corner of the image plane\n # and also with one of its edges.)\n p_prev = line[-1] if len(line) > 0 else None\n # ignore next point if end reached or next point is out of image\n p_next = None\n if not is_last_point and not is_next_ooi:\n p_next = points[i+1][0]\n dist_prev = None\n dist_next = None\n if p_prev is not None:\n dist_prev = np.linalg.norm(\n np.float32(coord) - np.float32(p_prev))\n if p_next is not None:\n dist_next = np.linalg.norm(\n np.float32(coord) - np.float32(p_next))\n\n dist_prev_ok = (dist_prev is None or dist_prev > 1e-2)\n dist_next_ok = (dist_next is None or dist_next > 1e-2)\n if dist_prev_ok and dist_next_ok:\n line.append(coord)\n\n if len(line) > 0:\n lines.append(line)\n\n lines = [line for line in lines if len(line) > 0]\n return [self.deepcopy(coords=line) for line in lines]", "response": "Clip off all parts of the line_string that are outside of the image."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_intersections_with(self, other):\n import shapely.geometry\n\n geom = _convert_var_to_shapely_geometry(other)\n\n result = []\n for p_start, p_end in zip(self.coords[:-1], self.coords[1:]):\n ls = shapely.geometry.LineString([p_start, p_end])\n intersections = ls.intersection(geom)\n intersections = list(_flatten_shapely_collection(intersections))\n\n intersections_points = []\n for inter in intersections:\n if isinstance(inter, shapely.geometry.linestring.LineString):\n inter_start = (inter.coords[0][0], inter.coords[0][1])\n inter_end = (inter.coords[-1][0], inter.coords[-1][1])\n intersections_points.extend([inter_start, inter_end])\n else:\n assert isinstance(inter, shapely.geometry.point.Point), (\n \"Expected to find shapely.geometry.point.Point or \"\n \"shapely.geometry.linestring.LineString intersection, \"\n \"actually found %s.\" % (type(inter),))\n intersections_points.append((inter.x, inter.y))\n\n # sort by distance to start point, this makes it later on easier\n # to remove duplicate points\n inter_sorted = sorted(\n intersections_points,\n key=lambda p: np.linalg.norm(np.float32(p) - p_start)\n )\n\n result.append(inter_sorted)\n return result", "response": "Find all intersection points between the line string and other."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef shift(self, top=None, right=None, bottom=None, left=None):\n top = top if top is not None else 0\n right = right if right is not None else 0\n bottom = bottom if bottom is not None else 0\n left = left if left is not None else 0\n coords = np.copy(self.coords)\n coords[:, 0] += left - right\n coords[:, 1] += top - bottom\n return self.copy(coords=coords)", "response": "Shift the line string from one or more image sides."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef draw_mask(self, image_shape, size_lines=1, size_points=0,\n raise_if_out_of_image=False):\n \"\"\"\n Draw this line segment as a binary image mask.\n\n Parameters\n ----------\n image_shape : tuple of int\n The shape of the image onto which to draw the line mask.\n\n size_lines : int, optional\n Thickness of the line segments.\n\n size_points : int, optional\n Size of the points in pixels.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n Boolean line mask of shape `image_shape` (no channel axis).\n\n \"\"\"\n heatmap = self.draw_heatmap_array(\n image_shape,\n alpha_lines=1.0, alpha_points=1.0,\n size_lines=size_lines, size_points=size_points,\n antialiased=False,\n raise_if_out_of_image=raise_if_out_of_image)\n return heatmap > 0.5", "response": "Draw this line segment as a binary image mask."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef draw_lines_heatmap_array(self, image_shape, alpha=1.0,\n size=1, antialiased=True,\n raise_if_out_of_image=False):\n \"\"\"\n Draw the line segments of the line string as a heatmap array.\n\n Parameters\n ----------\n image_shape : tuple of int\n The shape of the image onto which to draw the line mask.\n\n alpha : float, optional\n Opacity of the line string. Higher values denote a more visible\n line string.\n\n size : int, optional\n Thickness of the line segments.\n\n antialiased : bool, optional\n Whether to draw the line with anti-aliasing activated.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n Float array of shape `image_shape` (no channel axis) with drawn\n line string. All values are in the interval ``[0.0, 1.0]``.\n\n \"\"\"\n assert len(image_shape) == 2 or (\n len(image_shape) == 3 and image_shape[-1] == 1), (\n \"Expected (H,W) or (H,W,1) as image_shape, got %s.\" % (\n image_shape,))\n\n arr = self.draw_lines_on_image(\n np.zeros(image_shape, dtype=np.uint8),\n color=255, alpha=alpha, size=size,\n antialiased=antialiased,\n raise_if_out_of_image=raise_if_out_of_image\n )\n return arr.astype(np.float32) / 255.0", "response": "Draw the lines of the line string on the image."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef draw_points_heatmap_array(self, image_shape, alpha=1.0,\n size=1, raise_if_out_of_image=False):\n \"\"\"\n Draw the points of the line string as a heatmap array.\n\n Parameters\n ----------\n image_shape : tuple of int\n The shape of the image onto which to draw the point mask.\n\n alpha : float, optional\n Opacity of the line string points. Higher values denote a more\n visible points.\n\n size : int, optional\n Size of the points in pixels.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n Float array of shape `image_shape` (no channel axis) with drawn\n line string points. All values are in the interval ``[0.0, 1.0]``.\n\n \"\"\"\n assert len(image_shape) == 2 or (\n len(image_shape) == 3 and image_shape[-1] == 1), (\n \"Expected (H,W) or (H,W,1) as image_shape, got %s.\" % (\n image_shape,))\n\n arr = self.draw_points_on_image(\n np.zeros(image_shape, dtype=np.uint8),\n color=255, alpha=alpha, size=size,\n raise_if_out_of_image=raise_if_out_of_image\n )\n return arr.astype(np.float32) / 255.0", "response": "Draw the points of the line string as a heatmap array."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef draw_heatmap_array(self, image_shape, alpha_lines=1.0, alpha_points=1.0,\n size_lines=1, size_points=0, antialiased=True,\n raise_if_out_of_image=False):\n \"\"\"\n Draw the line segments and points of the line string as a heatmap array.\n\n Parameters\n ----------\n image_shape : tuple of int\n The shape of the image onto which to draw the line mask.\n\n alpha_lines : float, optional\n Opacity of the line string. Higher values denote a more visible\n line string.\n\n alpha_points : float, optional\n Opacity of the line string points. Higher values denote a more\n visible points.\n\n size_lines : int, optional\n Thickness of the line segments.\n\n size_points : int, optional\n Size of the points in pixels.\n\n antialiased : bool, optional\n Whether to draw the line with anti-aliasing activated.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n Float array of shape `image_shape` (no channel axis) with drawn\n line segments and points. All values are in the\n interval ``[0.0, 1.0]``.\n\n \"\"\"\n heatmap_lines = self.draw_lines_heatmap_array(\n image_shape,\n alpha=alpha_lines,\n size=size_lines,\n antialiased=antialiased,\n raise_if_out_of_image=raise_if_out_of_image)\n if size_points <= 0:\n return heatmap_lines\n\n heatmap_points = self.draw_points_heatmap_array(\n image_shape,\n alpha=alpha_points,\n size=size_points,\n raise_if_out_of_image=raise_if_out_of_image)\n\n heatmap = np.dstack([heatmap_lines, heatmap_points])\n return np.max(heatmap, axis=2)", "response": "Draw the line segments and points of the line string as a heatmap array."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndraws the lines of the line string on a given image.", "response": "def draw_lines_on_image(self, image, color=(0, 255, 0),\n alpha=1.0, size=3,\n antialiased=True,\n raise_if_out_of_image=False):\n \"\"\"\n Draw the line segments of the line string on a given image.\n\n Parameters\n ----------\n image : ndarray or tuple of int\n The image onto which to draw.\n Expected to be ``uint8`` and of shape ``(H, W, C)`` with ``C``\n usually being ``3`` (other values are not tested).\n If a tuple, expected to be ``(H, W, C)`` and will lead to a new\n ``uint8`` array of zeros being created.\n\n color : int or iterable of int\n Color to use as RGB, i.e. three values.\n\n alpha : float, optional\n Opacity of the line string. Higher values denote a more visible\n line string.\n\n size : int, optional\n Thickness of the line segments.\n\n antialiased : bool, optional\n Whether to draw the line with anti-aliasing activated.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n `image` with line drawn on it.\n\n \"\"\"\n from .. import dtypes as iadt\n from ..augmenters import blend as blendlib\n\n image_was_empty = False\n if isinstance(image, tuple):\n image_was_empty = True\n image = np.zeros(image, dtype=np.uint8)\n assert image.ndim in [2, 3], (\n (\"Expected image or shape of form (H,W) or (H,W,C), \"\n + \"got shape %s.\") % (image.shape,))\n\n if len(self.coords) <= 1 or alpha < 0 + 1e-4 or size < 1:\n return np.copy(image)\n\n if raise_if_out_of_image \\\n and self.is_out_of_image(image, partly=False, fully=True):\n raise Exception(\n \"Cannot draw line string '%s' on image with shape %s, because \"\n \"it would be out of bounds.\" % (\n self.__str__(), image.shape))\n\n if image.ndim == 2:\n assert ia.is_single_number(color), (\n \"Got a 2D image. Expected then 'color' to be a single number, \"\n \"but got %s.\" % (str(color),))\n color = [color]\n elif image.ndim == 3 and ia.is_single_number(color):\n color = [color] * image.shape[-1]\n\n image = image.astype(np.float32)\n height, width = image.shape[0:2]\n\n # We can't trivially exclude lines outside of the image here, because\n # even if start and end point are outside, there can still be parts of\n # the line inside the image.\n # TODO Do this with edge-wise intersection tests\n lines = []\n for line_start, line_end in zip(self.coords[:-1], self.coords[1:]):\n # note that line() expects order (y1, x1, y2, x2), hence ([1], [0])\n lines.append((line_start[1], line_start[0],\n line_end[1], line_end[0]))\n\n # skimage.draw.line can only handle integers\n lines = np.round(np.float32(lines)).astype(np.int32)\n\n # size == 0 is already covered above\n # Note here that we have to be careful not to draw lines two times\n # at their intersection points, e.g. for (p0, p1), (p1, 2) we could\n # end up drawing at p1 twice, leading to higher values if alpha is used.\n color = np.float32(color)\n heatmap = np.zeros(image.shape[0:2], dtype=np.float32)\n for line in lines:\n if antialiased:\n rr, cc, val = skimage.draw.line_aa(*line)\n else:\n rr, cc = skimage.draw.line(*line)\n val = 1.0\n\n # mask check here, because line() can generate coordinates\n # outside of the image plane\n rr_mask = np.logical_and(0 <= rr, rr < height)\n cc_mask = np.logical_and(0 <= cc, cc < width)\n mask = np.logical_and(rr_mask, cc_mask)\n\n if np.any(mask):\n rr = rr[mask]\n cc = cc[mask]\n val = val[mask] if not ia.is_single_number(val) else val\n heatmap[rr, cc] = val * alpha\n\n if size > 1:\n kernel = np.ones((size, size), dtype=np.uint8)\n heatmap = cv2.dilate(heatmap, kernel)\n\n if image_was_empty:\n image_blend = image + heatmap * color\n else:\n image_color_shape = image.shape[0:2]\n if image.ndim == 3:\n image_color_shape = image_color_shape + (1,)\n image_color = np.tile(color, image_color_shape)\n image_blend = blendlib.blend_alpha(image_color, image, heatmap)\n\n image_blend = iadt.restore_dtypes_(image_blend, np.uint8)\n return image_blend"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef draw_points_on_image(self, image, color=(0, 128, 0),\n alpha=1.0, size=3,\n copy=True, raise_if_out_of_image=False):\n \"\"\"\n Draw the points of the line string on a given image.\n\n Parameters\n ----------\n image : ndarray or tuple of int\n The image onto which to draw.\n Expected to be ``uint8`` and of shape ``(H, W, C)`` with ``C``\n usually being ``3`` (other values are not tested).\n If a tuple, expected to be ``(H, W, C)`` and will lead to a new\n ``uint8`` array of zeros being created.\n\n color : iterable of int\n Color to use as RGB, i.e. three values.\n\n alpha : float, optional\n Opacity of the line string points. Higher values denote a more\n visible points.\n\n size : int, optional\n Size of the points in pixels.\n\n copy : bool, optional\n Whether it is allowed to draw directly in the input\n array (``False``) or it has to be copied (``True``).\n The routine may still have to copy, even if ``copy=False`` was\n used. Always use the return value.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n Float array of shape `image_shape` (no channel axis) with drawn\n line string points. All values are in the interval ``[0.0, 1.0]``.\n\n \"\"\"\n from .kps import KeypointsOnImage\n kpsoi = KeypointsOnImage.from_xy_array(self.coords, shape=image.shape)\n image = kpsoi.draw_on_image(\n image, color=color, alpha=alpha,\n size=size, copy=copy,\n raise_if_out_of_image=raise_if_out_of_image)\n\n return image", "response": "Draw the points of the line string on a given image."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndraw the line string on an image.", "response": "def draw_on_image(self, image,\n color=(0, 255, 0), color_lines=None, color_points=None,\n alpha=1.0, alpha_lines=None, alpha_points=None,\n size=1, size_lines=None, size_points=None,\n antialiased=True,\n raise_if_out_of_image=False):\n \"\"\"\n Draw the line string on an image.\n\n Parameters\n ----------\n image : ndarray\n The `(H,W,C)` `uint8` image onto which to draw the line string.\n\n color : iterable of int, optional\n Color to use as RGB, i.e. three values.\n The color of the line and points are derived from this value,\n unless they are set.\n\n color_lines : None or iterable of int\n Color to use for the line segments as RGB, i.e. three values.\n If ``None``, this value is derived from `color`.\n\n color_points : None or iterable of int\n Color to use for the points as RGB, i.e. three values.\n If ``None``, this value is derived from ``0.5 * color``.\n\n alpha : float, optional\n Opacity of the line string. Higher values denote more visible\n points.\n The alphas of the line and points are derived from this value,\n unless they are set.\n\n alpha_lines : None or float, optional\n Opacity of the line string. Higher values denote more visible\n line string.\n If ``None``, this value is derived from `alpha`.\n\n alpha_points : None or float, optional\n Opacity of the line string points. Higher values denote more\n visible points.\n If ``None``, this value is derived from `alpha`.\n\n size : int, optional\n Size of the line string.\n The sizes of the line and points are derived from this value,\n unless they are set.\n\n size_lines : None or int, optional\n Thickness of the line segments.\n If ``None``, this value is derived from `size`.\n\n size_points : None or int, optional\n Size of the points in pixels.\n If ``None``, this value is derived from ``3 * size``.\n\n antialiased : bool, optional\n Whether to draw the line with anti-aliasing activated.\n This does currently not affect the point drawing.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n Image with line string drawn on it.\n\n \"\"\"\n assert color is not None\n assert alpha is not None\n assert size is not None\n\n color_lines = color_lines if color_lines is not None \\\n else np.float32(color)\n color_points = color_points if color_points is not None \\\n else np.float32(color) * 0.5\n\n alpha_lines = alpha_lines if alpha_lines is not None \\\n else np.float32(alpha)\n alpha_points = alpha_points if alpha_points is not None \\\n else np.float32(alpha)\n\n size_lines = size_lines if size_lines is not None else size\n size_points = size_points if size_points is not None else size * 3\n\n image = self.draw_lines_on_image(\n image, color=np.array(color_lines).astype(np.uint8),\n alpha=alpha_lines, size=size_lines,\n antialiased=antialiased,\n raise_if_out_of_image=raise_if_out_of_image)\n\n image = self.draw_points_on_image(\n image, color=np.array(color_points).astype(np.uint8),\n alpha=alpha_points, size=size_points,\n copy=False,\n raise_if_out_of_image=raise_if_out_of_image)\n\n return image"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nextracting the image pixels covered by the line string. It will only extract pixels overlapped by the line string. This function will by default zero-pad the image if the line string is partially/fully outside of the image. This is for consistency with the same implementations for bounding boxes and polygons. Parameters ---------- image : ndarray The image of shape `(H,W,[C])` from which to extract the pixels within the line string. size : int, optional Thickness of the line. pad : bool, optional Whether to zero-pad the image if the object is partially/fully outside of it. pad_max : None or int, optional The maximum number of pixels that may be zero-paded on any side, i.e. if this has value ``N`` the total maximum of added pixels is ``4*N``. This option exists to prevent extremely large images as a result of single points being moved very far away during augmentation. antialiased : bool, optional Whether to apply anti-aliasing to the line string. prevent_zero_size : bool, optional Whether to prevent height or width of the extracted image from becoming zero. If this is set to True and height or width of the line string is below 1, the height/width will be increased to 1. This can be useful to prevent problems, e.g. with image saving or plotting. If it is set to False, images will be returned as ``(H', W')`` or ``(H', W', 3)`` with ``H`` or ``W`` potentially being 0. Returns ------- image : (H',W') ndarray or (H',W',C) ndarray Pixels overlapping with the line string. Zero-padded if the line string is partially/fully outside of the image and ``pad=True``. If `prevent_zero_size` is activated, it is guarantueed that ``H'>0`` and ``W'>0``, otherwise only ``H'>=0`` and ``W'>=0``.", "response": "def extract_from_image(self, image, size=1, pad=True, pad_max=None,\n antialiased=True, prevent_zero_size=True):\n \"\"\"\n Extract the image pixels covered by the line string.\n\n It will only extract pixels overlapped by the line string.\n\n This function will by default zero-pad the image if the line string is\n partially/fully outside of the image. This is for consistency with\n the same implementations for bounding boxes and polygons.\n\n Parameters\n ----------\n image : ndarray\n The image of shape `(H,W,[C])` from which to extract the pixels\n within the line string.\n\n size : int, optional\n Thickness of the line.\n\n pad : bool, optional\n Whether to zero-pad the image if the object is partially/fully\n outside of it.\n\n pad_max : None or int, optional\n The maximum number of pixels that may be zero-paded on any side,\n i.e. if this has value ``N`` the total maximum of added pixels\n is ``4*N``.\n This option exists to prevent extremely large images as a result of\n single points being moved very far away during augmentation.\n\n antialiased : bool, optional\n Whether to apply anti-aliasing to the line string.\n\n prevent_zero_size : bool, optional\n Whether to prevent height or width of the extracted image from\n becoming zero. If this is set to True and height or width of the\n line string is below 1, the height/width will be increased to 1.\n This can be useful to prevent problems, e.g. with image saving or\n plotting. If it is set to False, images will be returned as\n ``(H', W')`` or ``(H', W', 3)`` with ``H`` or ``W`` potentially\n being 0.\n\n Returns\n -------\n image : (H',W') ndarray or (H',W',C) ndarray\n Pixels overlapping with the line string. Zero-padded if the\n line string is partially/fully outside of the image and\n ``pad=True``. If `prevent_zero_size` is activated, it is\n guarantueed that ``H'>0`` and ``W'>0``, otherwise only\n ``H'>=0`` and ``W'>=0``.\n\n \"\"\"\n from .bbs import BoundingBox\n\n assert image.ndim in [2, 3], (\n \"Expected image of shape (H,W,[C]), \"\n \"got shape %s.\" % (image.shape,))\n\n if len(self.coords) == 0 or size <= 0:\n if prevent_zero_size:\n return np.zeros((1, 1) + image.shape[2:], dtype=image.dtype)\n return np.zeros((0, 0) + image.shape[2:], dtype=image.dtype)\n\n xx = self.xx_int\n yy = self.yy_int\n\n # this would probably work if drawing was subpixel-accurate\n # x1 = np.min(self.coords[:, 0]) - (size / 2)\n # y1 = np.min(self.coords[:, 1]) - (size / 2)\n # x2 = np.max(self.coords[:, 0]) + (size / 2)\n # y2 = np.max(self.coords[:, 1]) + (size / 2)\n\n # this works currently with non-subpixel-accurate drawing\n sizeh = (size - 1) / 2\n x1 = np.min(xx) - sizeh\n y1 = np.min(yy) - sizeh\n x2 = np.max(xx) + 1 + sizeh\n y2 = np.max(yy) + 1 + sizeh\n bb = BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2)\n\n if len(self.coords) == 1:\n return bb.extract_from_image(image, pad=pad, pad_max=pad_max,\n prevent_zero_size=prevent_zero_size)\n\n heatmap = self.draw_lines_heatmap_array(\n image.shape[0:2], alpha=1.0, size=size, antialiased=antialiased)\n if image.ndim == 3:\n heatmap = np.atleast_3d(heatmap)\n image_masked = image.astype(np.float32) * heatmap\n extract = bb.extract_from_image(image_masked, pad=pad, pad_max=pad_max,\n prevent_zero_size=prevent_zero_size)\n return np.clip(np.round(extract), 0, 255).astype(np.uint8)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef concatenate(self, other):\n if not isinstance(other, LineString):\n other = LineString(other)\n return self.deepcopy(\n coords=np.concatenate([self.coords, other.coords], axis=0))", "response": "Concatenate this line string with another line string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsubdividing this LineString into two separate lines.", "response": "def subdivide(self, points_per_edge):\n \"\"\"\n Adds ``N`` interpolated points with uniform spacing to each edge.\n\n For each edge between points ``A`` and ``B`` this adds points\n at ``A + (i/(1+N)) * (B - A)``, where ``i`` is the index of the added\n point and ``N`` is the number of points to add per edge.\n\n Calling this method two times will split each edge at its center\n and then again split each newly created edge at their center.\n It is equivalent to calling `subdivide(3)`.\n\n Parameters\n ----------\n points_per_edge : int\n Number of points to interpolate on each edge.\n\n Returns\n -------\n LineString\n Line string with subdivided edges.\n\n \"\"\"\n if len(self.coords) <= 1 or points_per_edge < 1:\n return self.deepcopy()\n coords = interpolate_points(self.coords, nb_steps=points_per_edge,\n closed=False)\n return self.deepcopy(coords=coords)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts the line string points to keypoints.", "response": "def to_keypoints(self):\n \"\"\"\n Convert the line string points to keypoints.\n\n Returns\n -------\n list of imgaug.augmentables.kps.Keypoint\n Points of the line string as keypoints.\n\n \"\"\"\n # TODO get rid of this deferred import\n from imgaug.augmentables.kps import Keypoint\n return [Keypoint(x=x, y=y) for (x, y) in self.coords]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate a bounding box encapsulating the line string.", "response": "def to_bounding_box(self):\n \"\"\"\n Generate a bounding box encapsulating the line string.\n\n Returns\n -------\n None or imgaug.augmentables.bbs.BoundingBox\n Bounding box encapsulating the line string.\n ``None`` if the line string contained no points.\n\n \"\"\"\n from .bbs import BoundingBox\n # we don't have to mind the case of len(.) == 1 here, because\n # zero-sized BBs are considered valid\n if len(self.coords) == 0:\n return None\n return BoundingBox(x1=np.min(self.xx), y1=np.min(self.yy),\n x2=np.max(self.xx), y2=np.max(self.yy),\n label=self.label)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef to_polygon(self):\n from .polys import Polygon\n return Polygon(self.coords, label=self.label)", "response": "Generate a polygon from the line string points."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngenerate a heatmap object from the line string.", "response": "def to_heatmap(self, image_shape, size_lines=1, size_points=0,\n antialiased=True, raise_if_out_of_image=False):\n \"\"\"\n Generate a heatmap object from the line string.\n\n This is similar to\n :func:`imgaug.augmentables.lines.LineString.draw_lines_heatmap_array`\n executed with ``alpha=1.0``. The result is wrapped in a\n ``HeatmapsOnImage`` object instead of just an array.\n No points are drawn.\n\n Parameters\n ----------\n image_shape : tuple of int\n The shape of the image onto which to draw the line mask.\n\n size_lines : int, optional\n Thickness of the line.\n\n size_points : int, optional\n Size of the points in pixels.\n\n antialiased : bool, optional\n Whether to draw the line with anti-aliasing activated.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n imgaug.augmentables.heatmaps.HeatmapOnImage\n Heatmap object containing drawn line string.\n\n \"\"\"\n from .heatmaps import HeatmapsOnImage\n return HeatmapsOnImage(\n self.draw_heatmap_array(\n image_shape, size_lines=size_lines, size_points=size_points,\n antialiased=antialiased,\n raise_if_out_of_image=raise_if_out_of_image),\n shape=image_shape\n )"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating a segmentation map object from the line string.", "response": "def to_segmentation_map(self, image_shape, size_lines=1, size_points=0,\n raise_if_out_of_image=False):\n \"\"\"\n Generate a segmentation map object from the line string.\n\n This is similar to\n :func:`imgaug.augmentables.lines.LineString.draw_mask`.\n The result is wrapped in a ``SegmentationMapOnImage`` object\n instead of just an array.\n\n Parameters\n ----------\n image_shape : tuple of int\n The shape of the image onto which to draw the line mask.\n\n size_lines : int, optional\n Thickness of the line.\n\n size_points : int, optional\n Size of the points in pixels.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if the line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n imgaug.augmentables.segmaps.SegmentationMapOnImage\n Segmentation map object containing drawn line string.\n\n \"\"\"\n from .segmaps import SegmentationMapOnImage\n return SegmentationMapOnImage(\n self.draw_mask(\n image_shape, size_lines=size_lines, size_points=size_points,\n raise_if_out_of_image=raise_if_out_of_image),\n shape=image_shape\n )"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef coords_almost_equals(self, other, max_distance=1e-6, points_per_edge=8):\n if isinstance(other, LineString):\n pass\n elif isinstance(other, tuple):\n other = LineString([other])\n else:\n other = LineString(other)\n\n if len(self.coords) == 0 and len(other.coords) == 0:\n return True\n elif 0 in [len(self.coords), len(other.coords)]:\n # only one of the two line strings has no coords\n return False\n\n self_subd = self.subdivide(points_per_edge)\n other_subd = other.subdivide(points_per_edge)\n\n dist_self2other = self_subd.compute_pointwise_distances(other_subd)\n dist_other2self = other_subd.compute_pointwise_distances(self_subd)\n dist = max(np.max(dist_self2other), np.max(dist_other2self))\n return dist < max_distance", "response": "Compare two line strings and return True if the two line strings are almost identical."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncompares this line string and another line string.", "response": "def almost_equals(self, other, max_distance=1e-4, points_per_edge=8):\n \"\"\"\n Compare this and another LineString.\n\n Parameters\n ----------\n other: imgaug.augmentables.lines.LineString\n The other line string. Must be a LineString instance, not just\n its coordinates.\n\n max_distance : float, optional\n See :func:`imgaug.augmentables.lines.LineString.coords_almost_equals`.\n\n points_per_edge : int, optional\n See :func:`imgaug.augmentables.lines.LineString.coords_almost_equals`.\n\n Returns\n -------\n bool\n ``True`` if the coordinates are almost equal according to\n :func:`imgaug.augmentables.lines.LineString.coords_almost_equals`\n and additionally the labels are identical. Otherwise ``False``.\n\n \"\"\"\n if self.label != other.label:\n return False\n return self.coords_almost_equals(\n other, max_distance=max_distance, points_per_edge=points_per_edge)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a shallow copy of the LineString object.", "response": "def copy(self, coords=None, label=None):\n \"\"\"\n Create a shallow copy of the LineString object.\n\n Parameters\n ----------\n coords : None or iterable of tuple of number or ndarray\n If not ``None``, then the coords of the copied object will be set\n to this value.\n\n label : None or str\n If not ``None``, then the label of the copied object will be set to\n this value.\n\n Returns\n -------\n imgaug.augmentables.lines.LineString\n Shallow copy.\n\n \"\"\"\n return LineString(coords=self.coords if coords is None else coords,\n label=self.label if label is None else label)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef deepcopy(self, coords=None, label=None):\n return LineString(\n coords=np.copy(self.coords) if coords is None else coords,\n label=copylib.deepcopy(self.label) if label is None else label)", "response": "Create a deep copy of the BoundingBox object."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprojecting bounding boxes from one image to a new one.", "response": "def on(self, image):\n \"\"\"\n Project bounding boxes from one image to a new one.\n\n Parameters\n ----------\n image : ndarray or tuple of int\n The new image onto which to project.\n Either an image with shape ``(H,W,[C])`` or a tuple denoting\n such an image shape.\n\n Returns\n -------\n line_strings : imgaug.augmentables.lines.LineStrings\n Object containing all projected line strings.\n\n \"\"\"\n shape = normalize_shape(image)\n if shape[0:2] == self.shape[0:2]:\n return self.deepcopy()\n line_strings = [ls.project(self.shape, shape)\n for ls in self.line_strings]\n return self.deepcopy(line_strings=line_strings, shape=shape)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert an array of point coordinates or iterable of arrays to a LineStringsOnImage object.", "response": "def from_xy_arrays(cls, xy, shape):\n \"\"\"\n Convert an `(N,M,2)` ndarray to a LineStringsOnImage object.\n\n This is the inverse of\n :func:`imgaug.augmentables.lines.LineStringsOnImage.to_xy_array`.\n\n Parameters\n ----------\n xy : (N,M,2) ndarray or iterable of (M,2) ndarray\n Array containing the point coordinates ``N`` line strings\n with each ``M`` points given as ``(x,y)`` coordinates.\n ``M`` may differ if an iterable of arrays is used.\n Each array should usually be of dtype ``float32``.\n\n shape : tuple of int\n ``(H,W,[C])`` shape of the image on which the line strings are\n placed.\n\n Returns\n -------\n imgaug.augmentables.lines.LineStringsOnImage\n Object containing a list of ``LineString`` objects following the\n provided point coordinates.\n\n \"\"\"\n lss = []\n for xy_ls in xy:\n lss.append(LineString(xy_ls))\n return cls(lss, shape)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef to_xy_arrays(self, dtype=np.float32):\n from .. import dtypes as iadt\n return [iadt.restore_dtypes_(np.copy(ls.coords), dtype)\n for ls in self.line_strings]", "response": "Convert this object to an iterable of arrays of point coordinates each given as M 2."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef draw_on_image(self, image,\n color=(0, 255, 0), color_lines=None, color_points=None,\n alpha=1.0, alpha_lines=None, alpha_points=None,\n size=1, size_lines=None, size_points=None,\n antialiased=True,\n raise_if_out_of_image=False):\n \"\"\"\n Draw all line strings onto a given image.\n\n Parameters\n ----------\n image : ndarray\n The `(H,W,C)` `uint8` image onto which to draw the line strings.\n\n color : iterable of int, optional\n Color to use as RGB, i.e. three values.\n The color of the lines and points are derived from this value,\n unless they are set.\n\n color_lines : None or iterable of int\n Color to use for the line segments as RGB, i.e. three values.\n If ``None``, this value is derived from `color`.\n\n color_points : None or iterable of int\n Color to use for the points as RGB, i.e. three values.\n If ``None``, this value is derived from ``0.5 * color``.\n\n alpha : float, optional\n Opacity of the line strings. Higher values denote more visible\n points.\n The alphas of the line and points are derived from this value,\n unless they are set.\n\n alpha_lines : None or float, optional\n Opacity of the line strings. Higher values denote more visible\n line string.\n If ``None``, this value is derived from `alpha`.\n\n alpha_points : None or float, optional\n Opacity of the line string points. Higher values denote more\n visible points.\n If ``None``, this value is derived from `alpha`.\n\n size : int, optional\n Size of the line strings.\n The sizes of the line and points are derived from this value,\n unless they are set.\n\n size_lines : None or int, optional\n Thickness of the line segments.\n If ``None``, this value is derived from `size`.\n\n size_points : None or int, optional\n Size of the points in pixels.\n If ``None``, this value is derived from ``3 * size``.\n\n antialiased : bool, optional\n Whether to draw the lines with anti-aliasing activated.\n This does currently not affect the point drawing.\n\n raise_if_out_of_image : bool, optional\n Whether to raise an error if a line string is fully\n outside of the image. If set to False, no error will be raised and\n only the parts inside the image will be drawn.\n\n Returns\n -------\n ndarray\n Image with line strings drawn on it.\n\n \"\"\"\n # TODO improve efficiency here by copying only once\n for ls in self.line_strings:\n image = ls.draw_on_image(\n image,\n color=color, color_lines=color_lines, color_points=color_points,\n alpha=alpha, alpha_lines=alpha_lines, alpha_points=alpha_points,\n size=size, size_lines=size_lines, size_points=size_points,\n antialiased=antialiased,\n raise_if_out_of_image=raise_if_out_of_image\n )\n\n return image", "response": "Draw all line strings onto a given image."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a reduced set of line strings that are not part of the image.", "response": "def remove_out_of_image(self, fully=True, partly=False):\n \"\"\"\n Remove all line strings that are fully/partially outside of the image.\n\n Parameters\n ----------\n fully : bool, optional\n Whether to remove line strings that are fully outside of the image.\n\n partly : bool, optional\n Whether to remove line strings that are partially outside of the\n image.\n\n Returns\n -------\n imgaug.augmentables.lines.LineStringsOnImage\n Reduced set of line strings, with those that were fully/partially\n outside of the image removed.\n\n \"\"\"\n lss_clean = [ls for ls in self.line_strings\n if not ls.is_out_of_image(\n self.shape, fully=fully, partly=partly)]\n return LineStringsOnImage(lss_clean, shape=self.shape)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef clip_out_of_image(self):\n lss_cut = [ls_clipped\n for ls in self.line_strings\n for ls_clipped in ls.clip_out_of_image(self.shape)]\n return LineStringsOnImage(lss_cut, shape=self.shape)", "response": "Clip off all parts of the line strings that are outside of the image."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef shift(self, top=None, right=None, bottom=None, left=None):\n lss_new = [ls.shift(top=top, right=right, bottom=bottom, left=left)\n for ls in self.line_strings]\n return LineStringsOnImage(lss_new, shape=self.shape)", "response": "Shifts the line strings from one or more image sides."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a shallow copy of the LineStringsOnImage object.", "response": "def copy(self, line_strings=None, shape=None):\n \"\"\"\n Create a shallow copy of the LineStringsOnImage object.\n\n Parameters\n ----------\n line_strings : None \\\n or list of imgaug.augmentables.lines.LineString, optional\n List of line strings on the image.\n If not ``None``, then the ``line_strings`` attribute of the copied\n object will be set to this value.\n\n shape : None or tuple of int or ndarray, optional\n The shape of the image on which the objects are placed.\n Either an image with shape ``(H,W,[C])`` or a tuple denoting\n such an image shape.\n If not ``None``, then the ``shape`` attribute of the copied object\n will be set to this value.\n\n Returns\n -------\n imgaug.augmentables.lines.LineStringsOnImage\n Shallow copy.\n\n \"\"\"\n lss = self.line_strings if line_strings is None else line_strings\n shape = self.shape if shape is None else shape\n return LineStringsOnImage(line_strings=lss, shape=shape)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a deep copy of the LineStringsOnImage object.", "response": "def deepcopy(self, line_strings=None, shape=None):\n \"\"\"\n Create a deep copy of the LineStringsOnImage object.\n\n Parameters\n ----------\n line_strings : None \\\n or list of imgaug.augmentables.lines.LineString, optional\n List of line strings on the image.\n If not ``None``, then the ``line_strings`` attribute of the copied\n object will be set to this value.\n\n shape : None or tuple of int or ndarray, optional\n The shape of the image on which the objects are placed.\n Either an image with shape ``(H,W,[C])`` or a tuple denoting\n such an image shape.\n If not ``None``, then the ``shape`` attribute of the copied object\n will be set to this value.\n\n Returns\n -------\n imgaug.augmentables.lines.LineStringsOnImage\n Deep copy.\n\n \"\"\"\n lss = self.line_strings if line_strings is None else line_strings\n shape = self.shape if shape is None else shape\n return LineStringsOnImage(\n line_strings=[ls.deepcopy() for ls in lss],\n shape=tuple(shape))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef blend_alpha(image_fg, image_bg, alpha, eps=1e-2):\n assert image_fg.shape == image_bg.shape\n assert image_fg.dtype.kind == image_bg.dtype.kind\n # TODO switch to gate_dtypes()\n assert image_fg.dtype.name not in [\"float128\"]\n assert image_bg.dtype.name not in [\"float128\"]\n\n # TODO add test for this\n input_was_2d = (len(image_fg.shape) == 2)\n if input_was_2d:\n image_fg = np.atleast_3d(image_fg)\n image_bg = np.atleast_3d(image_bg)\n\n input_was_bool = False\n if image_fg.dtype.kind == \"b\":\n input_was_bool = True\n # use float32 instead of float16 here because it seems to be faster\n image_fg = image_fg.astype(np.float32)\n image_bg = image_bg.astype(np.float32)\n\n alpha = np.array(alpha, dtype=np.float64)\n if alpha.size == 1:\n pass\n else:\n if alpha.ndim == 2:\n assert alpha.shape == image_fg.shape[0:2]\n alpha = alpha.reshape((alpha.shape[0], alpha.shape[1], 1))\n elif alpha.ndim == 3:\n assert alpha.shape == image_fg.shape or alpha.shape == image_fg.shape[0:2] + (1,)\n else:\n alpha = alpha.reshape((1, 1, -1))\n if alpha.shape[2] != image_fg.shape[2]:\n alpha = np.tile(alpha, (1, 1, image_fg.shape[2]))\n\n if not input_was_bool:\n if np.all(alpha >= 1.0 - eps):\n return np.copy(image_fg)\n elif np.all(alpha <= eps):\n return np.copy(image_bg)\n\n # for efficiency reaons, only test one value of alpha here, even if alpha is much larger\n assert 0 <= alpha.item(0) <= 1.0\n\n dt_images = iadt.get_minimal_dtype([image_fg, image_bg])\n\n # doing this only for non-float images led to inaccuracies for large floats values\n isize = dt_images.itemsize * 2\n isize = max(isize, 4) # at least 4 bytes (=float32), tends to be faster than float16\n dt_blend = np.dtype(\"f%d\" % (isize,))\n\n if alpha.dtype != dt_blend:\n alpha = alpha.astype(dt_blend)\n if image_fg.dtype != dt_blend:\n image_fg = image_fg.astype(dt_blend)\n if image_bg.dtype != dt_blend:\n image_bg = image_bg.astype(dt_blend)\n\n # the following is equivalent to\n # image_blend = alpha * image_fg + (1 - alpha) * image_bg\n # but supposedly faster\n image_blend = image_bg + alpha * (image_fg - image_bg)\n\n if input_was_bool:\n image_blend = image_blend > 0.5\n else:\n # skip clip, because alpha is expected to be in range [0.0, 1.0] and both images must have same dtype\n # dont skip round, because otherwise it is very unlikely to hit the image's max possible value\n image_blend = iadt.restore_dtypes_(image_blend, dt_images, clip=False, round=True)\n\n if input_was_2d:\n return image_blend[:, :, 0]\n return image_blend", "response": "Blends two images using an alpha blending."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef OneOf(children, name=None, deterministic=False, random_state=None):\n return SomeOf(n=1, children=children, random_order=False, name=name, deterministic=deterministic,\n random_state=random_state)", "response": "Augmenter that always executes exactly one of its children."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef AssertShape(shape, check_images=True, check_heatmaps=True,\n check_keypoints=True, check_polygons=True,\n name=None, deterministic=False, random_state=None):\n \"\"\"\n Augmenter to make assumptions about the shape of input image(s), heatmaps and keypoints.\n\n dtype support::\n\n * ``uint8``: yes; fully tested\n * ``uint16``: yes; tested\n * ``uint32``: yes; tested\n * ``uint64``: yes; tested\n * ``int8``: yes; tested\n * ``int16``: yes; tested\n * ``int32``: yes; tested\n * ``int64``: yes; tested\n * ``float16``: yes; tested\n * ``float32``: yes; tested\n * ``float64``: yes; tested\n * ``float128``: yes; tested\n * ``bool``: yes; tested\n\n Parameters\n ----------\n shape : tuple\n The expected shape, given as a tuple. The number of entries in the tuple must match the\n number of dimensions, i.e. it must contain four entries for ``(N, H, W, C)``. If only a\n single image is augmented via ``augment_image()``, then ``N`` is viewed as 1 by this\n augmenter. If the input image(s) don't have a channel axis, then ``C`` is viewed as 1\n by this augmenter.\n Each of the four entries may be None or a tuple of two ints or a list of ints.\n\n * If an entry is None, any value for that dimensions is accepted.\n * If an entry is int, exactly that integer value will be accepted\n or no other value.\n * If an entry is a tuple of two ints with values ``a`` and ``b``, only a\n value ``x`` with ``a <= x < b`` will be accepted for the dimension.\n * If an entry is a list of ints, only a value for the dimension\n will be accepted which is contained in the list.\n\n check_images : bool, optional\n Whether to validate input images via the given shape.\n\n check_heatmaps : bool, optional\n Whether to validate input heatmaps via the given shape.\n The number of heatmaps will be checked and for each Heatmaps\n instance its array's height and width, but not the channel\n count as the channel number denotes the expected number of channels\n in images.\n\n check_keypoints : bool, optional\n Whether to validate input keypoints via the given shape.\n This will check (a) the number of keypoints and (b) for each\n KeypointsOnImage instance the ``.shape``, i.e. the shape of the\n corresponding image.\n\n check_polygons : bool, optional\n Whether to validate input keypoints via the given shape.\n This will check (a) the number of polygons and (b) for each\n PolygonsOnImage instance the ``.shape``, i.e. the shape of the\n corresponding image.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Examples\n --------\n >>> seq = iaa.Sequential([\n >>> iaa.AssertShape((None, 32, 32, 3)),\n >>> iaa.Fliplr(0.5)\n >>> ])\n\n will first check for each image batch, if it contains a variable number of\n ``32x32`` images with 3 channels each. Only if that check succeeds, the\n horizontal flip will be executed (otherwise an assertion error will be\n thrown).\n\n >>> seq = iaa.Sequential([\n >>> iaa.AssertShape((None, (32, 64), 32, [1, 3])),\n >>> iaa.Fliplr(0.5)\n >>> ])\n\n like above, but now the height may be in the range ``32 <= H < 64`` and\n the number of channels may be either 1 or 3.\n\n \"\"\"\n ia.do_assert(len(shape) == 4, \"Expected shape to have length 4, got %d with shape: %s.\" % (len(shape), str(shape)))\n\n def compare(observed, expected, dimension, image_index):\n if expected is not None:\n if ia.is_single_integer(expected):\n ia.do_assert(observed == expected,\n \"Expected dim %d (entry index: %s) to have value %d, got %d.\" % (\n dimension, image_index, expected, observed))\n elif isinstance(expected, tuple):\n ia.do_assert(len(expected) == 2)\n ia.do_assert(expected[0] <= observed < expected[1],\n \"Expected dim %d (entry index: %s) to have value in range [%d, %d), got %d.\" % (\n dimension, image_index, expected[0], expected[1], observed))\n elif isinstance(expected, list):\n ia.do_assert(any([observed == val for val in expected]),\n \"Expected dim %d (entry index: %s) to have any value of %s, got %d.\" % (\n dimension, image_index, str(expected), observed))\n else:\n raise Exception((\"Invalid datatype for shape entry %d, expected each entry to be an integer, \"\n + \"a tuple (with two entries) or a list, got %s.\") % (dimension, type(expected),))\n\n def func_images(images, _random_state, _parents, _hooks):\n if check_images:\n if isinstance(images, list):\n if shape[0] is not None:\n compare(len(images), shape[0], 0, \"ALL\")\n\n for i in sm.xrange(len(images)):\n image = images[i]\n ia.do_assert(len(image.shape) == 3,\n \"Expected image number %d to have a shape of length 3, got %d (shape: %s).\" % (\n i, len(image.shape), str(image.shape)))\n for j in sm.xrange(len(shape)-1):\n expected = shape[j+1]\n observed = image.shape[j]\n compare(observed, expected, j, i)\n else:\n ia.do_assert(len(images.shape) == 4,\n \"Expected image's shape to have length 4, got %d (shape: %s).\" % (\n len(images.shape), str(images.shape)))\n for i in range(4):\n expected = shape[i]\n observed = images.shape[i]\n compare(observed, expected, i, \"ALL\")\n return images\n\n def func_heatmaps(heatmaps, _random_state, _parents, _hooks):\n if check_heatmaps:\n if shape[0] is not None:\n compare(len(heatmaps), shape[0], 0, \"ALL\")\n\n for i in sm.xrange(len(heatmaps)):\n heatmaps_i = heatmaps[i]\n for j in sm.xrange(len(shape[0:2])):\n expected = shape[j+1]\n observed = heatmaps_i.arr_0to1.shape[j]\n compare(observed, expected, j, i)\n return heatmaps\n\n def func_keypoints(keypoints_on_images, _random_state, _parents, _hooks):\n if check_keypoints:\n if shape[0] is not None:\n compare(len(keypoints_on_images), shape[0], 0, \"ALL\")\n\n for i in sm.xrange(len(keypoints_on_images)):\n keypoints_on_image = keypoints_on_images[i]\n for j in sm.xrange(len(shape[0:2])):\n expected = shape[j+1]\n observed = keypoints_on_image.shape[j]\n compare(observed, expected, j, i)\n return keypoints_on_images\n\n def func_polygons(polygons_on_images, _random_state, _parents, _hooks):\n if check_polygons:\n if shape[0] is not None:\n compare(len(polygons_on_images), shape[0], 0, \"ALL\")\n\n for i in sm.xrange(len(polygons_on_images)):\n polygons_on_image = polygons_on_images[i]\n for j in sm.xrange(len(shape[0:2])):\n expected = shape[j+1]\n observed = polygons_on_image.shape[j]\n compare(observed, expected, j, i)\n return polygons_on_images\n\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return Lambda(func_images, func_heatmaps, func_keypoints, func_polygons,\n name=name, deterministic=deterministic,\n random_state=random_state)", "response": "Augmenter to make assumptions about the shape of input image and keypoints."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nshuffles the channels in an image.", "response": "def shuffle_channels(image, random_state, channels=None):\n \"\"\"\n Randomize the order of (color) channels in an image.\n\n dtype support::\n\n * ``uint8``: yes; fully tested\n * ``uint16``: yes; indirectly tested (1)\n * ``uint32``: yes; indirectly tested (1)\n * ``uint64``: yes; indirectly tested (1)\n * ``int8``: yes; indirectly tested (1)\n * ``int16``: yes; indirectly tested (1)\n * ``int32``: yes; indirectly tested (1)\n * ``int64``: yes; indirectly tested (1)\n * ``float16``: yes; indirectly tested (1)\n * ``float32``: yes; indirectly tested (1)\n * ``float64``: yes; indirectly tested (1)\n * ``float128``: yes; indirectly tested (1)\n * ``bool``: yes; indirectly tested (1)\n\n - (1) Indirectly tested via ``ChannelShuffle``.\n\n Parameters\n ----------\n image : (H,W,[C]) ndarray\n Image of any dtype for which to shuffle the channels.\n\n random_state : numpy.random.RandomState\n The random state to use for this shuffling operation.\n\n channels : None or imgaug.ALL or list of int, optional\n Which channels are allowed to be shuffled with each other.\n If this is ``None`` or ``imgaug.ALL``, then all channels may be shuffled. If it is a list of integers,\n then only the channels with indices in that list may be shuffled. (Values start at 0. All channel indices in\n the list must exist in each image.)\n\n Returns\n -------\n ndarray\n The input image with shuffled channels.\n\n \"\"\"\n if image.ndim < 3 or image.shape[2] == 1:\n return image\n nb_channels = image.shape[2]\n all_channels = np.arange(nb_channels)\n is_all_channels = (\n channels is None\n or channels == ia.ALL\n or len(set(all_channels).difference(set(channels))) == 0\n )\n if is_all_channels:\n # note that if this is the case, then 'channels' may be None or imgaug.ALL, so don't simply move the\n # assignment outside of the if/else\n channels_perm = random_state.permutation(all_channels)\n return image[..., channels_perm]\n else:\n channels_perm = random_state.permutation(channels)\n channels_perm_full = all_channels\n for channel_source, channel_target in zip(channels, channels_perm):\n channels_perm_full[channel_source] = channel_target\n return image[..., channels_perm_full]"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nblurs an image using gaussian blurring.", "response": "def blur_gaussian_(image, sigma, ksize=None, backend=\"auto\", eps=1e-3):\n \"\"\"\n Blur an image using gaussian blurring.\n\n This operation might change the input image in-place.\n\n dtype support::\n\n if (backend=\"auto\")::\n\n * ``uint8``: yes; fully tested (1)\n * ``uint16``: yes; tested (1)\n * ``uint32``: yes; tested (2)\n * ``uint64``: yes; tested (2)\n * ``int8``: yes; tested (1)\n * ``int16``: yes; tested (1)\n * ``int32``: yes; tested (1)\n * ``int64``: yes; tested (2)\n * ``float16``: yes; tested (1)\n * ``float32``: yes; tested (1)\n * ``float64``: yes; tested (1)\n * ``float128``: no\n * ``bool``: yes; tested (1)\n\n - (1) Handled by ``cv2``. See ``backend=\"cv2\"``.\n - (2) Handled by ``scipy``. See ``backend=\"scipy\"``.\n\n if (backend=\"cv2\")::\n\n * ``uint8``: yes; fully tested\n * ``uint16``: yes; tested\n * ``uint32``: no (2)\n * ``uint64``: no (3)\n * ``int8``: yes; tested (4)\n * ``int16``: yes; tested\n * ``int32``: yes; tested (5)\n * ``int64``: no (6)\n * ``float16``: yes; tested (7)\n * ``float32``: yes; tested\n * ``float64``: yes; tested\n * ``float128``: no (8)\n * ``bool``: yes; tested (1)\n\n - (1) Mapped internally to ``float32``. Otherwise causes ``TypeError: src data type = 0 is not supported``.\n - (2) Causes ``TypeError: src data type = 6 is not supported``.\n - (3) Causes ``cv2.error: OpenCV(3.4.5) (...)/filter.cpp:2957: error: (-213:The function/feature is not\n implemented) Unsupported combination of source format (=4), and buffer format (=5) in function\n 'getLinearRowFilter'``.\n - (4) Mapped internally to ``int16``. Otherwise causes ``cv2.error: OpenCV(3.4.5) (...)/filter.cpp:2957:\n error: (-213:The function/feature is not implemented) Unsupported combination of source format (=1),\n and buffer format (=5) in function 'getLinearRowFilter'``.\n - (5) Mapped internally to ``float64``. Otherwise causes ``cv2.error: OpenCV(3.4.5) (...)/filter.cpp:2957:\n error: (-213:The function/feature is not implemented) Unsupported combination of source format (=4),\n and buffer format (=5) in function 'getLinearRowFilter'``.\n - (6) Causes ``cv2.error: OpenCV(3.4.5) (...)/filter.cpp:2957: error: (-213:The function/feature is not\n implemented) Unsupported combination of source format (=4), and buffer format (=5) in function\n 'getLinearRowFilter'``.\n - (7) Mapped internally to ``float32``. Otherwise causes ``TypeError: src data type = 23 is not supported``.\n - (8) Causes ``TypeError: src data type = 13 is not supported``.\n\n\n if (backend=\"scipy\")::\n\n * ``uint8``: yes; fully tested\n * ``uint16``: yes; tested\n * ``uint32``: yes; tested\n * ``uint64``: yes; tested\n * ``int8``: yes; tested\n * ``int16``: yes; tested\n * ``int32``: yes; tested\n * ``int64``: yes; tested\n * ``float16``: yes; tested (1)\n * ``float32``: yes; tested\n * ``float64``: yes; tested\n * ``float128``: no (2)\n * ``bool``: yes; tested (3)\n\n - (1) Mapped internally to ``float32``. Otherwise causes ``RuntimeError: array type dtype('float16')\n not supported``.\n - (2) Causes ``RuntimeError: array type dtype('float128') not supported``.\n - (3) Mapped internally to ``float32``. Otherwise too inaccurate.\n\n Parameters\n ----------\n image : numpy.ndarray\n The image to blur. Expected to be of shape ``(H, W)`` or ``(H, W, C)``.\n\n sigma : number\n Standard deviation of the gaussian blur. Larger numbers result in more large-scale blurring, which is overall\n slower than small-scale blurring.\n\n ksize : None or int, optional\n Size in height/width of the gaussian kernel. This argument is only understood by the ``cv2`` backend.\n If it is set to None, an appropriate value for `ksize` will automatically be derived from `sigma`.\n The value is chosen tighter for larger sigmas to avoid as much as possible very large kernel sizes\n and therey improve performance.\n\n backend : {'auto', 'cv2', 'scipy'}, optional\n Backend library to use. If ``auto``, then the likely best library will be automatically picked per image. That\n is usually equivalent to ``cv2`` (OpenCV) and it will fall back to ``scipy`` for datatypes not supported by\n OpenCV.\n\n eps : number, optional\n A threshold used to decide whether `sigma` can be considered zero.\n\n Returns\n -------\n image : numpy.ndarray\n The blurred image. Same shape and dtype as the input.\n\n \"\"\"\n if sigma > 0 + eps:\n dtype = image.dtype\n\n iadt.gate_dtypes(image,\n allowed=[\"bool\",\n \"uint8\", \"uint16\", \"uint32\",\n \"int8\", \"int16\", \"int32\", \"int64\", \"uint64\",\n \"float16\", \"float32\", \"float64\"],\n disallowed=[\"uint128\", \"uint256\",\n \"int128\", \"int256\",\n \"float96\", \"float128\", \"float256\"],\n augmenter=None)\n\n dts_not_supported_by_cv2 = [\"uint32\", \"uint64\", \"int64\", \"float128\"]\n backend_to_use = backend\n if backend == \"auto\":\n backend_to_use = \"cv2\" if image.dtype.name not in dts_not_supported_by_cv2 else \"scipy\"\n elif backend == \"cv2\":\n assert image.dtype.name not in dts_not_supported_by_cv2,\\\n (\"Requested 'cv2' backend, but provided %s input image, which \"\n + \"cannot be handled by that backend. Choose a different backend or \"\n + \"set backend to 'auto' or use a different datatype.\") % (image.dtype.name,)\n elif backend == \"scipy\":\n # can handle all dtypes that were allowed in gate_dtypes()\n pass\n\n if backend_to_use == \"scipy\":\n if dtype.name == \"bool\":\n # We convert bool to float32 here, because gaussian_filter() seems to only return True when\n # the underlying value is approximately 1.0, not when it is above 0.5. So we do that here manually.\n # cv2 does not support bool for gaussian blur\n image = image.astype(np.float32, copy=False)\n elif dtype.name == \"float16\":\n image = image.astype(np.float32, copy=False)\n\n # gaussian_filter() has no ksize argument\n # TODO it does have a truncate argument that truncates at x standard deviations -- maybe can be used\n # similarly to ksize\n if ksize is not None:\n warnings.warn(\"Requested 'scipy' backend or picked it automatically by backend='auto' \"\n \"in blur_gaussian_(), but also provided 'ksize' argument, which is not understood by \"\n \"that backend and will be ignored.\")\n\n # Note that while gaussian_filter can be applied to all channels at the same time, that should not\n # be done here, because then the blurring would also happen across channels (e.g. red values might\n # be mixed with blue values in RGB)\n if image.ndim == 2:\n image[:, :] = ndimage.gaussian_filter(image[:, :], sigma, mode=\"mirror\")\n else:\n nb_channels = image.shape[2]\n for channel in sm.xrange(nb_channels):\n image[:, :, channel] = ndimage.gaussian_filter(image[:, :, channel], sigma, mode=\"mirror\")\n else:\n if dtype.name == \"bool\":\n image = image.astype(np.float32, copy=False)\n elif dtype.name == \"float16\":\n image = image.astype(np.float32, copy=False)\n elif dtype.name == \"int8\":\n image = image.astype(np.int16, copy=False)\n elif dtype.name == \"int32\":\n image = image.astype(np.float64, copy=False)\n\n # ksize here is derived from the equation to compute sigma based on ksize,\n # see https://docs.opencv.org/3.1.0/d4/d86/group__imgproc__filter.html -> cv::getGaussianKernel()\n # example values:\n # sig = 0.1 -> ksize = -1.666\n # sig = 0.5 -> ksize = 0.9999\n # sig = 1.0 -> ksize = 1.0\n # sig = 2.0 -> ksize = 11.0\n # sig = 3.0 -> ksize = 17.666\n # ksize = ((sig - 0.8)/0.3 + 1)/0.5 + 1\n\n if ksize is None:\n if sigma < 3.0:\n ksize = 3.3 * sigma # 99% of weight\n elif sigma < 5.0:\n ksize = 2.9 * sigma # 97% of weight\n else:\n ksize = 2.6 * sigma # 95% of weight\n\n # we use 5x5 here as the minimum size as that simplifies comparisons with gaussian_filter() in the tests\n # TODO reduce this to 3x3\n ksize = int(max(ksize, 5))\n else:\n assert ia.is_single_integer(ksize), \"Expected 'ksize' argument to be a number, got %s.\" % (type(ksize),)\n\n ksize = ksize + 1 if ksize % 2 == 0 else ksize\n\n if ksize > 0:\n image_warped = cv2.GaussianBlur(image, (ksize, ksize), sigmaX=sigma, sigmaY=sigma,\n borderType=cv2.BORDER_REFLECT_101)\n\n # re-add channel axis removed by cv2 if input was (H, W, 1)\n image = image_warped[..., np.newaxis] if image.ndim == 3 and image_warped.ndim == 2 else image_warped\n\n if dtype.name == \"bool\":\n image = image > 0.5\n elif dtype.name != image.dtype.name:\n image = iadt.restore_dtypes_(image, dtype)\n\n return image"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Fog(name=None, deterministic=False, random_state=None):\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n return CloudLayer(\n intensity_mean=(220, 255), intensity_freq_exponent=(-2.0, -1.5), intensity_coarse_scale=2,\n alpha_min=(0.7, 0.9), alpha_multiplier=0.3, alpha_size_px_max=(2, 8), alpha_freq_exponent=(-4.0, -2.0),\n sparsity=0.9, density_multiplier=(0.4, 0.9),\n name=name, deterministic=deterministic, random_state=random_state\n )", "response": "Augmenter that draws a fairly dense cloud with low - frequency patterns."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Snowflakes(density=(0.005, 0.075), density_uniformity=(0.3, 0.9), flake_size=(0.2, 0.7),\n flake_size_uniformity=(0.4, 0.8), angle=(-30, 30), speed=(0.007, 0.03),\n name=None, deterministic=False, random_state=None):\n \"\"\"\n Augmenter to add falling snowflakes to images.\n\n This is a wrapper around ``SnowflakesLayer``. It executes 1 to 3 layers per image.\n\n dtype support::\n\n * ``uint8``: yes; tested\n * ``uint16``: no (1)\n * ``uint32``: no (1)\n * ``uint64``: no (1)\n * ``int8``: no (1)\n * ``int16``: no (1)\n * ``int32``: no (1)\n * ``int64``: no (1)\n * ``float16``: no (1)\n * ``float32``: no (1)\n * ``float64``: no (1)\n * ``float128``: no (1)\n * ``bool``: no (1)\n\n - (1) Parameters of this augmenter are optimized for the value range of uint8.\n While other dtypes may be accepted, they will lead to images augmented in\n ways inappropriate for the respective dtype.\n\n Parameters\n ----------\n density : number or tuple of number or list of number or imgaug.parameters.StochasticParameter\n Density of the snowflake layer, as a probability of each pixel in low resolution space to be a snowflake.\n Valid value range is ``(0.0, 1.0)``. Recommended to be around ``(0.01, 0.075)``.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the continuous range ``[a, b]`` will be used.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n density_uniformity : number or tuple of number or list of number or imgaug.parameters.StochasticParameter\n Size uniformity of the snowflakes. Higher values denote more similarly sized snowflakes.\n Valid value range is ``(0.0, 1.0)``. Recommended to be around ``0.5``.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the continuous range ``[a, b]`` will be used.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n flake_size : number or tuple of number or list of number or imgaug.parameters.StochasticParameter\n Size of the snowflakes. This parameter controls the resolution at which snowflakes are sampled.\n Higher values mean that the resolution is closer to the input image's resolution and hence each sampled\n snowflake will be smaller (because of the smaller pixel size).\n\n Valid value range is ``[0.0, 1.0)``. Recommended values:\n\n * On ``96x128`` a value of ``(0.1, 0.4)`` worked well.\n * On ``192x256`` a value of ``(0.2, 0.7)`` worked well.\n * On ``960x1280`` a value of ``(0.7, 0.95)`` worked well.\n\n Allowed datatypes:\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the continuous range ``[a, b]`` will be used.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n flake_size_uniformity : number or tuple of number or list of number or imgaug.parameters.StochasticParameter\n Controls the size uniformity of the snowflakes. Higher values mean that the snowflakes are more similarly\n sized. Valid value range is ``(0.0, 1.0)``. Recommended to be around ``0.5``.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the continuous range ``[a, b]`` will be used.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n angle : number or tuple of number or list of number or imgaug.parameters.StochasticParameter\n Angle in degrees of motion blur applied to the snowflakes, where ``0.0`` is motion blur that points straight\n upwards. Recommended to be around ``(-30, 30)``.\n See also :func:`imgaug.augmenters.blur.MotionBlur.__init__`.\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the continuous range ``[a, b]`` will be used.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n speed : number or tuple of number or list of number or imgaug.parameters.StochasticParameter\n Perceived falling speed of the snowflakes. This parameter controls the motion blur's kernel size.\n It follows roughly the form ``kernel_size = image_size * speed``. Hence,\n Values around ``1.0`` denote that the motion blur should \"stretch\" each snowflake over the whole image.\n\n Valid value range is ``(0.0, 1.0)``. Recommended values:\n\n * On ``96x128`` a value of ``(0.01, 0.05)`` worked well.\n * On ``192x256`` a value of ``(0.007, 0.03)`` worked well.\n * On ``960x1280`` a value of ``(0.001, 0.03)`` worked well.\n\n\n Allowed datatypes:\n\n * If a number, then that value will be used for all images.\n * If a tuple ``(a, b)``, then a value from the continuous range ``[a, b]`` will be used.\n * If a list, then a random value will be sampled from that list per image.\n * If a StochasticParameter, then a value will be sampled per image from that parameter.\n\n name : None or str, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n deterministic : bool, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n random_state : None or int or numpy.random.RandomState, optional\n See :func:`imgaug.augmenters.meta.Augmenter.__init__`.\n\n Examples\n --------\n >>> aug = iaa.Snowflakes(flake_size=(0.1, 0.4), speed=(0.01, 0.05))\n\n Adds snowflakes to small images (around ``96x128``).\n\n >>> aug = iaa.Snowflakes(flake_size=(0.2, 0.7), speed=(0.007, 0.03))\n\n Adds snowflakes to medium-sized images (around ``192x256``).\n\n >>> aug = iaa.Snowflakes(flake_size=(0.7, 0.95), speed=(0.001, 0.03))\n\n Adds snowflakes to large images (around ``960x1280``).\n\n \"\"\"\n if name is None:\n name = \"Unnamed%s\" % (ia.caller_name(),)\n\n layer = SnowflakesLayer(\n density=density, density_uniformity=density_uniformity,\n flake_size=flake_size, flake_size_uniformity=flake_size_uniformity,\n angle=angle, speed=speed,\n blur_sigma_fraction=(0.0001, 0.001)\n )\n\n return meta.SomeOf(\n (1, 3), children=[layer.deepcopy() for _ in range(3)],\n random_order=False, name=name, deterministic=deterministic, random_state=random_state\n )", "response": "Augmenter to add falling snowflakes to images."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the segmentation map array as an integer array of shape H W.", "response": "def get_arr_int(self, background_threshold=0.01, background_class_id=None):\n \"\"\"\n Get the segmentation map array as an integer array of shape (H, W).\n\n Each pixel in that array contains an integer value representing the pixel's class.\n If multiple classes overlap, the one with the highest local float value is picked.\n If that highest local value is below `background_threshold`, the method instead uses\n the background class id as the pixel's class value.\n By default, class id 0 is the background class. This may only be changed if the original\n input to the segmentation map object was an integer map.\n\n Parameters\n ----------\n background_threshold : float, optional\n At each pixel, each class-heatmap has a value between 0.0 and 1.0. If none of the\n class-heatmaps has a value above this threshold, the method uses the background class\n id instead.\n\n background_class_id : None or int, optional\n Class id to fall back to if no class-heatmap passes the threshold at a spatial\n location. May only be provided if the original input was an integer mask and in these\n cases defaults to 0. If the input were float or boolean masks, the background class id\n may not be set as it is assumed that the background is implicitly defined\n as 'any spatial location that has zero-like values in all masks'.\n\n Returns\n -------\n result : (H,W) ndarray\n Segmentation map array (int32).\n If the original input consisted of boolean or float masks, then the highest possible\n class id is ``1+C``, where ``C`` is the number of provided float/boolean masks. The value\n ``0`` in the integer mask then denotes the background class.\n\n \"\"\"\n if self.input_was[0] in [\"bool\", \"float\"]:\n ia.do_assert(background_class_id is None,\n \"The background class id may only be changed if the original input to SegmentationMapOnImage \"\n + \"was an *integer* based segmentation map.\")\n\n if background_class_id is None:\n background_class_id = 0\n\n channelwise_max_idx = np.argmax(self.arr, axis=2)\n # for bool and float input masks, we assume that the background is implicitly given,\n # i.e. anything where all masks/channels have zero-like values\n # for int, we assume that the background class is explicitly given and has the index 0\n if self.input_was[0] in [\"bool\", \"float\"]:\n result = 1 + channelwise_max_idx\n else: # integer mask was provided\n result = channelwise_max_idx\n if background_threshold is not None and background_threshold > 0:\n probs = np.amax(self.arr, axis=2)\n result[probs < background_threshold] = background_class_id\n\n return result.astype(np.int32)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef draw(self, size=None, background_threshold=0.01, background_class_id=None, colors=None,\n return_foreground_mask=False):\n \"\"\"\n Render the segmentation map as an RGB image.\n\n Parameters\n ----------\n size : None or float or iterable of int or iterable of float, optional\n Size of the rendered RGB image as ``(height, width)``.\n See :func:`imgaug.imgaug.imresize_single_image` for details.\n If set to None, no resizing is performed and the size of the segmentation map array is used.\n\n background_threshold : float, optional\n See :func:`imgaug.SegmentationMapOnImage.get_arr_int`.\n\n background_class_id : None or int, optional\n See :func:`imgaug.SegmentationMapOnImage.get_arr_int`.\n\n colors : None or list of tuple of int, optional\n Colors to use. One for each class to draw. If None, then default colors will be used.\n\n return_foreground_mask : bool, optional\n Whether to return a mask of the same size as the drawn segmentation map, containing\n True at any spatial location that is not the background class and False everywhere else.\n\n Returns\n -------\n segmap_drawn : (H,W,3) ndarray\n Rendered segmentation map (dtype is uint8).\n\n foreground_mask : (H,W) ndarray\n Mask indicating the locations of foreground classes (dtype is bool).\n This value is only returned if `return_foreground_mask` is True.\n\n \"\"\"\n arr = self.get_arr_int(background_threshold=background_threshold, background_class_id=background_class_id)\n nb_classes = 1 + np.max(arr)\n segmap_drawn = np.zeros((arr.shape[0], arr.shape[1], 3), dtype=np.uint8)\n if colors is None:\n colors = SegmentationMapOnImage.DEFAULT_SEGMENT_COLORS\n ia.do_assert(nb_classes <= len(colors),\n \"Can't draw all %d classes as it would exceed the maximum number of %d available colors.\" % (\n nb_classes, len(colors),))\n\n ids_in_map = np.unique(arr)\n for c, color in zip(sm.xrange(nb_classes), colors):\n if c in ids_in_map:\n class_mask = (arr == c)\n segmap_drawn[class_mask] = color\n\n if return_foreground_mask:\n background_class_id = 0 if background_class_id is None else background_class_id\n foreground_mask = (arr != background_class_id)\n else:\n foreground_mask = None\n\n if size is not None:\n segmap_drawn = ia.imresize_single_image(segmap_drawn, size, interpolation=\"nearest\")\n if foreground_mask is not None:\n foreground_mask = ia.imresize_single_image(\n foreground_mask.astype(np.uint8), size, interpolation=\"nearest\") > 0\n\n if foreground_mask is not None:\n return segmap_drawn, foreground_mask\n return segmap_drawn", "response": "Draws the segmentation map as an RGB image."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef draw_on_image(self, image, alpha=0.75, resize=\"segmentation_map\", background_threshold=0.01,\n background_class_id=None, colors=None, draw_background=False):\n \"\"\"\n Draw the segmentation map as an overlay over an image.\n\n Parameters\n ----------\n image : (H,W,3) ndarray\n Image onto which to draw the segmentation map. Dtype is expected to be uint8.\n\n alpha : float, optional\n Alpha/opacity value to use for the mixing of image and segmentation map.\n Higher values mean that the segmentation map will be more visible and the image less visible.\n\n resize : {'segmentation_map', 'image'}, optional\n In case of size differences between the image and segmentation map, either the image or\n the segmentation map can be resized. This parameter controls which of the two will be\n resized to the other's size.\n\n background_threshold : float, optional\n See :func:`imgaug.SegmentationMapOnImage.get_arr_int`.\n\n background_class_id : None or int, optional\n See :func:`imgaug.SegmentationMapOnImage.get_arr_int`.\n\n colors : None or list of tuple of int, optional\n Colors to use. One for each class to draw. If None, then default colors will be used.\n\n draw_background : bool, optional\n If True, the background will be drawn like any other class.\n If False, the background will not be drawn, i.e. the respective background pixels\n will be identical with the image's RGB color at the corresponding spatial location\n and no color overlay will be applied.\n\n Returns\n -------\n mix : (H,W,3) ndarray\n Rendered overlays (dtype is uint8).\n\n \"\"\"\n # assert RGB image\n ia.do_assert(image.ndim == 3)\n ia.do_assert(image.shape[2] == 3)\n ia.do_assert(image.dtype.type == np.uint8)\n\n ia.do_assert(0 - 1e-8 <= alpha <= 1.0 + 1e-8)\n ia.do_assert(resize in [\"segmentation_map\", \"image\"])\n\n if resize == \"image\":\n image = ia.imresize_single_image(image, self.arr.shape[0:2], interpolation=\"cubic\")\n\n segmap_drawn, foreground_mask = self.draw(\n background_threshold=background_threshold,\n background_class_id=background_class_id,\n size=image.shape[0:2] if resize == \"segmentation_map\" else None,\n colors=colors,\n return_foreground_mask=True\n )\n\n if draw_background:\n mix = np.clip(\n (1-alpha) * image + alpha * segmap_drawn,\n 0,\n 255\n ).astype(np.uint8)\n else:\n foreground_mask = foreground_mask[..., np.newaxis]\n mix = np.zeros_like(image)\n mix += (~foreground_mask).astype(np.uint8) * image\n mix += foreground_mask.astype(np.uint8) * np.clip(\n (1-alpha) * image + alpha * segmap_drawn,\n 0,\n 255\n ).astype(np.uint8)\n return mix", "response": "Draw the segmentation map on an image."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pad(self, top=0, right=0, bottom=0, left=0, mode=\"constant\", cval=0.0):\n arr_padded = ia.pad(self.arr, top=top, right=right, bottom=bottom, left=left, mode=mode, cval=cval)\n segmap = SegmentationMapOnImage(arr_padded, shape=self.shape)\n segmap.input_was = self.input_was\n return segmap", "response": "Pads the segmentation map on its top right bottom and left side."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\npad the segmentation map on its sides so that the image s width and height match the target aspect ratio.", "response": "def pad_to_aspect_ratio(self, aspect_ratio, mode=\"constant\", cval=0.0, return_pad_amounts=False):\n \"\"\"\n Pad the segmentation map on its sides so that its matches a target aspect ratio.\n\n Depending on which dimension is smaller (height or width), only the corresponding\n sides (left/right or top/bottom) will be padded. In each case, both of the sides will\n be padded equally.\n\n Parameters\n ----------\n aspect_ratio : float\n Target aspect ratio, given as width/height. E.g. 2.0 denotes the image having twice\n as much width as height.\n\n mode : str, optional\n Padding mode to use. See :func:`numpy.pad` for details.\n\n cval : number, optional\n Value to use for padding if `mode` is ``constant``. See :func:`numpy.pad` for details.\n\n return_pad_amounts : bool, optional\n If False, then only the padded image will be returned. If True, a tuple with two\n entries will be returned, where the first entry is the padded image and the second\n entry are the amounts by which each image side was padded. These amounts are again a\n tuple of the form (top, right, bottom, left), with each value being an integer.\n\n Returns\n -------\n segmap : imgaug.SegmentationMapOnImage\n Padded segmentation map as SegmentationMapOnImage object.\n\n pad_amounts : tuple of int\n Amounts by which the segmentation map was padded on each side, given as a\n tuple ``(top, right, bottom, left)``.\n This tuple is only returned if `return_pad_amounts` was set to True.\n\n \"\"\"\n arr_padded, pad_amounts = ia.pad_to_aspect_ratio(self.arr, aspect_ratio=aspect_ratio, mode=mode, cval=cval,\n return_pad_amounts=True)\n segmap = SegmentationMapOnImage(arr_padded, shape=self.shape)\n segmap.input_was = self.input_was\n if return_pad_amounts:\n return segmap, pad_amounts\n else:\n return segmap"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef resize(self, sizes, interpolation=\"cubic\"):\n arr_resized = ia.imresize_single_image(self.arr, sizes, interpolation=interpolation)\n\n # cubic interpolation can lead to values outside of [0.0, 1.0],\n # see https://github.com/opencv/opencv/issues/7195\n # TODO area interpolation too?\n arr_resized = np.clip(arr_resized, 0.0, 1.0)\n segmap = SegmentationMapOnImage(arr_resized, shape=self.shape)\n segmap.input_was = self.input_was\n return segmap", "response": "Resizes the segmentation map array to the provided size."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert the segment map to a list of heatmap channels.", "response": "def to_heatmaps(self, only_nonempty=False, not_none_if_no_nonempty=False):\n \"\"\"\n Convert segmentation map to heatmaps object.\n\n Each segmentation map class will be represented as a single heatmap channel.\n\n Parameters\n ----------\n only_nonempty : bool, optional\n If True, then only heatmaps for classes that appear in the segmentation map will be\n generated. Additionally, a list of these class ids will be returned.\n\n not_none_if_no_nonempty : bool, optional\n If `only_nonempty` is True and for a segmentation map no channel was non-empty,\n this function usually returns None as the heatmaps object. If however this parameter\n is set to True, a heatmaps object with one channel (representing class 0)\n will be returned as a fallback in these cases.\n\n Returns\n -------\n imgaug.HeatmapsOnImage or None\n Segmentation map as a heatmaps object.\n If `only_nonempty` was set to True and no class appeared in the segmentation map,\n then this is None.\n\n class_indices : list of int\n Class ids (0 to C-1) of the classes that were actually added to the heatmaps.\n Only returned if `only_nonempty` was set to True.\n\n \"\"\"\n # TODO get rid of this deferred import\n from imgaug.augmentables.heatmaps import HeatmapsOnImage\n\n if not only_nonempty:\n return HeatmapsOnImage.from_0to1(self.arr, self.shape, min_value=0.0, max_value=1.0)\n else:\n nonempty_mask = np.sum(self.arr, axis=(0, 1)) > 0 + 1e-4\n if np.sum(nonempty_mask) == 0:\n if not_none_if_no_nonempty:\n nonempty_mask[0] = True\n else:\n return None, []\n\n class_indices = np.arange(self.arr.shape[2])[nonempty_mask]\n channels = self.arr[..., class_indices]\n return HeatmapsOnImage(channels, self.shape, min_value=0.0, max_value=1.0), class_indices"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting a list of heatmap channels to a segment map.", "response": "def from_heatmaps(heatmaps, class_indices=None, nb_classes=None):\n \"\"\"\n Convert heatmaps to segmentation map.\n\n Assumes that each class is represented as a single heatmap channel.\n\n Parameters\n ----------\n heatmaps : imgaug.HeatmapsOnImage\n Heatmaps to convert.\n\n class_indices : None or list of int, optional\n List of class indices represented by each heatmap channel. See also the\n secondary output of :func:`imgaug.SegmentationMapOnImage.to_heatmap`.\n If this is provided, it must have the same length as the number of heatmap channels.\n\n nb_classes : None or int, optional\n Number of classes. Must be provided if class_indices is set.\n\n Returns\n -------\n imgaug.SegmentationMapOnImage\n Segmentation map derived from heatmaps.\n\n \"\"\"\n if class_indices is None:\n return SegmentationMapOnImage(heatmaps.arr_0to1, shape=heatmaps.shape)\n else:\n ia.do_assert(nb_classes is not None)\n ia.do_assert(min(class_indices) >= 0)\n ia.do_assert(max(class_indices) < nb_classes)\n ia.do_assert(len(class_indices) == heatmaps.arr_0to1.shape[2])\n arr_0to1 = heatmaps.arr_0to1\n arr_0to1_full = np.zeros((arr_0to1.shape[0], arr_0to1.shape[1], nb_classes), dtype=np.float32)\n for heatmap_channel, mapped_channel in enumerate(class_indices):\n arr_0to1_full[:, :, mapped_channel] = arr_0to1[:, :, heatmap_channel]\n return SegmentationMapOnImage(arr_0to1_full, shape=heatmaps.shape)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef deepcopy(self):\n segmap = SegmentationMapOnImage(self.arr, shape=self.shape, nb_classes=self.nb_classes)\n segmap.input_was = self.input_was\n return segmap", "response": "Create a deep copy of the segmentation map object. Returns ------- segmap"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\noffer a new event at point p in this queue.", "response": "def offer(self, p, e: Event):\n \"\"\"\n Offer a new event ``s`` at point ``p`` in this queue.\n \"\"\"\n existing = self.events_scan.setdefault(\n p, ([], [], [], []) if USE_VERTICAL else\n ([], [], []))\n # Can use double linked-list for easy insertion at beginning/end\n '''\n if e.type == Event.Type.END:\n existing.insert(0, e)\n else:\n existing.append(e)\n '''\n\n existing[e.type].append(e)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the array of the internal representation of the object.", "response": "def get_arr(self):\n \"\"\"\n Get the heatmap's array within the value range originally provided in ``__init__()``.\n\n The HeatmapsOnImage object saves heatmaps internally in the value range ``(min=0.0, max=1.0)``.\n This function converts the internal representation to ``(min=min_value, max=max_value)``,\n where ``min_value`` and ``max_value`` are provided upon instantiation of the object.\n\n Returns\n -------\n result : (H,W) ndarray or (H,W,C) ndarray\n Heatmap array. Dtype is float32.\n\n \"\"\"\n if self.arr_was_2d and self.arr_0to1.shape[2] == 1:\n arr = self.arr_0to1[:, :, 0]\n else:\n arr = self.arr_0to1\n\n eps = np.finfo(np.float32).eps\n min_is_zero = 0.0 - eps < self.min_value < 0.0 + eps\n max_is_one = 1.0 - eps < self.max_value < 1.0 + eps\n if min_is_zero and max_is_one:\n return np.copy(arr)\n else:\n diff = self.max_value - self.min_value\n return self.min_value + diff * arr"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef draw(self, size=None, cmap=\"jet\"):\n heatmaps_uint8 = self.to_uint8()\n heatmaps_drawn = []\n\n for c in sm.xrange(heatmaps_uint8.shape[2]):\n # c:c+1 here, because the additional axis is needed by imresize_single_image\n heatmap_c = heatmaps_uint8[..., c:c+1]\n\n if size is not None:\n heatmap_c_rs = ia.imresize_single_image(heatmap_c, size, interpolation=\"nearest\")\n else:\n heatmap_c_rs = heatmap_c\n heatmap_c_rs = np.squeeze(heatmap_c_rs).astype(np.float32) / 255.0\n\n if cmap is not None:\n # import only when necessary (faster startup; optional dependency; less fragile -- see issue #225)\n import matplotlib.pyplot as plt\n\n cmap_func = plt.get_cmap(cmap)\n heatmap_cmapped = cmap_func(heatmap_c_rs)\n heatmap_cmapped = np.delete(heatmap_cmapped, 3, 2)\n else:\n heatmap_cmapped = np.tile(heatmap_c_rs[..., np.newaxis], (1, 1, 3))\n\n heatmap_cmapped = np.clip(heatmap_cmapped * 255, 0, 255).astype(np.uint8)\n\n heatmaps_drawn.append(heatmap_cmapped)\n return heatmaps_drawn", "response": "Draws the contents of the array as RGB images."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndraw the heatmaps as overlays over an image.", "response": "def draw_on_image(self, image, alpha=0.75, cmap=\"jet\", resize=\"heatmaps\"):\n \"\"\"\n Draw the heatmaps as overlays over an image.\n\n Parameters\n ----------\n image : (H,W,3) ndarray\n Image onto which to draw the heatmaps. Expected to be of dtype uint8.\n\n alpha : float, optional\n Alpha/opacity value to use for the mixing of image and heatmaps.\n Higher values mean that the heatmaps will be more visible and the image less visible.\n\n cmap : str or None, optional\n Color map to use. See :func:`imgaug.HeatmapsOnImage.draw` for details.\n\n resize : {'heatmaps', 'image'}, optional\n In case of size differences between the image and heatmaps, either the image or\n the heatmaps can be resized. This parameter controls which of the two will be resized\n to the other's size.\n\n Returns\n -------\n mix : list of (H,W,3) ndarray\n Rendered overlays. One per heatmap array channel. Dtype is uint8.\n\n \"\"\"\n # assert RGB image\n ia.do_assert(image.ndim == 3)\n ia.do_assert(image.shape[2] == 3)\n ia.do_assert(image.dtype.type == np.uint8)\n\n ia.do_assert(0 - 1e-8 <= alpha <= 1.0 + 1e-8)\n ia.do_assert(resize in [\"heatmaps\", \"image\"])\n\n if resize == \"image\":\n image = ia.imresize_single_image(image, self.arr_0to1.shape[0:2], interpolation=\"cubic\")\n\n heatmaps_drawn = self.draw(\n size=image.shape[0:2] if resize == \"heatmaps\" else None,\n cmap=cmap\n )\n\n mix = [\n np.clip((1-alpha) * image + alpha * heatmap_i, 0, 255).astype(np.uint8)\n for heatmap_i\n in heatmaps_drawn\n ]\n\n return mix"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef invert(self):\n arr_inv = HeatmapsOnImage.from_0to1(1 - self.arr_0to1, shape=self.shape, min_value=self.min_value,\n max_value=self.max_value)\n arr_inv.arr_was_2d = self.arr_was_2d\n return arr_inv", "response": "Inverts each value in the heatmap shifting low towards high values and vice versa."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pad(self, top=0, right=0, bottom=0, left=0, mode=\"constant\", cval=0.0):\n arr_0to1_padded = ia.pad(self.arr_0to1, top=top, right=right, bottom=bottom, left=left, mode=mode, cval=cval)\n return HeatmapsOnImage.from_0to1(arr_0to1_padded, shape=self.shape, min_value=self.min_value,\n max_value=self.max_value)", "response": "Pads the heatmaps on the image to the specified dimensions."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npads the heatmaps on the image to match the given aspect ratio.", "response": "def pad_to_aspect_ratio(self, aspect_ratio, mode=\"constant\", cval=0.0, return_pad_amounts=False):\n \"\"\"\n Pad the heatmaps on their sides so that they match a target aspect ratio.\n\n Depending on which dimension is smaller (height or width), only the corresponding\n sides (left/right or top/bottom) will be padded. In each case, both of the sides will\n be padded equally.\n\n Parameters\n ----------\n aspect_ratio : float\n Target aspect ratio, given as width/height. E.g. 2.0 denotes the image having twice\n as much width as height.\n\n mode : str, optional\n Padding mode to use. See :func:`numpy.pad` for details.\n\n cval : number, optional\n Value to use for padding if `mode` is ``constant``. See :func:`numpy.pad` for details.\n\n return_pad_amounts : bool, optional\n If False, then only the padded image will be returned. If True, a tuple with two\n entries will be returned, where the first entry is the padded image and the second\n entry are the amounts by which each image side was padded. These amounts are again a\n tuple of the form (top, right, bottom, left), with each value being an integer.\n\n Returns\n -------\n heatmaps : imgaug.HeatmapsOnImage\n Padded heatmaps as HeatmapsOnImage object.\n\n pad_amounts : tuple of int\n Amounts by which the heatmaps were padded on each side, given as a tuple ``(top, right, bottom, left)``.\n This tuple is only returned if `return_pad_amounts` was set to True.\n\n \"\"\"\n arr_0to1_padded, pad_amounts = ia.pad_to_aspect_ratio(self.arr_0to1, aspect_ratio=aspect_ratio, mode=mode,\n cval=cval, return_pad_amounts=True)\n heatmaps = HeatmapsOnImage.from_0to1(arr_0to1_padded, shape=self.shape, min_value=self.min_value,\n max_value=self.max_value)\n if return_pad_amounts:\n return heatmaps, pad_amounts\n else:\n return heatmaps"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef max_pool(self, block_size):\n arr_0to1_reduced = ia.max_pool(self.arr_0to1, block_size)\n return HeatmapsOnImage.from_0to1(arr_0to1_reduced, shape=self.shape, min_value=self.min_value,\n max_value=self.max_value)", "response": "Resize the array using max - pooling."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef resize(self, sizes, interpolation=\"cubic\"):\n arr_0to1_resized = ia.imresize_single_image(self.arr_0to1, sizes, interpolation=interpolation)\n\n # cubic interpolation can lead to values outside of [0.0, 1.0],\n # see https://github.com/opencv/opencv/issues/7195\n # TODO area interpolation too?\n arr_0to1_resized = np.clip(arr_0to1_resized, 0.0, 1.0)\n\n return HeatmapsOnImage.from_0to1(arr_0to1_resized, shape=self.shape, min_value=self.min_value,\n max_value=self.max_value)", "response": "Resize the array to the provided size."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to_uint8(self):\n # TODO this always returns (H,W,C), even if input ndarray was originall (H,W)\n # does it make sense here to also return (H,W) if self.arr_was_2d?\n arr_0to255 = np.clip(np.round(self.arr_0to1 * 255), 0, 255)\n arr_uint8 = arr_0to255.astype(np.uint8)\n return arr_uint8", "response": "Convert this heatmaps object to a 0 - to - 255 array."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a new imgaug. HeatmapsOnImage object from an array of uint8 values.", "response": "def from_uint8(arr_uint8, shape, min_value=0.0, max_value=1.0):\n \"\"\"\n Create a heatmaps object from an heatmap array containing values ranging from 0 to 255.\n\n Parameters\n ----------\n arr_uint8 : (H,W) ndarray or (H,W,C) ndarray\n Heatmap(s) array, where ``H`` is height, ``W`` is width and ``C`` is the number of heatmap channels.\n Expected dtype is uint8.\n\n shape : tuple of int\n Shape of the image on which the heatmap(s) is/are placed. NOT the shape of the\n heatmap(s) array, unless it is identical to the image shape (note the likely\n difference between the arrays in the number of channels).\n If there is not a corresponding image, use the shape of the heatmaps array.\n\n min_value : float, optional\n Minimum value for the heatmaps that the 0-to-255 array represents. This will usually\n be 0.0. It is used when calling :func:`imgaug.HeatmapsOnImage.get_arr`, which converts the\n underlying ``(0, 255)`` array to value range ``(min_value, max_value)``.\n\n max_value : float, optional\n Maximum value for the heatmaps that 0-to-255 array represents.\n See parameter `min_value` for details.\n\n Returns\n -------\n imgaug.HeatmapsOnImage\n Heatmaps object.\n\n \"\"\"\n arr_0to1 = arr_uint8.astype(np.float32) / 255.0\n return HeatmapsOnImage.from_0to1(arr_0to1, shape, min_value=min_value, max_value=max_value)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a new object from an array containing values ranging from 0. 0 to 1. 0.", "response": "def from_0to1(arr_0to1, shape, min_value=0.0, max_value=1.0):\n \"\"\"\n Create a heatmaps object from an heatmap array containing values ranging from 0.0 to 1.0.\n\n Parameters\n ----------\n arr_0to1 : (H,W) or (H,W,C) ndarray\n Heatmap(s) array, where ``H`` is height, ``W`` is width and ``C`` is the number of heatmap channels.\n Expected dtype is float32.\n\n shape : tuple of ints\n Shape of the image on which the heatmap(s) is/are placed. NOT the shape of the\n heatmap(s) array, unless it is identical to the image shape (note the likely\n difference between the arrays in the number of channels).\n If there is not a corresponding image, use the shape of the heatmaps array.\n\n min_value : float, optional\n Minimum value for the heatmaps that the 0-to-1 array represents. This will usually\n be 0.0. It is used when calling :func:`imgaug.HeatmapsOnImage.get_arr`, which converts the\n underlying ``(0.0, 1.0)`` array to value range ``(min_value, max_value)``.\n E.g. if you started with heatmaps in the range ``(-1.0, 1.0)`` and projected these\n to (0.0, 1.0), you should call this function with ``min_value=-1.0``, ``max_value=1.0``\n so that :func:`imgaug.HeatmapsOnImage.get_arr` returns heatmap arrays having value\n range (-1.0, 1.0).\n\n max_value : float, optional\n Maximum value for the heatmaps that to 0-to-255 array represents.\n See parameter min_value for details.\n\n Returns\n -------\n heatmaps : imgaug.HeatmapsOnImage\n Heatmaps object.\n\n \"\"\"\n heatmaps = HeatmapsOnImage(arr_0to1, shape, min_value=0.0, max_value=1.0)\n heatmaps.min_value = min_value\n heatmaps.max_value = max_value\n return heatmaps"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchanging the value range of a heatmap array to one min - max to another min - max.", "response": "def change_normalization(cls, arr, source, target):\n \"\"\"\n Change the value range of a heatmap from one min-max to another min-max.\n\n E.g. the value range may be changed from min=0.0, max=1.0 to min=-1.0, max=1.0.\n\n Parameters\n ----------\n arr : ndarray\n Heatmap array to modify.\n\n source : tuple of float\n Current value range of the input array, given as (min, max), where both are float values.\n\n target : tuple of float\n Desired output value range of the array, given as (min, max), where both are float values.\n\n Returns\n -------\n arr_target : ndarray\n Input array, with value range projected to the desired target value range.\n\n \"\"\"\n ia.do_assert(ia.is_np_array(arr))\n\n if isinstance(source, HeatmapsOnImage):\n source = (source.min_value, source.max_value)\n else:\n ia.do_assert(isinstance(source, tuple))\n ia.do_assert(len(source) == 2)\n ia.do_assert(source[0] < source[1])\n\n if isinstance(target, HeatmapsOnImage):\n target = (target.min_value, target.max_value)\n else:\n ia.do_assert(isinstance(target, tuple))\n ia.do_assert(len(target) == 2)\n ia.do_assert(target[0] < target[1])\n\n # Check if source and target are the same (with a tiny bit of tolerance)\n # if so, evade compuation and just copy the array instead.\n # This is reasonable, as source and target will often both be (0.0, 1.0).\n eps = np.finfo(arr.dtype).eps\n mins_same = source[0] - 10*eps < target[0] < source[0] + 10*eps\n maxs_same = source[1] - 10*eps < target[1] < source[1] + 10*eps\n if mins_same and maxs_same:\n return np.copy(arr)\n\n min_source, max_source = source\n min_target, max_target = target\n\n diff_source = max_source - min_source\n diff_target = max_target - min_target\n\n arr_0to1 = (arr - min_source) / diff_source\n arr_target = min_target + arr_0to1 * diff_target\n\n return arr_target"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a deep copy of the Heatmaps object. Returns ------- imgaug. HeatmapsOnImage Deep copy of the Heatmaps object.", "response": "def deepcopy(self):\n \"\"\"\n Create a deep copy of the Heatmaps object.\n\n Returns\n -------\n imgaug.HeatmapsOnImage\n Deep copy.\n\n \"\"\"\n return HeatmapsOnImage(self.get_arr(), shape=self.shape, min_value=self.min_value, max_value=self.max_value)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef setdefault(self, key: str, value: str) -> str:\n set_key = key.lower().encode(\"latin-1\")\n set_value = value.encode(\"latin-1\")\n\n for idx, (item_key, item_value) in enumerate(self._list):\n if item_key == set_key:\n return item_value.decode(\"latin-1\")\n self._list.append((set_key, set_value))\n return value", "response": "Sets the value of the header key to value."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nappend a new entry to the list.", "response": "def append(self, key: str, value: str) -> None:\n \"\"\"\n Append a header, preserving any duplicate entries.\n \"\"\"\n append_key = key.lower().encode(\"latin-1\")\n append_value = value.encode(\"latin-1\")\n self._list.append((append_key, append_value))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef request_response(func: typing.Callable) -> ASGIApp:\n is_coroutine = asyncio.iscoroutinefunction(func)\n\n async def app(scope: Scope, receive: Receive, send: Send) -> None:\n request = Request(scope, receive=receive)\n if is_coroutine:\n response = await func(request)\n else:\n response = await run_in_threadpool(func, request)\n await response(scope, receive, send)\n\n return app", "response": "Takes a function or coroutine func ( request response ) -> ASGIApp and returns an ASGI application."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef websocket_session(func: typing.Callable) -> ASGIApp:\n # assert asyncio.iscoroutinefunction(func), \"WebSocket endpoints must be async\"\n\n async def app(scope: Scope, receive: Receive, send: Send) -> None:\n session = WebSocket(scope, receive=receive, send=send)\n await func(session)\n\n return app", "response": "Takes a coroutine func and returns an ASGI application."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef compile_path(\n path: str\n) -> typing.Tuple[typing.Pattern, str, typing.Dict[str, Convertor]]:\n \"\"\"\n Given a path string, like: \"/{username:str}\", return a three-tuple\n of (regex, format, {param_name:convertor}).\n\n regex: \"/(?P[^/]+)\"\n format: \"/{username}\"\n convertors: {\"username\": StringConvertor()}\n \"\"\"\n path_regex = \"^\"\n path_format = \"\"\n\n idx = 0\n param_convertors = {}\n for match in PARAM_REGEX.finditer(path):\n param_name, convertor_type = match.groups(\"str\")\n convertor_type = convertor_type.lstrip(\":\")\n assert (\n convertor_type in CONVERTOR_TYPES\n ), f\"Unknown path convertor '{convertor_type}'\"\n convertor = CONVERTOR_TYPES[convertor_type]\n\n path_regex += path[idx : match.start()]\n path_regex += f\"(?P<{param_name}>{convertor.regex})\"\n\n path_format += path[idx : match.start()]\n path_format += \"{%s}\" % param_name\n\n param_convertors[param_name] = convertor\n\n idx = match.end()\n\n path_regex += path[idx:] + \"$\"\n path_format += path[idx:]\n\n return re.compile(path_regex), path_format, param_convertors", "response": "Given a path string return a three - tuple containing a regex format param_name and convertors."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_endpoints(\n self, routes: typing.List[BaseRoute]\n ) -> typing.List[EndpointInfo]:\n \"\"\"\n Given the routes, yields the following information:\n\n - path\n eg: /users/\n - http_method\n one of 'get', 'post', 'put', 'patch', 'delete', 'options'\n - func\n method ready to extract the docstring\n \"\"\"\n endpoints_info: list = []\n\n for route in routes:\n if isinstance(route, Mount):\n routes = route.routes or []\n sub_endpoints = [\n EndpointInfo(\n path=\"\".join((route.path, sub_endpoint.path)),\n http_method=sub_endpoint.http_method,\n func=sub_endpoint.func,\n )\n for sub_endpoint in self.get_endpoints(routes)\n ]\n endpoints_info.extend(sub_endpoints)\n\n elif not isinstance(route, Route) or not route.include_in_schema:\n continue\n\n elif inspect.isfunction(route.endpoint) or inspect.ismethod(route.endpoint):\n for method in route.methods or [\"GET\"]:\n if method == \"HEAD\":\n continue\n endpoints_info.append(\n EndpointInfo(route.path, method.lower(), route.endpoint)\n )\n else:\n for method in [\"get\", \"post\", \"put\", \"patch\", \"delete\", \"options\"]:\n if not hasattr(route.endpoint, method):\n continue\n func = getattr(route.endpoint, method)\n endpoints_info.append(\n EndpointInfo(route.path, method.lower(), func)\n )\n\n return endpoints_info", "response": "Given the routes yields the list of endpoints that can be used to create the resource."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_docstring(self, func_or_method: typing.Callable) -> dict:\n docstring = func_or_method.__doc__\n if not docstring:\n return {}\n\n # We support having regular docstrings before the schema\n # definition. Here we return just the schema part from\n # the docstring.\n docstring = docstring.split(\"---\")[-1]\n\n parsed = yaml.safe_load(docstring)\n\n if not isinstance(parsed, dict):\n # A regular docstring (not yaml formatted) can return\n # a simple string here, which wouldn't follow the schema.\n return {}\n\n return parsed", "response": "Given a function parse the docstring as YAML and return a dictionary of info."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngive a directory and packages arugments return a list of all the directories that should be used for serving static files from.", "response": "def get_directories(\n self, directory: str = None, packages: typing.List[str] = None\n ) -> typing.List[str]:\n \"\"\"\n Given `directory` and `packages` arugments, return a list of all the\n directories that should be used for serving static files from.\n \"\"\"\n directories = []\n if directory is not None:\n directories.append(directory)\n\n for package in packages or []:\n spec = importlib.util.find_spec(package)\n assert spec is not None, f\"Package {package!r} could not be found.\"\n assert (\n spec.origin is not None\n ), \"Directory 'statics' in package {package!r} could not be found.\"\n directory = os.path.normpath(os.path.join(spec.origin, \"..\", \"statics\"))\n assert os.path.isdir(\n directory\n ), \"Directory 'statics' in package {package!r} could not be found.\"\n directories.append(directory)\n\n return directories"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_path(self, scope: Scope) -> str:\n return os.path.normpath(os.path.join(*scope[\"path\"].split(\"/\")))", "response": "Given the ASGI scope return the path to serve up."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\nasync def get_response(self, path: str, scope: Scope) -> Response:\n if scope[\"method\"] not in (\"GET\", \"HEAD\"):\n return PlainTextResponse(\"Method Not Allowed\", status_code=405)\n\n if path.startswith(\"..\"):\n # Most clients will normalize the path, so we shouldn't normally\n # get this, but don't allow misbehaving clients to break out of\n # the static files directory.\n return PlainTextResponse(\"Not Found\", status_code=404)\n\n full_path, stat_result = await self.lookup_path(path)\n\n if stat_result and stat.S_ISREG(stat_result.st_mode):\n # We have a static file to serve.\n return self.file_response(full_path, stat_result, scope)\n\n elif stat_result and stat.S_ISDIR(stat_result.st_mode) and self.html:\n # We're in HTML mode, and have got a directory URL.\n # Check if we have 'index.html' file to serve.\n index_path = os.path.join(path, \"index.html\")\n full_path, stat_result = await self.lookup_path(index_path)\n if stat_result is not None and stat.S_ISREG(stat_result.st_mode):\n if not scope[\"path\"].endswith(\"/\"):\n # Directory URLs should redirect to always end in \"/\".\n url = URL(scope=scope)\n url = url.replace(path=url.path + \"/\")\n return RedirectResponse(url=url)\n return self.file_response(full_path, stat_result, scope)\n\n if self.html:\n # Check for '404.html' if we're in HTML mode.\n full_path, stat_result = await self.lookup_path(\"404.html\")\n if stat_result is not None and stat.S_ISREG(stat_result.st_mode):\n return self.file_response(\n full_path, stat_result, scope, status_code=404\n )\n\n return PlainTextResponse(\"Not Found\", status_code=404)", "response": "Returns an HTTP response given the incoming path method and request headers."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\nasync def check_config(self) -> None:\n if self.directory is None:\n return\n\n try:\n stat_result = await aio_stat(self.directory)\n except FileNotFoundError:\n raise RuntimeError(\n f\"StaticFiles directory '{self.directory}' does not exist.\"\n )\n if not (stat.S_ISDIR(stat_result.st_mode) or stat.S_ISLNK(stat_result.st_mode)):\n raise RuntimeError(\n f\"StaticFiles path '{self.directory}' is not a directory.\"\n )", "response": "Perform a one - off configuration check that StaticFiles is actually stored at a directory."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngiving the request and response headers return True if an HTTP Not Modified response could be returned instead.", "response": "def is_not_modified(\n self, response_headers: Headers, request_headers: Headers\n ) -> bool:\n \"\"\"\n Given the request and response headers, return `True` if an HTTP\n \"Not Modified\" response could be returned instead.\n \"\"\"\n try:\n if_none_match = request_headers[\"if-none-match\"]\n etag = response_headers[\"etag\"]\n if if_none_match == etag:\n return True\n except KeyError:\n pass\n\n try:\n if_modified_since = parsedate(request_headers[\"if-modified-since\"])\n last_modified = parsedate(response_headers[\"last-modified\"])\n if (\n if_modified_since is not None\n and last_modified is not None\n and if_modified_since >= last_modified\n ):\n return True\n except KeyError:\n pass\n\n return False"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef build_environ(scope: Scope, body: bytes) -> dict:\n environ = {\n \"REQUEST_METHOD\": scope[\"method\"],\n \"SCRIPT_NAME\": scope.get(\"root_path\", \"\"),\n \"PATH_INFO\": scope[\"path\"],\n \"QUERY_STRING\": scope[\"query_string\"].decode(\"ascii\"),\n \"SERVER_PROTOCOL\": f\"HTTP/{scope['http_version']}\",\n \"wsgi.version\": (1, 0),\n \"wsgi.url_scheme\": scope.get(\"scheme\", \"http\"),\n \"wsgi.input\": io.BytesIO(body),\n \"wsgi.errors\": sys.stdout,\n \"wsgi.multithread\": True,\n \"wsgi.multiprocess\": True,\n \"wsgi.run_once\": False,\n }\n\n # Get server name and port - required in WSGI, not in ASGI\n server = scope.get(\"server\") or (\"localhost\", 80)\n environ[\"SERVER_NAME\"] = server[0]\n environ[\"SERVER_PORT\"] = server[1]\n\n # Get client IP address\n if scope.get(\"client\"):\n environ[\"REMOTE_ADDR\"] = scope[\"client\"][0]\n\n # Go through headers and make them into environ entries\n for name, value in scope.get(\"headers\", []):\n name = name.decode(\"latin1\")\n if name == \"content-length\":\n corrected_name = \"CONTENT_LENGTH\"\n elif name == \"content-type\":\n corrected_name = \"CONTENT_TYPE\"\n else:\n corrected_name = f\"HTTP_{name}\".upper().replace(\"-\", \"_\")\n # HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case\n value = value.decode(\"latin1\")\n if corrected_name in environ:\n value = environ[corrected_name] + \",\" + value\n environ[corrected_name] = value\n return environ", "response": "Builds a WSGI environment object from a scope and request body."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreceiving ASGI websocket messages.", "response": "async def receive(self) -> Message:\n \"\"\"\n Receive ASGI websocket messages, ensuring valid state transitions.\n \"\"\"\n if self.client_state == WebSocketState.CONNECTING:\n message = await self._receive()\n message_type = message[\"type\"]\n assert message_type == \"websocket.connect\"\n self.client_state = WebSocketState.CONNECTED\n return message\n elif self.client_state == WebSocketState.CONNECTED:\n message = await self._receive()\n message_type = message[\"type\"]\n assert message_type in {\"websocket.receive\", \"websocket.disconnect\"}\n if message_type == \"websocket.disconnect\":\n self.client_state = WebSocketState.DISCONNECTED\n return message\n else:\n raise RuntimeError(\n 'Cannot call \"receive\" once a disconnect message has been received.'\n )"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsend ASGI websocket messages ensuring valid state transitions.", "response": "async def send(self, message: Message) -> None:\n \"\"\"\n Send ASGI websocket messages, ensuring valid state transitions.\n \"\"\"\n if self.application_state == WebSocketState.CONNECTING:\n message_type = message[\"type\"]\n assert message_type in {\"websocket.accept\", \"websocket.close\"}\n if message_type == \"websocket.close\":\n self.application_state = WebSocketState.DISCONNECTED\n else:\n self.application_state = WebSocketState.CONNECTED\n await self._send(message)\n elif self.application_state == WebSocketState.CONNECTED:\n message_type = message[\"type\"]\n assert message_type in {\"websocket.send\", \"websocket.close\"}\n if message_type == \"websocket.close\":\n self.application_state = WebSocketState.DISCONNECTED\n await self._send(message)\n else:\n raise RuntimeError('Cannot call \"send\" once a close message has been sent.')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the DataFrame containing the top long short and absolute positions.", "response": "def get_top_long_short_abs(positions, top=10):\n \"\"\"\n Finds the top long, short, and absolute positions.\n\n Parameters\n ----------\n positions : pd.DataFrame\n The positions that the strategy takes over time.\n top : int, optional\n How many of each to find (default 10).\n\n Returns\n -------\n df_top_long : pd.DataFrame\n Top long positions.\n df_top_short : pd.DataFrame\n Top short positions.\n df_top_abs : pd.DataFrame\n Top absolute positions.\n \"\"\"\n\n positions = positions.drop('cash', axis='columns')\n df_max = positions.max()\n df_min = positions.min()\n df_abs_max = positions.abs().max()\n df_top_long = df_max[df_max > 0].nlargest(top)\n df_top_short = df_min[df_min < 0].nsmallest(top)\n df_top_abs = df_abs_max.nlargest(top)\n return df_top_long, df_top_short, df_top_abs"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_max_median_position_concentration(positions):\n\n expos = get_percent_alloc(positions)\n expos = expos.drop('cash', axis=1)\n\n longs = expos.where(expos.applymap(lambda x: x > 0))\n shorts = expos.where(expos.applymap(lambda x: x < 0))\n\n alloc_summary = pd.DataFrame()\n alloc_summary['max_long'] = longs.max(axis=1)\n alloc_summary['median_long'] = longs.median(axis=1)\n alloc_summary['median_short'] = shorts.median(axis=1)\n alloc_summary['max_short'] = shorts.min(axis=1)\n\n return alloc_summary", "response": "Returns the max and median long and median short position concentrations for the given positions."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef extract_pos(positions, cash):\n\n positions = positions.copy()\n positions['values'] = positions.amount * positions.last_sale_price\n\n cash.name = 'cash'\n\n values = positions.reset_index().pivot_table(index='index',\n columns='sid',\n values='values')\n\n if ZIPLINE:\n for asset in values.columns:\n if type(asset) in [Equity, Future]:\n values[asset] = values[asset] * asset.price_multiplier\n\n values = values.join(cash).fillna(0)\n\n # NOTE: Set name of DataFrame.columns to sid, to match the behavior\n # of DataFrame.join in earlier versions of pandas.\n values.columns.name = 'sid'\n\n return values", "response": "Extract position values from a dataframe containing a single cash."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_sector_exposures(positions, symbol_sector_map):\n\n cash = positions['cash']\n positions = positions.drop('cash', axis=1)\n\n unmapped_pos = np.setdiff1d(positions.columns.values,\n list(symbol_sector_map.keys()))\n if len(unmapped_pos) > 0:\n warn_message = \"\"\"Warning: Symbols {} have no sector mapping.\n They will not be included in sector allocations\"\"\".format(\n \", \".join(map(str, unmapped_pos)))\n warnings.warn(warn_message, UserWarning)\n\n sector_exp = positions.groupby(\n by=symbol_sector_map, axis=1).sum()\n\n sector_exp['cash'] = cash\n\n return sector_exp", "response": "Returns a DataFrame containing all possible exposures for a given set of positions."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndetermining the long and short allocations in a portfolio.", "response": "def get_long_short_pos(positions):\n \"\"\"\n Determines the long and short allocations in a portfolio.\n\n Parameters\n ----------\n positions : pd.DataFrame\n The positions that the strategy takes over time.\n\n Returns\n -------\n df_long_short : pd.DataFrame\n Long and short allocations as a decimal\n percentage of the total net liquidation\n \"\"\"\n\n pos_wo_cash = positions.drop('cash', axis=1)\n longs = pos_wo_cash[pos_wo_cash > 0].sum(axis=1).fillna(0)\n shorts = pos_wo_cash[pos_wo_cash < 0].sum(axis=1).fillna(0)\n cash = positions.cash\n net_liquidation = longs + shorts + cash\n df_pos = pd.DataFrame({'long': longs.divide(net_liquidation, axis='index'),\n 'short': shorts.divide(net_liquidation,\n axis='index')})\n df_pos['net exposure'] = df_pos['long'] + df_pos['short']\n return df_pos"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the style factor exposure of an algorithm s positions and risk factor.", "response": "def compute_style_factor_exposures(positions, risk_factor):\n \"\"\"\n Returns style factor exposure of an algorithm's positions\n\n Parameters\n ----------\n positions : pd.DataFrame\n Daily equity positions of algorithm, in dollars.\n - See full explanation in create_risk_tear_sheet\n\n risk_factor : pd.DataFrame\n Daily risk factor per asset.\n - DataFrame with dates as index and equities as columns\n - Example:\n Equity(24 Equity(62\n [AAPL]) [ABT])\n 2017-04-03\t -0.51284 1.39173\n 2017-04-04\t -0.73381 0.98149\n 2017-04-05\t -0.90132 1.13981\n \"\"\"\n\n positions_wo_cash = positions.drop('cash', axis='columns')\n gross_exposure = positions_wo_cash.abs().sum(axis='columns')\n\n style_factor_exposure = positions_wo_cash.multiply(risk_factor) \\\n .divide(gross_exposure, axis='index')\n tot_style_factor_exposure = style_factor_exposure.sum(axis='columns',\n skipna=True)\n\n return tot_style_factor_exposure"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nplotting the factor exposures of a daily style factor.", "response": "def plot_style_factor_exposures(tot_style_factor_exposure, factor_name=None,\n ax=None):\n \"\"\"\n Plots DataFrame output of compute_style_factor_exposures as a line graph\n\n Parameters\n ----------\n tot_style_factor_exposure : pd.Series\n Daily style factor exposures (output of compute_style_factor_exposures)\n - Time series with decimal style factor exposures\n - Example:\n 2017-04-24 0.037820\n 2017-04-25 0.016413\n 2017-04-26 -0.021472\n 2017-04-27 -0.024859\n\n factor_name : string\n Name of style factor, for use in graph title\n - Defaults to tot_style_factor_exposure.name\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n if factor_name is None:\n factor_name = tot_style_factor_exposure.name\n\n ax.plot(tot_style_factor_exposure.index, tot_style_factor_exposure,\n label=factor_name)\n avg = tot_style_factor_exposure.mean()\n ax.axhline(avg, linestyle='-.', label='Mean = {:.3}'.format(avg))\n ax.axhline(0, color='k', linestyle='-')\n _, _, y1, y2 = plt.axis()\n lim = max(abs(y1), abs(y2))\n ax.set(title='Exposure to {}'.format(factor_name),\n ylabel='{} \\n weighted exposure'.format(factor_name),\n ylim=(-lim, lim))\n ax.legend(frameon=True, framealpha=0.5)\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef compute_sector_exposures(positions, sectors, sector_dict=SECTORS):\n\n sector_ids = sector_dict.keys()\n\n long_exposures = []\n short_exposures = []\n gross_exposures = []\n net_exposures = []\n\n positions_wo_cash = positions.drop('cash', axis='columns')\n long_exposure = positions_wo_cash[positions_wo_cash > 0] \\\n .sum(axis='columns')\n short_exposure = positions_wo_cash[positions_wo_cash < 0] \\\n .abs().sum(axis='columns')\n gross_exposure = positions_wo_cash.abs().sum(axis='columns')\n\n for sector_id in sector_ids:\n in_sector = positions_wo_cash[sectors == sector_id]\n\n long_sector = in_sector[in_sector > 0] \\\n .sum(axis='columns').divide(long_exposure)\n short_sector = in_sector[in_sector < 0] \\\n .sum(axis='columns').divide(short_exposure)\n gross_sector = in_sector.abs().sum(axis='columns') \\\n .divide(gross_exposure)\n net_sector = long_sector.subtract(short_sector)\n\n long_exposures.append(long_sector)\n short_exposures.append(short_sector)\n gross_exposures.append(gross_sector)\n net_exposures.append(net_sector)\n\n return long_exposures, short_exposures, gross_exposures, net_exposures", "response": "Computes the long short and gross sector exposures of an algorithm s sectors."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nplot the long and short exposures of a single sector.", "response": "def plot_sector_exposures_longshort(long_exposures, short_exposures,\n sector_dict=SECTORS, ax=None):\n \"\"\"\n Plots outputs of compute_sector_exposures as area charts\n\n Parameters\n ----------\n long_exposures, short_exposures : arrays\n Arrays of long and short sector exposures (output of\n compute_sector_exposures).\n\n sector_dict : dict or OrderedDict\n Dictionary of all sectors\n - See full description in compute_sector_exposures\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n if sector_dict is None:\n sector_names = SECTORS.values()\n else:\n sector_names = sector_dict.values()\n\n color_list = plt.cm.gist_rainbow(np.linspace(0, 1, 11))\n\n ax.stackplot(long_exposures[0].index, long_exposures,\n labels=sector_names, colors=color_list, alpha=0.8,\n baseline='zero')\n ax.stackplot(long_exposures[0].index, short_exposures,\n colors=color_list, alpha=0.8, baseline='zero')\n ax.axhline(0, color='k', linestyle='-')\n ax.set(title='Long and short exposures to sectors',\n ylabel='Proportion of long/short exposure in sectors')\n ax.legend(loc='upper left', frameon=True, framealpha=0.5)\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef plot_sector_exposures_gross(gross_exposures, sector_dict=None, ax=None):\n\n if ax is None:\n ax = plt.gca()\n\n if sector_dict is None:\n sector_names = SECTORS.values()\n else:\n sector_names = sector_dict.values()\n\n color_list = plt.cm.gist_rainbow(np.linspace(0, 1, 11))\n\n ax.stackplot(gross_exposures[0].index, gross_exposures,\n labels=sector_names, colors=color_list, alpha=0.8,\n baseline='zero')\n ax.axhline(0, color='k', linestyle='-')\n ax.set(title='Gross exposure to sectors',\n ylabel='Proportion of gross exposure \\n in sectors')\n\n return ax", "response": "Plots the gross exposures of all sectors."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_sector_exposures_net(net_exposures, sector_dict=None, ax=None):\n\n if ax is None:\n ax = plt.gca()\n\n if sector_dict is None:\n sector_names = SECTORS.values()\n else:\n sector_names = sector_dict.values()\n\n color_list = plt.cm.gist_rainbow(np.linspace(0, 1, 11))\n\n for i in range(len(net_exposures)):\n ax.plot(net_exposures[i], color=color_list[i], alpha=0.8,\n label=sector_names[i])\n ax.set(title='Net exposures to sectors',\n ylabel='Proportion of net exposure \\n in sectors')\n\n return ax", "response": "Plots net exposures as line graphs for each sector in the list of sectors."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef compute_cap_exposures(positions, caps):\n\n long_exposures = []\n short_exposures = []\n gross_exposures = []\n net_exposures = []\n\n positions_wo_cash = positions.drop('cash', axis='columns')\n tot_gross_exposure = positions_wo_cash.abs().sum(axis='columns')\n tot_long_exposure = positions_wo_cash[positions_wo_cash > 0] \\\n .sum(axis='columns')\n tot_short_exposure = positions_wo_cash[positions_wo_cash < 0] \\\n .abs().sum(axis='columns')\n\n for bucket_name, boundaries in CAP_BUCKETS.items():\n in_bucket = positions_wo_cash[(caps >= boundaries[0]) &\n (caps <= boundaries[1])]\n\n gross_bucket = in_bucket.abs().sum(axis='columns') \\\n .divide(tot_gross_exposure)\n long_bucket = in_bucket[in_bucket > 0] \\\n .sum(axis='columns').divide(tot_long_exposure)\n short_bucket = in_bucket[in_bucket < 0] \\\n .sum(axis='columns').divide(tot_short_exposure)\n net_bucket = long_bucket.subtract(short_bucket)\n\n gross_exposures.append(gross_bucket)\n long_exposures.append(long_bucket)\n short_exposures.append(short_bucket)\n net_exposures.append(net_bucket)\n\n return long_exposures, short_exposures, gross_exposures, net_exposures", "response": "Computes the long short and gross market cap exposures of an anonymized algorithm s positions."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef plot_cap_exposures_longshort(long_exposures, short_exposures, ax=None):\n\n if ax is None:\n ax = plt.gca()\n\n color_list = plt.cm.gist_rainbow(np.linspace(0, 1, 5))\n\n ax.stackplot(long_exposures[0].index, long_exposures,\n labels=CAP_BUCKETS.keys(), colors=color_list, alpha=0.8,\n baseline='zero')\n ax.stackplot(long_exposures[0].index, short_exposures, colors=color_list,\n alpha=0.8, baseline='zero')\n ax.axhline(0, color='k', linestyle='-')\n ax.set(title='Long and short exposures to market caps',\n ylabel='Proportion of long/short exposure in market cap buckets')\n ax.legend(loc='upper left', frameon=True, framealpha=0.5)\n\n return ax", "response": "Plots long and short market cap exposures."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_cap_exposures_gross(gross_exposures, ax=None):\n\n if ax is None:\n ax = plt.gca()\n\n color_list = plt.cm.gist_rainbow(np.linspace(0, 1, 5))\n\n ax.stackplot(gross_exposures[0].index, gross_exposures,\n labels=CAP_BUCKETS.keys(), colors=color_list, alpha=0.8,\n baseline='zero')\n ax.axhline(0, color='k', linestyle='-')\n ax.set(title='Gross exposure to market caps',\n ylabel='Proportion of gross exposure \\n in market cap buckets')\n\n return ax", "response": "Plots the gross exposures of the market caps."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef plot_cap_exposures_net(net_exposures, ax=None):\n\n if ax is None:\n ax = plt.gca()\n\n color_list = plt.cm.gist_rainbow(np.linspace(0, 1, 5))\n\n cap_names = CAP_BUCKETS.keys()\n for i in range(len(net_exposures)):\n ax.plot(net_exposures[i], color=color_list[i], alpha=0.8,\n label=cap_names[i])\n ax.axhline(0, color='k', linestyle='-')\n ax.set(title='Net exposure to market caps',\n ylabel='Proportion of net exposure \\n in market cap buckets')\n\n return ax", "response": "Plots the net exposure to market caps as line graphs."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the exposures of the given volume in a given set of shares.", "response": "def compute_volume_exposures(shares_held, volumes, percentile):\n \"\"\"\n Returns arrays of pth percentile of long, short and gross volume exposures\n of an algorithm's held shares\n\n Parameters\n ----------\n shares_held : pd.DataFrame\n Daily number of shares held by an algorithm.\n - See full explanation in create_risk_tear_sheet\n\n volume : pd.DataFrame\n Daily volume per asset\n - See full explanation in create_risk_tear_sheet\n\n percentile : float\n Percentile to use when computing and plotting volume exposures\n - See full explanation in create_risk_tear_sheet\n \"\"\"\n\n shares_held = shares_held.replace(0, np.nan)\n\n shares_longed = shares_held[shares_held > 0]\n shares_shorted = -1 * shares_held[shares_held < 0]\n shares_grossed = shares_held.abs()\n\n longed_frac = shares_longed.divide(volumes)\n shorted_frac = shares_shorted.divide(volumes)\n grossed_frac = shares_grossed.divide(volumes)\n\n # NOTE: To work around a bug in `quantile` with nan-handling in\n # pandas 0.18, use np.nanpercentile by applying to each row of\n # the dataframe. This is fixed in pandas 0.19.\n #\n # longed_threshold = 100*longed_frac.quantile(percentile, axis='columns')\n # shorted_threshold = 100*shorted_frac.quantile(percentile, axis='columns')\n # grossed_threshold = 100*grossed_frac.quantile(percentile, axis='columns')\n\n longed_threshold = 100 * longed_frac.apply(\n partial(np.nanpercentile, q=100 * percentile),\n axis='columns',\n )\n shorted_threshold = 100 * shorted_frac.apply(\n partial(np.nanpercentile, q=100 * percentile),\n axis='columns',\n )\n grossed_threshold = 100 * grossed_frac.apply(\n partial(np.nanpercentile, q=100 * percentile),\n axis='columns',\n )\n\n return longed_threshold, shorted_threshold, grossed_threshold"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nplots the long and short exposures of a resource in a single line.", "response": "def plot_volume_exposures_longshort(longed_threshold, shorted_threshold,\n percentile, ax=None):\n \"\"\"\n Plots outputs of compute_volume_exposures as line graphs\n\n Parameters\n ----------\n longed_threshold, shorted_threshold : pd.Series\n Series of longed and shorted volume exposures (output of\n compute_volume_exposures).\n\n percentile : float\n Percentile to use when computing and plotting volume exposures.\n - See full explanation in create_risk_tear_sheet\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n ax.plot(longed_threshold.index, longed_threshold,\n color='b', label='long')\n ax.plot(shorted_threshold.index, shorted_threshold,\n color='r', label='short')\n ax.axhline(0, color='k')\n ax.set(title='Long and short exposures to illiquidity',\n ylabel='{}th percentile of proportion of volume (%)'\n .format(100 * percentile))\n ax.legend(frameon=True, framealpha=0.5)\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nplots the gross exposure of the grossed_threshold percentile of the volume.", "response": "def plot_volume_exposures_gross(grossed_threshold, percentile, ax=None):\n \"\"\"\n Plots outputs of compute_volume_exposures as line graphs\n\n Parameters\n ----------\n grossed_threshold : pd.Series\n Series of grossed volume exposures (output of\n compute_volume_exposures).\n\n percentile : float\n Percentile to use when computing and plotting volume exposures\n - See full explanation in create_risk_tear_sheet\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n ax.plot(grossed_threshold.index, grossed_threshold,\n color='b', label='gross')\n ax.axhline(0, color='k')\n ax.set(title='Gross exposure to illiquidity',\n ylabel='{}th percentile of \\n proportion of volume (%)'\n .format(100 * percentile))\n ax.legend(frameon=True, framealpha=0.5)\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_full_tear_sheet(returns,\n positions=None,\n transactions=None,\n market_data=None,\n benchmark_rets=None,\n slippage=None,\n live_start_date=None,\n sector_mappings=None,\n bayesian=False,\n round_trips=False,\n estimate_intraday='infer',\n hide_positions=False,\n cone_std=(1.0, 1.5, 2.0),\n bootstrap=False,\n unadjusted_returns=None,\n style_factor_panel=None,\n sectors=None,\n caps=None,\n shares_held=None,\n volumes=None,\n percentile=None,\n turnover_denom='AGB',\n set_context=True,\n factor_returns=None,\n factor_loadings=None,\n pos_in_dollars=True,\n header_rows=None,\n factor_partitions=FACTOR_PARTITIONS):\n \"\"\"\n Generate a number of tear sheets that are useful\n for analyzing a strategy's performance.\n\n - Fetches benchmarks if needed.\n - Creates tear sheets for returns, and significant events.\n If possible, also creates tear sheets for position analysis,\n transaction analysis, and Bayesian analysis.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - Time series with decimal returns.\n - Example:\n 2015-07-16 -0.012143\n 2015-07-17 0.045350\n 2015-07-20 0.030957\n 2015-07-21 0.004902\n positions : pd.DataFrame, optional\n Daily net position values.\n - Time series of dollar amount invested in each position and cash.\n - Days where stocks are not held can be represented by 0 or NaN.\n - Non-working capital is labelled 'cash'\n - Example:\n index 'AAPL' 'MSFT' cash\n 2004-01-09 13939.3800 -14012.9930 711.5585\n 2004-01-12 14492.6300 -14624.8700 27.1821\n 2004-01-13 -13853.2800 13653.6400 -43.6375\n transactions : pd.DataFrame, optional\n Executed trade volumes and fill prices.\n - One row per trade.\n - Trades on different names that occur at the\n same time will have identical indicies.\n - Example:\n index amount price symbol\n 2004-01-09 12:18:01 483 324.12 'AAPL'\n 2004-01-09 12:18:01 122 83.10 'MSFT'\n 2004-01-13 14:12:23 -75 340.43 'AAPL'\n market_data : pd.Panel, optional\n Panel with items axis of 'price' and 'volume' DataFrames.\n The major and minor axes should match those of the\n the passed positions DataFrame (same dates and symbols).\n slippage : int/float, optional\n Basis points of slippage to apply to returns before generating\n tearsheet stats and plots.\n If a value is provided, slippage parameter sweep\n plots will be generated from the unadjusted returns.\n Transactions and positions must also be passed.\n - See txn.adjust_returns_for_slippage for more details.\n live_start_date : datetime, optional\n The point in time when the strategy began live trading,\n after its backtest period. This datetime should be normalized.\n hide_positions : bool, optional\n If True, will not output any symbol names.\n bayesian: boolean, optional\n If True, causes the generation of a Bayesian tear sheet.\n round_trips: boolean, optional\n If True, causes the generation of a round trip tear sheet.\n sector_mappings : dict or pd.Series, optional\n Security identifier to sector mapping.\n Security ids as keys, sectors as values.\n estimate_intraday: boolean or str, optional\n Instead of using the end-of-day positions, use the point in the day\n where we have the most $ invested. This will adjust positions to\n better approximate and represent how an intraday strategy behaves.\n By default, this is 'infer', and an attempt will be made to detect\n an intraday strategy. Specifying this value will prevent detection.\n cone_std : float, or tuple, optional\n If float, The standard deviation to use for the cone plots.\n If tuple, Tuple of standard deviation values to use for the cone plots\n - The cone is a normal distribution with this standard deviation\n centered around a linear regression.\n bootstrap : boolean (optional)\n Whether to perform bootstrap analysis for the performance\n metrics. Takes a few minutes longer.\n turnover_denom : str\n Either AGB or portfolio_value, default AGB.\n - See full explanation in txn.get_turnover.\n factor_returns : pd.Dataframe, optional\n Returns by factor, with date as index and factors as columns\n factor_loadings : pd.Dataframe, optional\n Factor loadings for all days in the date range, with date and\n ticker as index, and factors as columns.\n pos_in_dollars : boolean, optional\n indicates whether positions is in dollars\n header_rows : dict or OrderedDict, optional\n Extra rows to display at the top of the perf stats table.\n set_context : boolean, optional\n If True, set default plotting style context.\n - See plotting.context().\n factor_partitions : dict, optional\n dict specifying how factors should be separated in perf attrib\n factor returns and risk exposures plots\n - See create_perf_attrib_tear_sheet().\n \"\"\"\n\n if (unadjusted_returns is None) and (slippage is not None) and\\\n (transactions is not None):\n unadjusted_returns = returns.copy()\n returns = txn.adjust_returns_for_slippage(returns, positions,\n transactions, slippage)\n\n positions = utils.check_intraday(estimate_intraday, returns,\n positions, transactions)\n\n create_returns_tear_sheet(\n returns,\n positions=positions,\n transactions=transactions,\n live_start_date=live_start_date,\n cone_std=cone_std,\n benchmark_rets=benchmark_rets,\n bootstrap=bootstrap,\n turnover_denom=turnover_denom,\n header_rows=header_rows,\n set_context=set_context)\n\n create_interesting_times_tear_sheet(returns,\n benchmark_rets=benchmark_rets,\n set_context=set_context)\n\n if positions is not None:\n create_position_tear_sheet(returns, positions,\n hide_positions=hide_positions,\n set_context=set_context,\n sector_mappings=sector_mappings,\n estimate_intraday=False)\n\n if transactions is not None:\n create_txn_tear_sheet(returns, positions, transactions,\n unadjusted_returns=unadjusted_returns,\n estimate_intraday=False,\n set_context=set_context)\n if round_trips:\n create_round_trip_tear_sheet(\n returns=returns,\n positions=positions,\n transactions=transactions,\n sector_mappings=sector_mappings,\n estimate_intraday=False)\n\n if market_data is not None:\n create_capacity_tear_sheet(returns, positions, transactions,\n market_data,\n liquidation_daily_vol_limit=0.2,\n last_n_days=125,\n estimate_intraday=False)\n\n if style_factor_panel is not None:\n create_risk_tear_sheet(positions, style_factor_panel, sectors,\n caps, shares_held, volumes, percentile)\n\n if factor_returns is not None and factor_loadings is not None:\n create_perf_attrib_tear_sheet(returns, positions, factor_returns,\n factor_loadings, transactions,\n pos_in_dollars=pos_in_dollars,\n factor_partitions=factor_partitions)\n\n if bayesian:\n create_bayesian_tear_sheet(returns,\n live_start_date=live_start_date,\n benchmark_rets=benchmark_rets,\n set_context=set_context)", "response": "Creates a full tear sheet for a given strategy."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_simple_tear_sheet(returns,\n positions=None,\n transactions=None,\n benchmark_rets=None,\n slippage=None,\n estimate_intraday='infer',\n live_start_date=None,\n turnover_denom='AGB',\n header_rows=None):\n \"\"\"\n Simpler version of create_full_tear_sheet; generates summary performance\n statistics and important plots as a single image.\n\n - Plots: cumulative returns, rolling beta, rolling Sharpe, underwater,\n exposure, top 10 holdings, total holdings, long/short holdings,\n daily turnover, transaction time distribution.\n - Never accept market_data input (market_data = None)\n - Never accept sector_mappings input (sector_mappings = None)\n - Never perform bootstrap analysis (bootstrap = False)\n - Never hide posistions on top 10 holdings plot (hide_positions = False)\n - Always use default cone_std (cone_std = (1.0, 1.5, 2.0))\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - Time series with decimal returns.\n - Example:\n 2015-07-16 -0.012143\n 2015-07-17 0.045350\n 2015-07-20 0.030957\n 2015-07-21 0.004902\n positions : pd.DataFrame, optional\n Daily net position values.\n - Time series of dollar amount invested in each position and cash.\n - Days where stocks are not held can be represented by 0 or NaN.\n - Non-working capital is labelled 'cash'\n - Example:\n index 'AAPL' 'MSFT' cash\n 2004-01-09 13939.3800 -14012.9930 711.5585\n 2004-01-12 14492.6300 -14624.8700 27.1821\n 2004-01-13 -13853.2800 13653.6400 -43.6375\n transactions : pd.DataFrame, optional\n Executed trade volumes and fill prices.\n - One row per trade.\n - Trades on different names that occur at the\n same time will have identical indicies.\n - Example:\n index amount price symbol\n 2004-01-09 12:18:01 483 324.12 'AAPL'\n 2004-01-09 12:18:01 122 83.10 'MSFT'\n 2004-01-13 14:12:23 -75 340.43 'AAPL'\n benchmark_rets : pd.Series, optional\n Daily returns of the benchmark, noncumulative.\n slippage : int/float, optional\n Basis points of slippage to apply to returns before generating\n tearsheet stats and plots.\n If a value is provided, slippage parameter sweep\n plots will be generated from the unadjusted returns.\n Transactions and positions must also be passed.\n - See txn.adjust_returns_for_slippage for more details.\n live_start_date : datetime, optional\n The point in time when the strategy began live trading,\n after its backtest period. This datetime should be normalized.\n turnover_denom : str, optional\n Either AGB or portfolio_value, default AGB.\n - See full explanation in txn.get_turnover.\n header_rows : dict or OrderedDict, optional\n Extra rows to display at the top of the perf stats table.\n set_context : boolean, optional\n If True, set default plotting style context.\n \"\"\"\n\n positions = utils.check_intraday(estimate_intraday, returns,\n positions, transactions)\n\n if (slippage is not None) and (transactions is not None):\n returns = txn.adjust_returns_for_slippage(returns, positions,\n transactions, slippage)\n\n always_sections = 4\n positions_sections = 4 if positions is not None else 0\n transactions_sections = 2 if transactions is not None else 0\n live_sections = 1 if live_start_date is not None else 0\n benchmark_sections = 1 if benchmark_rets is not None else 0\n\n vertical_sections = sum([\n always_sections,\n positions_sections,\n transactions_sections,\n live_sections,\n benchmark_sections,\n ])\n\n if live_start_date is not None:\n live_start_date = ep.utils.get_utc_timestamp(live_start_date)\n\n plotting.show_perf_stats(returns,\n benchmark_rets,\n positions=positions,\n transactions=transactions,\n turnover_denom=turnover_denom,\n live_start_date=live_start_date,\n header_rows=header_rows)\n\n fig = plt.figure(figsize=(14, vertical_sections * 6))\n gs = gridspec.GridSpec(vertical_sections, 3, wspace=0.5, hspace=0.5)\n\n ax_rolling_returns = plt.subplot(gs[:2, :])\n i = 2\n if benchmark_rets is not None:\n ax_rolling_beta = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n ax_rolling_sharpe = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n ax_underwater = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n\n plotting.plot_rolling_returns(returns,\n factor_returns=benchmark_rets,\n live_start_date=live_start_date,\n cone_std=(1.0, 1.5, 2.0),\n ax=ax_rolling_returns)\n ax_rolling_returns.set_title('Cumulative returns')\n\n if benchmark_rets is not None:\n plotting.plot_rolling_beta(returns, benchmark_rets, ax=ax_rolling_beta)\n\n plotting.plot_rolling_sharpe(returns, ax=ax_rolling_sharpe)\n\n plotting.plot_drawdown_underwater(returns, ax=ax_underwater)\n\n if positions is not None:\n # Plot simple positions tear sheet\n ax_exposures = plt.subplot(gs[i, :])\n i += 1\n ax_top_positions = plt.subplot(gs[i, :], sharex=ax_exposures)\n i += 1\n ax_holdings = plt.subplot(gs[i, :], sharex=ax_exposures)\n i += 1\n ax_long_short_holdings = plt.subplot(gs[i, :])\n i += 1\n\n positions_alloc = pos.get_percent_alloc(positions)\n\n plotting.plot_exposures(returns, positions, ax=ax_exposures)\n\n plotting.show_and_plot_top_positions(returns,\n positions_alloc,\n show_and_plot=0,\n hide_positions=False,\n ax=ax_top_positions)\n\n plotting.plot_holdings(returns, positions_alloc, ax=ax_holdings)\n\n plotting.plot_long_short_holdings(returns, positions_alloc,\n ax=ax_long_short_holdings)\n\n if transactions is not None:\n # Plot simple transactions tear sheet\n ax_turnover = plt.subplot(gs[i, :])\n i += 1\n ax_txn_timings = plt.subplot(gs[i, :])\n i += 1\n\n plotting.plot_turnover(returns,\n transactions,\n positions,\n ax=ax_turnover)\n\n plotting.plot_txn_time_hist(transactions, ax=ax_txn_timings)\n\n for ax in fig.axes:\n plt.setp(ax.get_xticklabels(), visible=True)", "response": "Create a simple tear sheet for the given return data."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_returns_tear_sheet(returns, positions=None,\n transactions=None,\n live_start_date=None,\n cone_std=(1.0, 1.5, 2.0),\n benchmark_rets=None,\n bootstrap=False,\n turnover_denom='AGB',\n header_rows=None,\n return_fig=False):\n \"\"\"\n Generate a number of plots for analyzing a strategy's returns.\n\n - Fetches benchmarks, then creates the plots on a single figure.\n - Plots: rolling returns (with cone), rolling beta, rolling sharpe,\n rolling Fama-French risk factors, drawdowns, underwater plot, monthly\n and annual return plots, daily similarity plots,\n and return quantile box plot.\n - Will also print the start and end dates of the strategy,\n performance statistics, drawdown periods, and the return range.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n positions : pd.DataFrame, optional\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame, optional\n Executed trade volumes and fill prices.\n - See full explanation in create_full_tear_sheet.\n live_start_date : datetime, optional\n The point in time when the strategy began live trading,\n after its backtest period.\n cone_std : float, or tuple, optional\n If float, The standard deviation to use for the cone plots.\n If tuple, Tuple of standard deviation values to use for the cone plots\n - The cone is a normal distribution with this standard deviation\n centered around a linear regression.\n benchmark_rets : pd.Series, optional\n Daily noncumulative returns of the benchmark.\n - This is in the same style as returns.\n bootstrap : boolean, optional\n Whether to perform bootstrap analysis for the performance\n metrics. Takes a few minutes longer.\n turnover_denom : str, optional\n Either AGB or portfolio_value, default AGB.\n - See full explanation in txn.get_turnover.\n header_rows : dict or OrderedDict, optional\n Extra rows to display at the top of the perf stats table.\n return_fig : boolean, optional\n If True, returns the figure that was plotted on.\n \"\"\"\n\n if benchmark_rets is not None:\n returns = utils.clip_returns_to_benchmark(returns, benchmark_rets)\n\n plotting.show_perf_stats(returns, benchmark_rets,\n positions=positions,\n transactions=transactions,\n turnover_denom=turnover_denom,\n bootstrap=bootstrap,\n live_start_date=live_start_date,\n header_rows=header_rows)\n\n plotting.show_worst_drawdown_periods(returns)\n\n vertical_sections = 11\n\n if live_start_date is not None:\n vertical_sections += 1\n live_start_date = ep.utils.get_utc_timestamp(live_start_date)\n\n if benchmark_rets is not None:\n vertical_sections += 1\n\n if bootstrap:\n vertical_sections += 1\n\n fig = plt.figure(figsize=(14, vertical_sections * 6))\n gs = gridspec.GridSpec(vertical_sections, 3, wspace=0.5, hspace=0.5)\n ax_rolling_returns = plt.subplot(gs[:2, :])\n\n i = 2\n ax_rolling_returns_vol_match = plt.subplot(gs[i, :],\n sharex=ax_rolling_returns)\n i += 1\n ax_rolling_returns_log = plt.subplot(gs[i, :],\n sharex=ax_rolling_returns)\n i += 1\n ax_returns = plt.subplot(gs[i, :],\n sharex=ax_rolling_returns)\n i += 1\n if benchmark_rets is not None:\n ax_rolling_beta = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n ax_rolling_volatility = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n ax_rolling_sharpe = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n ax_drawdown = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n ax_underwater = plt.subplot(gs[i, :], sharex=ax_rolling_returns)\n i += 1\n ax_monthly_heatmap = plt.subplot(gs[i, 0])\n ax_annual_returns = plt.subplot(gs[i, 1])\n ax_monthly_dist = plt.subplot(gs[i, 2])\n i += 1\n ax_return_quantiles = plt.subplot(gs[i, :])\n i += 1\n\n plotting.plot_rolling_returns(\n returns,\n factor_returns=benchmark_rets,\n live_start_date=live_start_date,\n cone_std=cone_std,\n ax=ax_rolling_returns)\n ax_rolling_returns.set_title(\n 'Cumulative returns')\n\n plotting.plot_rolling_returns(\n returns,\n factor_returns=benchmark_rets,\n live_start_date=live_start_date,\n cone_std=None,\n volatility_match=(benchmark_rets is not None),\n legend_loc=None,\n ax=ax_rolling_returns_vol_match)\n ax_rolling_returns_vol_match.set_title(\n 'Cumulative returns volatility matched to benchmark')\n\n plotting.plot_rolling_returns(\n returns,\n factor_returns=benchmark_rets,\n logy=True,\n live_start_date=live_start_date,\n cone_std=cone_std,\n ax=ax_rolling_returns_log)\n ax_rolling_returns_log.set_title(\n 'Cumulative returns on logarithmic scale')\n\n plotting.plot_returns(\n returns,\n live_start_date=live_start_date,\n ax=ax_returns,\n )\n ax_returns.set_title(\n 'Returns')\n\n if benchmark_rets is not None:\n plotting.plot_rolling_beta(\n returns, benchmark_rets, ax=ax_rolling_beta)\n\n plotting.plot_rolling_volatility(\n returns, factor_returns=benchmark_rets, ax=ax_rolling_volatility)\n\n plotting.plot_rolling_sharpe(\n returns, ax=ax_rolling_sharpe)\n\n # Drawdowns\n plotting.plot_drawdown_periods(\n returns, top=5, ax=ax_drawdown)\n\n plotting.plot_drawdown_underwater(\n returns=returns, ax=ax_underwater)\n\n plotting.plot_monthly_returns_heatmap(returns, ax=ax_monthly_heatmap)\n plotting.plot_annual_returns(returns, ax=ax_annual_returns)\n plotting.plot_monthly_returns_dist(returns, ax=ax_monthly_dist)\n\n plotting.plot_return_quantiles(\n returns,\n live_start_date=live_start_date,\n ax=ax_return_quantiles)\n\n if bootstrap and (benchmark_rets is not None):\n ax_bootstrap = plt.subplot(gs[i, :])\n plotting.plot_perf_stats(returns, benchmark_rets,\n ax=ax_bootstrap)\n elif bootstrap:\n raise ValueError('bootstrap requires passing of benchmark_rets.')\n\n for ax in fig.axes:\n plt.setp(ax.get_xticklabels(), visible=True)\n\n if return_fig:\n return fig", "response": "Creates a TEAR sheet that contains the number of plots for an analysis of a particular strategy s returns."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a tear sheet for an analyzing the positions and holdings of the avec strategy.", "response": "def create_position_tear_sheet(returns, positions,\n show_and_plot_top_pos=2, hide_positions=False,\n return_fig=False, sector_mappings=None,\n transactions=None, estimate_intraday='infer'):\n \"\"\"\n Generate a number of plots for analyzing a\n strategy's positions and holdings.\n\n - Plots: gross leverage, exposures, top positions, and holdings.\n - Will also print the top positions held.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n show_and_plot_top_pos : int, optional\n By default, this is 2, and both prints and plots the\n top 10 positions.\n If this is 0, it will only plot; if 1, it will only print.\n hide_positions : bool, optional\n If True, will not output any symbol names.\n Overrides show_and_plot_top_pos to 0 to suppress text output.\n return_fig : boolean, optional\n If True, returns the figure that was plotted on.\n sector_mappings : dict or pd.Series, optional\n Security identifier to sector mapping.\n Security ids as keys, sectors as values.\n transactions : pd.DataFrame, optional\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n estimate_intraday: boolean or str, optional\n Approximate returns for intraday strategies.\n See description in create_full_tear_sheet.\n \"\"\"\n\n positions = utils.check_intraday(estimate_intraday, returns,\n positions, transactions)\n\n if hide_positions:\n show_and_plot_top_pos = 0\n vertical_sections = 7 if sector_mappings is not None else 6\n\n fig = plt.figure(figsize=(14, vertical_sections * 6))\n gs = gridspec.GridSpec(vertical_sections, 3, wspace=0.5, hspace=0.5)\n ax_exposures = plt.subplot(gs[0, :])\n ax_top_positions = plt.subplot(gs[1, :], sharex=ax_exposures)\n ax_max_median_pos = plt.subplot(gs[2, :], sharex=ax_exposures)\n ax_holdings = plt.subplot(gs[3, :], sharex=ax_exposures)\n ax_long_short_holdings = plt.subplot(gs[4, :])\n ax_gross_leverage = plt.subplot(gs[5, :], sharex=ax_exposures)\n\n positions_alloc = pos.get_percent_alloc(positions)\n\n plotting.plot_exposures(returns, positions, ax=ax_exposures)\n\n plotting.show_and_plot_top_positions(\n returns,\n positions_alloc,\n show_and_plot=show_and_plot_top_pos,\n hide_positions=hide_positions,\n ax=ax_top_positions)\n\n plotting.plot_max_median_position_concentration(positions,\n ax=ax_max_median_pos)\n\n plotting.plot_holdings(returns, positions_alloc, ax=ax_holdings)\n\n plotting.plot_long_short_holdings(returns, positions_alloc,\n ax=ax_long_short_holdings)\n\n plotting.plot_gross_leverage(returns, positions,\n ax=ax_gross_leverage)\n\n if sector_mappings is not None:\n sector_exposures = pos.get_sector_exposures(positions,\n sector_mappings)\n if len(sector_exposures.columns) > 1:\n sector_alloc = pos.get_percent_alloc(sector_exposures)\n sector_alloc = sector_alloc.drop('cash', axis='columns')\n ax_sector_alloc = plt.subplot(gs[6, :], sharex=ax_exposures)\n plotting.plot_sector_allocations(returns, sector_alloc,\n ax=ax_sector_alloc)\n\n for ax in fig.axes:\n plt.setp(ax.get_xticklabels(), visible=True)\n\n if return_fig:\n return fig"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a tear sheet for an analyzing a strategy s transactions.", "response": "def create_txn_tear_sheet(returns, positions, transactions,\n unadjusted_returns=None, estimate_intraday='infer',\n return_fig=False):\n \"\"\"\n Generate a number of plots for analyzing a strategy's transactions.\n\n Plots: turnover, daily volume, and a histogram of daily volume.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n unadjusted_returns : pd.Series, optional\n Daily unadjusted returns of the strategy, noncumulative.\n Will plot additional swippage sweep analysis.\n - See pyfolio.plotting.plot_swippage_sleep and\n pyfolio.plotting.plot_slippage_sensitivity\n estimate_intraday: boolean or str, optional\n Approximate returns for intraday strategies.\n See description in create_full_tear_sheet.\n return_fig : boolean, optional\n If True, returns the figure that was plotted on.\n \"\"\"\n\n positions = utils.check_intraday(estimate_intraday, returns,\n positions, transactions)\n\n vertical_sections = 6 if unadjusted_returns is not None else 4\n\n fig = plt.figure(figsize=(14, vertical_sections * 6))\n gs = gridspec.GridSpec(vertical_sections, 3, wspace=0.5, hspace=0.5)\n ax_turnover = plt.subplot(gs[0, :])\n ax_daily_volume = plt.subplot(gs[1, :], sharex=ax_turnover)\n ax_turnover_hist = plt.subplot(gs[2, :])\n ax_txn_timings = plt.subplot(gs[3, :])\n\n plotting.plot_turnover(\n returns,\n transactions,\n positions,\n ax=ax_turnover)\n\n plotting.plot_daily_volume(returns, transactions, ax=ax_daily_volume)\n\n try:\n plotting.plot_daily_turnover_hist(transactions, positions,\n ax=ax_turnover_hist)\n except ValueError:\n warnings.warn('Unable to generate turnover plot.', UserWarning)\n\n plotting.plot_txn_time_hist(transactions, ax=ax_txn_timings)\n\n if unadjusted_returns is not None:\n ax_slippage_sweep = plt.subplot(gs[4, :])\n plotting.plot_slippage_sweep(unadjusted_returns,\n positions,\n transactions,\n ax=ax_slippage_sweep\n )\n ax_slippage_sensitivity = plt.subplot(gs[5, :])\n plotting.plot_slippage_sensitivity(unadjusted_returns,\n positions,\n transactions,\n ax=ax_slippage_sensitivity\n )\n for ax in fig.axes:\n plt.setp(ax.get_xticklabels(), visible=True)\n\n if return_fig:\n return fig"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_round_trip_tear_sheet(returns, positions, transactions,\n sector_mappings=None,\n estimate_intraday='infer', return_fig=False):\n \"\"\"\n Generate a number of figures and plots describing the duration,\n frequency, and profitability of trade \"round trips.\"\n A round trip is started when a new long or short position is\n opened and is only completed when the number of shares in that\n position returns to or crosses zero.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n sector_mappings : dict or pd.Series, optional\n Security identifier to sector mapping.\n Security ids as keys, sectors as values.\n estimate_intraday: boolean or str, optional\n Approximate returns for intraday strategies.\n See description in create_full_tear_sheet.\n return_fig : boolean, optional\n If True, returns the figure that was plotted on.\n \"\"\"\n\n positions = utils.check_intraday(estimate_intraday, returns,\n positions, transactions)\n\n transactions_closed = round_trips.add_closing_transactions(positions,\n transactions)\n # extract_round_trips requires BoD portfolio_value\n trades = round_trips.extract_round_trips(\n transactions_closed,\n portfolio_value=positions.sum(axis='columns') / (1 + returns)\n )\n\n if len(trades) < 5:\n warnings.warn(\n \"\"\"Fewer than 5 round-trip trades made.\n Skipping round trip tearsheet.\"\"\", UserWarning)\n return\n\n round_trips.print_round_trip_stats(trades)\n\n plotting.show_profit_attribution(trades)\n\n if sector_mappings is not None:\n sector_trades = round_trips.apply_sector_mappings_to_round_trips(\n trades, sector_mappings)\n plotting.show_profit_attribution(sector_trades)\n\n fig = plt.figure(figsize=(14, 3 * 6))\n\n gs = gridspec.GridSpec(3, 2, wspace=0.5, hspace=0.5)\n\n ax_trade_lifetimes = plt.subplot(gs[0, :])\n ax_prob_profit_trade = plt.subplot(gs[1, 0])\n ax_holding_time = plt.subplot(gs[1, 1])\n ax_pnl_per_round_trip_dollars = plt.subplot(gs[2, 0])\n ax_pnl_per_round_trip_pct = plt.subplot(gs[2, 1])\n\n plotting.plot_round_trip_lifetimes(trades, ax=ax_trade_lifetimes)\n\n plotting.plot_prob_profit_trade(trades, ax=ax_prob_profit_trade)\n\n trade_holding_times = [x.days for x in trades['duration']]\n sns.distplot(trade_holding_times, kde=False, ax=ax_holding_time)\n ax_holding_time.set(xlabel='Holding time in days')\n\n sns.distplot(trades.pnl, kde=False, ax=ax_pnl_per_round_trip_dollars)\n ax_pnl_per_round_trip_dollars.set(xlabel='PnL per round-trip trade in $')\n\n sns.distplot(trades.returns.dropna() * 100, kde=False,\n ax=ax_pnl_per_round_trip_pct)\n ax_pnl_per_round_trip_pct.set(\n xlabel='Round-trip returns in %')\n\n gs.tight_layout(fig)\n\n if return_fig:\n return fig", "response": "Create a round trip tear sheet."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_interesting_times_tear_sheet(\n returns, benchmark_rets=None, legend_loc='best', return_fig=False):\n \"\"\"\n Generate a number of returns plots around interesting points in time,\n like the flash crash and 9/11.\n\n Plots: returns around the dotcom bubble burst, Lehmann Brothers' failure,\n 9/11, US downgrade and EU debt crisis, Fukushima meltdown, US housing\n bubble burst, EZB IR, Great Recession (August 2007, March and September\n of 2008, Q1 & Q2 2009), flash crash, April and October 2014.\n\n benchmark_rets must be passed, as it is meaningless to analyze performance\n during interesting times without some benchmark to refer to.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n benchmark_rets : pd.Series\n Daily noncumulative returns of the benchmark.\n - This is in the same style as returns.\n legend_loc : plt.legend_loc, optional\n The legend's location.\n return_fig : boolean, optional\n If True, returns the figure that was plotted on.\n \"\"\"\n\n rets_interesting = timeseries.extract_interesting_date_ranges(returns)\n\n if not rets_interesting:\n warnings.warn('Passed returns do not overlap with any'\n 'interesting times.', UserWarning)\n return\n\n utils.print_table(pd.DataFrame(rets_interesting)\n .describe().transpose()\n .loc[:, ['mean', 'min', 'max']] * 100,\n name='Stress Events',\n float_format='{0:.2f}%'.format)\n\n if benchmark_rets is not None:\n returns = utils.clip_returns_to_benchmark(returns, benchmark_rets)\n\n bmark_interesting = timeseries.extract_interesting_date_ranges(\n benchmark_rets)\n\n num_plots = len(rets_interesting)\n # 2 plots, 1 row; 3 plots, 2 rows; 4 plots, 2 rows; etc.\n num_rows = int((num_plots + 1) / 2.0)\n fig = plt.figure(figsize=(14, num_rows * 6.0))\n gs = gridspec.GridSpec(num_rows, 2, wspace=0.5, hspace=0.5)\n\n for i, (name, rets_period) in enumerate(rets_interesting.items()):\n # i=0 -> 0, i=1 -> 0, i=2 -> 1 ;; i=0 -> 0, i=1 -> 1, i=2 -> 0\n ax = plt.subplot(gs[int(i / 2.0), i % 2])\n\n ep.cum_returns(rets_period).plot(\n ax=ax, color='forestgreen', label='algo', alpha=0.7, lw=2)\n\n if benchmark_rets is not None:\n ep.cum_returns(bmark_interesting[name]).plot(\n ax=ax, color='gray', label='benchmark', alpha=0.6)\n ax.legend(['Algo',\n 'benchmark'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n else:\n ax.legend(['Algo'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n\n ax.set_title(name)\n ax.set_ylabel('Returns')\n ax.set_xlabel('')\n\n if return_fig:\n return fig", "response": "Create a tear sheet that shows how interesting times are plotted."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a tear sheet that contains the capacity sweeps for the given order of positions and transactions.", "response": "def create_capacity_tear_sheet(returns, positions, transactions,\n market_data,\n liquidation_daily_vol_limit=0.2,\n trade_daily_vol_limit=0.05,\n last_n_days=utils.APPROX_BDAYS_PER_MONTH * 6,\n days_to_liquidate_limit=1,\n estimate_intraday='infer'):\n \"\"\"\n Generates a report detailing portfolio size constraints set by\n least liquid tickers. Plots a \"capacity sweep,\" a curve describing\n projected sharpe ratio given the slippage penalties that are\n applied at various capital bases.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n market_data : pd.Panel\n Panel with items axis of 'price' and 'volume' DataFrames.\n The major and minor axes should match those of the\n the passed positions DataFrame (same dates and symbols).\n liquidation_daily_vol_limit : float\n Max proportion of a daily bar that can be consumed in the\n process of liquidating a position in the\n \"days to liquidation\" analysis.\n trade_daily_vol_limit : float\n Flag daily transaction totals that exceed proportion of\n daily bar.\n last_n_days : integer\n Compute max position allocation and dollar volume for only\n the last N days of the backtest\n days_to_liquidate_limit : integer\n Display all tickers with greater max days to liquidation.\n estimate_intraday: boolean or str, optional\n Approximate returns for intraday strategies.\n See description in create_full_tear_sheet.\n \"\"\"\n\n positions = utils.check_intraday(estimate_intraday, returns,\n positions, transactions)\n\n print(\"Max days to liquidation is computed for each traded name \"\n \"assuming a 20% limit on daily bar consumption \\n\"\n \"and trailing 5 day mean volume as the available bar volume.\\n\\n\"\n \"Tickers with >1 day liquidation time at a\"\n \" constant $1m capital base:\")\n\n max_days_by_ticker = capacity.get_max_days_to_liquidate_by_ticker(\n positions, market_data,\n max_bar_consumption=liquidation_daily_vol_limit,\n capital_base=1e6,\n mean_volume_window=5)\n max_days_by_ticker.index = (\n max_days_by_ticker.index.map(utils.format_asset))\n\n print(\"Whole backtest:\")\n utils.print_table(\n max_days_by_ticker[max_days_by_ticker.days_to_liquidate >\n days_to_liquidate_limit])\n\n max_days_by_ticker_lnd = capacity.get_max_days_to_liquidate_by_ticker(\n positions, market_data,\n max_bar_consumption=liquidation_daily_vol_limit,\n capital_base=1e6,\n mean_volume_window=5,\n last_n_days=last_n_days)\n max_days_by_ticker_lnd.index = (\n max_days_by_ticker_lnd.index.map(utils.format_asset))\n\n print(\"Last {} trading days:\".format(last_n_days))\n utils.print_table(\n max_days_by_ticker_lnd[max_days_by_ticker_lnd.days_to_liquidate > 1])\n\n llt = capacity.get_low_liquidity_transactions(transactions, market_data)\n llt.index = llt.index.map(utils.format_asset)\n\n print('Tickers with daily transactions consuming >{}% of daily bar \\n'\n 'all backtest:'.format(trade_daily_vol_limit * 100))\n utils.print_table(\n llt[llt['max_pct_bar_consumed'] > trade_daily_vol_limit * 100])\n\n llt = capacity.get_low_liquidity_transactions(\n transactions, market_data, last_n_days=last_n_days)\n\n print(\"Last {} trading days:\".format(last_n_days))\n utils.print_table(\n llt[llt['max_pct_bar_consumed'] > trade_daily_vol_limit * 100])\n\n bt_starting_capital = positions.iloc[0].sum() / (1 + returns.iloc[0])\n fig, ax_capacity_sweep = plt.subplots(figsize=(14, 6))\n plotting.plot_capacity_sweep(returns, transactions, market_data,\n bt_starting_capital,\n min_pv=100000,\n max_pv=300000000,\n step_size=1000000,\n ax=ax_capacity_sweep)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_bayesian_tear_sheet(returns, benchmark_rets=None,\n live_start_date=None, samples=2000,\n return_fig=False, stoch_vol=False,\n progressbar=True):\n \"\"\"\n Generate a number of Bayesian distributions and a Bayesian\n cone plot of returns.\n\n Plots: Sharpe distribution, annual volatility distribution,\n annual alpha distribution, beta distribution, predicted 1 and 5\n day returns distributions, and a cumulative returns cone plot.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n benchmark_rets : pd.Series, optional\n Daily noncumulative returns of the benchmark.\n - This is in the same style as returns.\n live_start_date : datetime, optional\n The point in time when the strategy began live\n trading, after its backtest period.\n samples : int, optional\n Number of posterior samples to draw.\n return_fig : boolean, optional\n If True, returns the figure that was plotted on.\n stoch_vol : boolean, optional\n If True, run and plot the stochastic volatility model\n progressbar : boolean, optional\n If True, show a progress bar\n \"\"\"\n\n if not have_bayesian:\n raise NotImplementedError(\n \"Bayesian tear sheet requirements not found.\\n\"\n \"Run 'pip install pyfolio[bayesian]' to install \"\n \"bayesian requirements.\"\n )\n\n if live_start_date is None:\n raise NotImplementedError(\n 'Bayesian tear sheet requires setting of live_start_date'\n )\n\n live_start_date = ep.utils.get_utc_timestamp(live_start_date)\n df_train = returns.loc[returns.index < live_start_date]\n df_test = returns.loc[returns.index >= live_start_date]\n\n # Run T model with missing data\n print(\"Running T model\")\n previous_time = time()\n # track the total run time of the Bayesian tear sheet\n start_time = previous_time\n\n trace_t, ppc_t = bayesian.run_model('t', df_train,\n returns_test=df_test,\n samples=samples, ppc=True,\n progressbar=progressbar)\n previous_time = timer(\"T model\", previous_time)\n\n # Compute BEST model\n print(\"\\nRunning BEST model\")\n trace_best = bayesian.run_model('best', df_train,\n returns_test=df_test,\n samples=samples,\n progressbar=progressbar)\n previous_time = timer(\"BEST model\", previous_time)\n\n # Plot results\n\n fig = plt.figure(figsize=(14, 10 * 2))\n gs = gridspec.GridSpec(9, 2, wspace=0.3, hspace=0.3)\n\n axs = []\n row = 0\n\n # Plot Bayesian cone\n ax_cone = plt.subplot(gs[row, :])\n bayesian.plot_bayes_cone(df_train, df_test, ppc_t, ax=ax_cone)\n previous_time = timer(\"plotting Bayesian cone\", previous_time)\n\n # Plot BEST results\n row += 1\n axs.append(plt.subplot(gs[row, 0]))\n axs.append(plt.subplot(gs[row, 1]))\n row += 1\n axs.append(plt.subplot(gs[row, 0]))\n axs.append(plt.subplot(gs[row, 1]))\n row += 1\n axs.append(plt.subplot(gs[row, 0]))\n axs.append(plt.subplot(gs[row, 1]))\n row += 1\n # Effect size across two\n axs.append(plt.subplot(gs[row, :]))\n\n bayesian.plot_best(trace=trace_best, axs=axs)\n previous_time = timer(\"plotting BEST results\", previous_time)\n\n # Compute Bayesian predictions\n row += 1\n ax_ret_pred_day = plt.subplot(gs[row, 0])\n ax_ret_pred_week = plt.subplot(gs[row, 1])\n day_pred = ppc_t[:, 0]\n p5 = scipy.stats.scoreatpercentile(day_pred, 5)\n sns.distplot(day_pred,\n ax=ax_ret_pred_day\n )\n ax_ret_pred_day.axvline(p5, linestyle='--', linewidth=3.)\n ax_ret_pred_day.set_xlabel('Predicted returns 1 day')\n ax_ret_pred_day.set_ylabel('Frequency')\n ax_ret_pred_day.text(0.4, 0.9, 'Bayesian VaR = %.2f' % p5,\n verticalalignment='bottom',\n horizontalalignment='right',\n transform=ax_ret_pred_day.transAxes)\n previous_time = timer(\"computing Bayesian predictions\", previous_time)\n\n # Plot Bayesian VaRs\n week_pred = (\n np.cumprod(ppc_t[:, :5] + 1, 1) - 1)[:, -1]\n p5 = scipy.stats.scoreatpercentile(week_pred, 5)\n sns.distplot(week_pred,\n ax=ax_ret_pred_week\n )\n ax_ret_pred_week.axvline(p5, linestyle='--', linewidth=3.)\n ax_ret_pred_week.set_xlabel('Predicted cum returns 5 days')\n ax_ret_pred_week.set_ylabel('Frequency')\n ax_ret_pred_week.text(0.4, 0.9, 'Bayesian VaR = %.2f' % p5,\n verticalalignment='bottom',\n horizontalalignment='right',\n transform=ax_ret_pred_week.transAxes)\n previous_time = timer(\"plotting Bayesian VaRs estimate\", previous_time)\n\n # Run alpha beta model\n if benchmark_rets is not None:\n print(\"\\nRunning alpha beta model\")\n benchmark_rets = benchmark_rets.loc[df_train.index]\n trace_alpha_beta = bayesian.run_model('alpha_beta', df_train,\n bmark=benchmark_rets,\n samples=samples,\n progressbar=progressbar)\n previous_time = timer(\"running alpha beta model\", previous_time)\n\n # Plot alpha and beta\n row += 1\n ax_alpha = plt.subplot(gs[row, 0])\n ax_beta = plt.subplot(gs[row, 1])\n sns.distplot((1 + trace_alpha_beta['alpha'][100:])**252 - 1,\n ax=ax_alpha)\n sns.distplot(trace_alpha_beta['beta'][100:], ax=ax_beta)\n ax_alpha.set_xlabel('Annual Alpha')\n ax_alpha.set_ylabel('Belief')\n ax_beta.set_xlabel('Beta')\n ax_beta.set_ylabel('Belief')\n previous_time = timer(\"plotting alpha beta model\", previous_time)\n\n if stoch_vol:\n # run stochastic volatility model\n returns_cutoff = 400\n print(\n \"\\nRunning stochastic volatility model on \"\n \"most recent {} days of returns.\".format(returns_cutoff)\n )\n if df_train.size > returns_cutoff:\n df_train_truncated = df_train[-returns_cutoff:]\n _, trace_stoch_vol = bayesian.model_stoch_vol(df_train_truncated)\n previous_time = timer(\n \"running stochastic volatility model\", previous_time)\n\n # plot latent volatility\n row += 1\n ax_volatility = plt.subplot(gs[row, :])\n bayesian.plot_stoch_vol(\n df_train_truncated, trace=trace_stoch_vol, ax=ax_volatility)\n previous_time = timer(\n \"plotting stochastic volatility model\", previous_time)\n\n total_time = time() - start_time\n print(\"\\nTotal runtime was {:.2f} seconds.\".format(total_time))\n\n gs.tight_layout(fig)\n\n if return_fig:\n return fig", "response": "Create a Bayesian tear sheet for a given set of returns."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_risk_tear_sheet(positions,\n style_factor_panel=None,\n sectors=None,\n caps=None,\n shares_held=None,\n volumes=None,\n percentile=None,\n returns=None,\n transactions=None,\n estimate_intraday='infer',\n return_fig=False):\n '''\n Creates risk tear sheet: computes and plots style factor exposures, sector\n exposures, market cap exposures and volume exposures.\n\n Parameters\n ----------\n positions : pd.DataFrame\n Daily equity positions of algorithm, in dollars.\n - DataFrame with dates as index, equities as columns\n - Last column is cash held\n - Example:\n Equity(24 Equity(62\n [AAPL]) [ABT]) cash\n 2017-04-03\t-108062.40 \t 4401.540 2.247757e+07\n 2017-04-04\t-108852.00\t 4373.820 2.540999e+07\n 2017-04-05\t-119968.66\t 4336.200 2.839812e+07\n\n style_factor_panel : pd.Panel\n Panel where each item is a DataFrame that tabulates style factor per\n equity per day.\n - Each item has dates as index, equities as columns\n - Example item:\n Equity(24 Equity(62\n [AAPL]) [ABT])\n 2017-04-03\t -0.51284 1.39173\n 2017-04-04\t -0.73381 0.98149\n 2017-04-05\t -0.90132\t 1.13981\n\n sectors : pd.DataFrame\n Daily Morningstar sector code per asset\n - DataFrame with dates as index and equities as columns\n - Example:\n Equity(24 Equity(62\n [AAPL]) [ABT])\n 2017-04-03\t 311.0 206.0\n 2017-04-04\t 311.0 206.0\n 2017-04-05\t 311.0\t 206.0\n\n caps : pd.DataFrame\n Daily market cap per asset\n - DataFrame with dates as index and equities as columns\n - Example:\n Equity(24 Equity(62\n [AAPL]) [ABT])\n 2017-04-03 1.327160e+10 6.402460e+10\n 2017-04-04\t 1.329620e+10 6.403694e+10\n 2017-04-05\t 1.297464e+10\t 6.397187e+10\n\n shares_held : pd.DataFrame\n Daily number of shares held by an algorithm.\n - Example:\n Equity(24 Equity(62\n [AAPL]) [ABT])\n 2017-04-03 1915 -2595\n 2017-04-04\t 1968 -3272\n 2017-04-05\t 2104 -3917\n\n volumes : pd.DataFrame\n Daily volume per asset\n - DataFrame with dates as index and equities as columns\n - Example:\n Equity(24 Equity(62\n [AAPL]) [ABT])\n 2017-04-03 34940859.00 4665573.80\n 2017-04-04\t 35603329.10 4818463.90\n 2017-04-05\t 41846731.75\t 4129153.10\n\n percentile : float\n Percentile to use when computing and plotting volume exposures.\n - Defaults to 10th percentile\n '''\n\n positions = utils.check_intraday(estimate_intraday, returns,\n positions, transactions)\n\n idx = positions.index & style_factor_panel.iloc[0].index & sectors.index \\\n & caps.index & shares_held.index & volumes.index\n positions = positions.loc[idx]\n\n vertical_sections = 0\n if style_factor_panel is not None:\n vertical_sections += len(style_factor_panel.items)\n new_style_dict = {}\n for item in style_factor_panel.items:\n new_style_dict.update({item:\n style_factor_panel.loc[item].loc[idx]})\n style_factor_panel = pd.Panel()\n style_factor_panel = style_factor_panel.from_dict(new_style_dict)\n if sectors is not None:\n vertical_sections += 4\n sectors = sectors.loc[idx]\n if caps is not None:\n vertical_sections += 4\n caps = caps.loc[idx]\n if (shares_held is not None) & (volumes is not None) \\\n & (percentile is not None):\n vertical_sections += 3\n shares_held = shares_held.loc[idx]\n volumes = volumes.loc[idx]\n\n if percentile is None:\n percentile = 0.1\n\n fig = plt.figure(figsize=[14, vertical_sections * 6])\n gs = gridspec.GridSpec(vertical_sections, 3, wspace=0.5, hspace=0.5)\n\n if style_factor_panel is not None:\n style_axes = []\n style_axes.append(plt.subplot(gs[0, :]))\n for i in range(1, len(style_factor_panel.items)):\n style_axes.append(plt.subplot(gs[i, :], sharex=style_axes[0]))\n\n j = 0\n for name, df in style_factor_panel.iteritems():\n sfe = risk.compute_style_factor_exposures(positions, df)\n risk.plot_style_factor_exposures(sfe, name, style_axes[j])\n j += 1\n\n if sectors is not None:\n i += 1\n ax_sector_longshort = plt.subplot(gs[i:i+2, :], sharex=style_axes[0])\n i += 2\n ax_sector_gross = plt.subplot(gs[i, :], sharex=style_axes[0])\n i += 1\n ax_sector_net = plt.subplot(gs[i, :], sharex=style_axes[0])\n long_exposures, short_exposures, gross_exposures, net_exposures \\\n = risk.compute_sector_exposures(positions, sectors)\n risk.plot_sector_exposures_longshort(long_exposures, short_exposures,\n ax=ax_sector_longshort)\n risk.plot_sector_exposures_gross(gross_exposures, ax=ax_sector_gross)\n risk.plot_sector_exposures_net(net_exposures, ax=ax_sector_net)\n\n if caps is not None:\n i += 1\n ax_cap_longshort = plt.subplot(gs[i:i+2, :], sharex=style_axes[0])\n i += 2\n ax_cap_gross = plt.subplot(gs[i, :], sharex=style_axes[0])\n i += 1\n ax_cap_net = plt.subplot(gs[i, :], sharex=style_axes[0])\n long_exposures, short_exposures, gross_exposures, net_exposures \\\n = risk.compute_cap_exposures(positions, caps)\n risk.plot_cap_exposures_longshort(long_exposures, short_exposures,\n ax_cap_longshort)\n risk.plot_cap_exposures_gross(gross_exposures, ax_cap_gross)\n risk.plot_cap_exposures_net(net_exposures, ax_cap_net)\n\n if volumes is not None:\n i += 1\n ax_vol_longshort = plt.subplot(gs[i:i+2, :], sharex=style_axes[0])\n i += 2\n ax_vol_gross = plt.subplot(gs[i, :], sharex=style_axes[0])\n longed_threshold, shorted_threshold, grossed_threshold \\\n = risk.compute_volume_exposures(positions, volumes, percentile)\n risk.plot_volume_exposures_longshort(longed_threshold,\n shorted_threshold, percentile,\n ax_vol_longshort)\n risk.plot_volume_exposures_gross(grossed_threshold, percentile,\n ax_vol_gross)\n\n for ax in fig.axes:\n plt.setp(ax.get_xticklabels(), visible=True)\n\n if return_fig:\n return fig", "response": "Creates a risk tear sheet for the given set of positions sectors caps shares_held volumes and returns."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_perf_attrib_tear_sheet(returns,\n positions,\n factor_returns,\n factor_loadings,\n transactions=None,\n pos_in_dollars=True,\n return_fig=False,\n factor_partitions=FACTOR_PARTITIONS):\n \"\"\"\n Generate plots and tables for analyzing a strategy's performance.\n\n Parameters\n ----------\n returns : pd.Series\n Returns for each day in the date range.\n\n positions: pd.DataFrame\n Daily holdings (in dollars or percentages), indexed by date.\n Will be converted to percentages if positions are in dollars.\n Short positions show up as cash in the 'cash' column.\n\n factor_returns : pd.DataFrame\n Returns by factor, with date as index and factors as columns\n\n factor_loadings : pd.DataFrame\n Factor loadings for all days in the date range, with date\n and ticker as index, and factors as columns.\n\n transactions : pd.DataFrame, optional\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n - Default is None.\n\n pos_in_dollars : boolean, optional\n Flag indicating whether `positions` are in dollars or percentages\n If True, positions are in dollars.\n\n return_fig : boolean, optional\n If True, returns the figure that was plotted on.\n\n factor_partitions : dict\n dict specifying how factors should be separated in factor returns\n and risk exposures plots\n - Example:\n {'style': ['momentum', 'size', 'value', ...],\n 'sector': ['technology', 'materials', ... ]}\n \"\"\"\n portfolio_exposures, perf_attrib_data = perf_attrib.perf_attrib(\n returns, positions, factor_returns, factor_loadings, transactions,\n pos_in_dollars=pos_in_dollars\n )\n\n display(Markdown(\"## Performance Relative to Common Risk Factors\"))\n\n # aggregate perf attrib stats and show summary table\n perf_attrib.show_perf_attrib_stats(returns, positions, factor_returns,\n factor_loadings, transactions,\n pos_in_dollars)\n\n # one section for the returns plot, and for each factor grouping\n # one section for factor returns, and one for risk exposures\n vertical_sections = 1 + 2 * max(len(factor_partitions), 1)\n current_section = 0\n\n fig = plt.figure(figsize=[14, vertical_sections * 6])\n\n gs = gridspec.GridSpec(vertical_sections, 1,\n wspace=0.5, hspace=0.5)\n\n perf_attrib.plot_returns(perf_attrib_data,\n ax=plt.subplot(gs[current_section]))\n current_section += 1\n\n if factor_partitions is not None:\n\n for factor_type, partitions in factor_partitions.iteritems():\n\n columns_to_select = perf_attrib_data.columns.intersection(\n partitions\n )\n\n perf_attrib.plot_factor_contribution_to_perf(\n perf_attrib_data[columns_to_select],\n ax=plt.subplot(gs[current_section]),\n title=(\n 'Cumulative common {} returns attribution'\n ).format(factor_type)\n )\n current_section += 1\n\n for factor_type, partitions in factor_partitions.iteritems():\n\n perf_attrib.plot_risk_exposures(\n portfolio_exposures[portfolio_exposures.columns\n .intersection(partitions)],\n ax=plt.subplot(gs[current_section]),\n title='Daily {} factor exposures'.format(factor_type)\n )\n current_section += 1\n\n else:\n\n perf_attrib.plot_factor_contribution_to_perf(\n perf_attrib_data,\n ax=plt.subplot(gs[current_section])\n )\n current_section += 1\n\n perf_attrib.plot_risk_exposures(\n portfolio_exposures,\n ax=plt.subplot(gs[current_section])\n )\n\n gs.tight_layout(fig)\n\n if return_fig:\n return fig", "response": "Generates a TEAR sheet that displays the performance of a strategy s performance."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntake a dataframe of transactions and returns a DataFrame containing the daily prices and volume for each day - ticker combination.", "response": "def daily_txns_with_bar_data(transactions, market_data):\n \"\"\"\n Sums the absolute value of shares traded in each name on each day.\n Adds columns containing the closing price and total daily volume for\n each day-ticker combination.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet\n market_data : pd.Panel\n Contains \"volume\" and \"price\" DataFrames for the tickers\n in the passed positions DataFrames\n\n Returns\n -------\n txn_daily : pd.DataFrame\n Daily totals for transacted shares in each traded name.\n price and volume columns for close price and daily volume for\n the corresponding ticker, respectively.\n \"\"\"\n\n transactions.index.name = 'date'\n txn_daily = pd.DataFrame(transactions.assign(\n amount=abs(transactions.amount)).groupby(\n ['symbol', pd.TimeGrouper('D')]).sum()['amount'])\n txn_daily['price'] = market_data['price'].unstack()\n txn_daily['volume'] = market_data['volume'].unstack()\n\n txn_daily = txn_daily.reset_index().set_index('date')\n\n return txn_daily"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncompute the number of days that would have been required to fully liquidate each position on each day based on the trailing n day mean daily bar volume and a limit on the proportion of a daily bar that we are allowed to consume. This analysis uses portfolio allocations and a provided capital base rather than the dollar values in the positions DataFrame to remove the effect of compounding on days to liquidate. In other words, this function assumes that the net liquidation portfolio value will always remain constant at capital_base. Parameters ---------- positions: pd.DataFrame Contains daily position values including cash - See full explanation in tears.create_full_tear_sheet market_data : pd.Panel Panel with items axis of 'price' and 'volume' DataFrames. The major and minor axes should match those of the the passed positions DataFrame (same dates and symbols). max_bar_consumption : float Max proportion of a daily bar that can be consumed in the process of liquidating a position. capital_base : integer Capital base multiplied by portfolio allocation to compute position value that needs liquidating. mean_volume_window : float Trailing window to use in mean volume calculation. Returns ------- days_to_liquidate : pd.DataFrame Number of days required to fully liquidate daily positions. Datetime index, symbols as columns.", "response": "def days_to_liquidate_positions(positions, market_data,\n max_bar_consumption=0.2,\n capital_base=1e6,\n mean_volume_window=5):\n \"\"\"\n Compute the number of days that would have been required\n to fully liquidate each position on each day based on the\n trailing n day mean daily bar volume and a limit on the proportion\n of a daily bar that we are allowed to consume.\n\n This analysis uses portfolio allocations and a provided capital base\n rather than the dollar values in the positions DataFrame to remove the\n effect of compounding on days to liquidate. In other words, this function\n assumes that the net liquidation portfolio value will always remain\n constant at capital_base.\n\n Parameters\n ----------\n positions: pd.DataFrame\n Contains daily position values including cash\n - See full explanation in tears.create_full_tear_sheet\n market_data : pd.Panel\n Panel with items axis of 'price' and 'volume' DataFrames.\n The major and minor axes should match those of the\n the passed positions DataFrame (same dates and symbols).\n max_bar_consumption : float\n Max proportion of a daily bar that can be consumed in the\n process of liquidating a position.\n capital_base : integer\n Capital base multiplied by portfolio allocation to compute\n position value that needs liquidating.\n mean_volume_window : float\n Trailing window to use in mean volume calculation.\n\n Returns\n -------\n days_to_liquidate : pd.DataFrame\n Number of days required to fully liquidate daily positions.\n Datetime index, symbols as columns.\n \"\"\"\n\n DV = market_data['volume'] * market_data['price']\n roll_mean_dv = DV.rolling(window=mean_volume_window,\n center=False).mean().shift()\n roll_mean_dv = roll_mean_dv.replace(0, np.nan)\n\n positions_alloc = pos.get_percent_alloc(positions)\n positions_alloc = positions_alloc.drop('cash', axis=1)\n\n days_to_liquidate = (positions_alloc * capital_base) / \\\n (max_bar_consumption * roll_mean_dv)\n\n return days_to_liquidate.iloc[mean_volume_window:]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_max_days_to_liquidate_by_ticker(positions, market_data,\n max_bar_consumption=0.2,\n capital_base=1e6,\n mean_volume_window=5,\n last_n_days=None):\n \"\"\"\n Finds the longest estimated liquidation time for each traded\n name over the course of backtest (or last n days of the backtest).\n\n Parameters\n ----------\n positions: pd.DataFrame\n Contains daily position values including cash\n - See full explanation in tears.create_full_tear_sheet\n market_data : pd.Panel\n Panel with items axis of 'price' and 'volume' DataFrames.\n The major and minor axes should match those of the\n the passed positions DataFrame (same dates and symbols).\n max_bar_consumption : float\n Max proportion of a daily bar that can be consumed in the\n process of liquidating a position.\n capital_base : integer\n Capital base multiplied by portfolio allocation to compute\n position value that needs liquidating.\n mean_volume_window : float\n Trailing window to use in mean volume calculation.\n last_n_days : integer\n Compute for only the last n days of the passed backtest data.\n\n Returns\n -------\n days_to_liquidate : pd.DataFrame\n Max Number of days required to fully liquidate each traded name.\n Index of symbols. Columns for days_to_liquidate and the corresponding\n date and position_alloc on that day.\n \"\"\"\n\n dtlp = days_to_liquidate_positions(positions, market_data,\n max_bar_consumption=max_bar_consumption,\n capital_base=capital_base,\n mean_volume_window=mean_volume_window)\n\n if last_n_days is not None:\n dtlp = dtlp.loc[dtlp.index.max() - pd.Timedelta(days=last_n_days):]\n\n pos_alloc = pos.get_percent_alloc(positions)\n pos_alloc = pos_alloc.drop('cash', axis=1)\n\n liq_desc = pd.DataFrame()\n liq_desc['days_to_liquidate'] = dtlp.unstack()\n liq_desc['pos_alloc_pct'] = pos_alloc.unstack() * 100\n liq_desc.index.levels[0].name = 'symbol'\n liq_desc.index.levels[1].name = 'date'\n\n worst_liq = liq_desc.reset_index().sort_values(\n 'days_to_liquidate', ascending=False).groupby('symbol').first()\n\n return worst_liq", "response": "Returns a DataFrame that contains the maximum number of days to liquidate for each traded item in the order of market data."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_low_liquidity_transactions(transactions, market_data,\n last_n_days=None):\n \"\"\"\n For each traded name, find the daily transaction total that consumed\n the greatest proportion of available daily bar volume.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n market_data : pd.Panel\n Panel with items axis of 'price' and 'volume' DataFrames.\n The major and minor axes should match those of the\n the passed positions DataFrame (same dates and symbols).\n last_n_days : integer\n Compute for only the last n days of the passed backtest data.\n \"\"\"\n\n txn_daily_w_bar = daily_txns_with_bar_data(transactions, market_data)\n txn_daily_w_bar.index.name = 'date'\n txn_daily_w_bar = txn_daily_w_bar.reset_index()\n\n if last_n_days is not None:\n md = txn_daily_w_bar.date.max() - pd.Timedelta(days=last_n_days)\n txn_daily_w_bar = txn_daily_w_bar[txn_daily_w_bar.date > md]\n\n bar_consumption = txn_daily_w_bar.assign(\n max_pct_bar_consumed=(\n txn_daily_w_bar.amount/txn_daily_w_bar.volume)*100\n ).sort_values('max_pct_bar_consumed', ascending=False)\n max_bar_consumption = bar_consumption.groupby('symbol').first()\n\n return max_bar_consumption[['date', 'max_pct_bar_consumed']]", "response": "Given a DataFrame of transactions and a market data return the low liquidity transactions."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef apply_slippage_penalty(returns, txn_daily, simulate_starting_capital,\n backtest_starting_capital, impact=0.1):\n \"\"\"\n Applies quadratic volumeshare slippage model to daily returns based\n on the proportion of the observed historical daily bar dollar volume\n consumed by the strategy's trades. Scales the size of trades based\n on the ratio of the starting capital we wish to test to the starting\n capital of the passed backtest data.\n\n Parameters\n ----------\n returns : pd.Series\n Time series of daily returns.\n txn_daily : pd.Series\n Daily transaciton totals, closing price, and daily volume for\n each traded name. See price_volume_daily_txns for more details.\n simulate_starting_capital : integer\n capital at which we want to test\n backtest_starting_capital: capital base at which backtest was\n origionally run. impact: See Zipline volumeshare slippage model\n impact : float\n Scales the size of the slippage penalty.\n\n Returns\n -------\n adj_returns : pd.Series\n Slippage penalty adjusted daily returns.\n \"\"\"\n\n mult = simulate_starting_capital / backtest_starting_capital\n simulate_traded_shares = abs(mult * txn_daily.amount)\n simulate_traded_dollars = txn_daily.price * simulate_traded_shares\n simulate_pct_volume_used = simulate_traded_shares / txn_daily.volume\n\n penalties = simulate_pct_volume_used**2 \\\n * impact * simulate_traded_dollars\n\n daily_penalty = penalties.resample('D').sum()\n daily_penalty = daily_penalty.reindex(returns.index).fillna(0)\n\n # Since we are scaling the numerator of the penalties linearly\n # by capital base, it makes the most sense to scale the denominator\n # similarly. In other words, since we aren't applying compounding to\n # simulate_traded_shares, we shouldn't apply compounding to pv.\n portfolio_value = ep.cum_returns(\n returns, starting_value=backtest_starting_capital) * mult\n\n adj_returns = returns - (daily_penalty / portfolio_value)\n\n return adj_returns", "response": "Applies a quadratic volumeshare slippage model to daily returns based on the penalty adjusted by the original daily returns."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmapping a single transaction row to a dictionary.", "response": "def map_transaction(txn):\n \"\"\"\n Maps a single transaction row to a dictionary.\n\n Parameters\n ----------\n txn : pd.DataFrame\n A single transaction object to convert to a dictionary.\n\n Returns\n -------\n dict\n Mapped transaction.\n \"\"\"\n\n if isinstance(txn['sid'], dict):\n sid = txn['sid']['sid']\n symbol = txn['sid']['symbol']\n else:\n sid = txn['sid']\n symbol = txn['sid']\n\n return {'sid': sid,\n 'symbol': symbol,\n 'price': txn['price'],\n 'order_id': txn['order_id'],\n 'amount': txn['amount'],\n 'commission': txn['commission'],\n 'dt': txn['dt']}"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ntaking a dataframe of transactional data and returns a DataFrame with the appropriate amount and price.", "response": "def make_transaction_frame(transactions):\n \"\"\"\n Formats a transaction DataFrame.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Contains improperly formatted transactional data.\n\n Returns\n -------\n df : pd.DataFrame\n Daily transaction volume and dollar ammount.\n - See full explanation in tears.create_full_tear_sheet.\n \"\"\"\n\n transaction_list = []\n for dt in transactions.index:\n txns = transactions.loc[dt]\n if len(txns) == 0:\n continue\n\n for txn in txns:\n txn = map_transaction(txn)\n transaction_list.append(txn)\n df = pd.DataFrame(sorted(transaction_list, key=lambda x: x['dt']))\n df['txn_dollars'] = -df['amount'] * df['price']\n\n df.index = list(map(pd.Timestamp, df.dt.values))\n return df"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nextract daily transaction data from a set of transactions.", "response": "def get_txn_vol(transactions):\n \"\"\"\n Extract daily transaction data from set of transaction objects.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Time series containing one row per symbol (and potentially\n duplicate datetime indices) and columns for amount and\n price.\n\n Returns\n -------\n pd.DataFrame\n Daily transaction volume and number of shares.\n - See full explanation in tears.create_full_tear_sheet.\n \"\"\"\n\n txn_norm = transactions.copy()\n txn_norm.index = txn_norm.index.normalize()\n amounts = txn_norm.amount.abs()\n prices = txn_norm.price\n values = amounts * prices\n daily_amounts = amounts.groupby(amounts.index).sum()\n daily_values = values.groupby(values.index).sum()\n daily_amounts.name = \"txn_shares\"\n daily_values.name = \"txn_volume\"\n return pd.concat([daily_values, daily_amounts], axis=1)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef adjust_returns_for_slippage(returns, positions, transactions,\n slippage_bps):\n \"\"\"\n Apply a slippage penalty for every dollar traded.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n slippage_bps: int/float\n Basis points of slippage to apply.\n\n Returns\n -------\n pd.Series\n Time series of daily returns, adjusted for slippage.\n \"\"\"\n\n slippage = 0.0001 * slippage_bps\n portfolio_value = positions.sum(axis=1)\n pnl = portfolio_value * returns\n traded_value = get_txn_vol(transactions).txn_volume\n slippage_dollars = traded_value * slippage\n adjusted_pnl = pnl.add(-slippage_dollars, fill_value=0)\n adjusted_returns = returns * adjusted_pnl / pnl\n\n return adjusted_returns", "response": "Adjusts the daily returns for a slippage."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_turnover(positions, transactions, denominator='AGB'):\n\n txn_vol = get_txn_vol(transactions)\n traded_value = txn_vol.txn_volume\n\n if denominator == 'AGB':\n # Actual gross book is the same thing as the algo's GMV\n # We want our denom to be avg(AGB previous, AGB current)\n AGB = positions.drop('cash', axis=1).abs().sum(axis=1)\n denom = AGB.rolling(2).mean()\n\n # Since the first value of pd.rolling returns NaN, we\n # set our \"day 0\" AGB to 0.\n denom.iloc[0] = AGB.iloc[0] / 2\n elif denominator == 'portfolio_value':\n denom = positions.sum(axis=1)\n else:\n raise ValueError(\n \"Unexpected value for denominator '{}'. The \"\n \"denominator parameter must be either 'AGB'\"\n \" or 'portfolio_value'.\".format(denominator)\n )\n\n denom.index = denom.index.normalize()\n turnover = traded_value.div(denom, axis='index')\n turnover = turnover.fillna(0)\n return turnover", "response": "Returns the amount of turnover for the given set of transactions."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngroup transactions by consecutive amount.", "response": "def _groupby_consecutive(txn, max_delta=pd.Timedelta('8h')):\n \"\"\"Merge transactions of the same direction separated by less than\n max_delta time duration.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Prices and amounts of executed round_trips. One row per trade.\n - See full explanation in tears.create_full_tear_sheet\n\n max_delta : pandas.Timedelta (optional)\n Merge transactions in the same direction separated by less\n than max_delta time duration.\n\n\n Returns\n -------\n transactions : pd.DataFrame\n\n \"\"\"\n def vwap(transaction):\n if transaction.amount.sum() == 0:\n warnings.warn('Zero transacted shares, setting vwap to nan.')\n return np.nan\n return (transaction.amount * transaction.price).sum() / \\\n transaction.amount.sum()\n\n out = []\n for sym, t in txn.groupby('symbol'):\n t = t.sort_index()\n t.index.name = 'dt'\n t = t.reset_index()\n\n t['order_sign'] = t.amount > 0\n t['block_dir'] = (t.order_sign.shift(\n 1) != t.order_sign).astype(int).cumsum()\n t['block_time'] = ((t.dt.sub(t.dt.shift(1))) >\n max_delta).astype(int).cumsum()\n grouped_price = (t.groupby(('block_dir',\n 'block_time'))\n .apply(vwap))\n grouped_price.name = 'price'\n grouped_rest = t.groupby(('block_dir', 'block_time')).agg({\n 'amount': 'sum',\n 'symbol': 'first',\n 'dt': 'first'})\n\n grouped = grouped_rest.join(grouped_price)\n\n out.append(grouped)\n\n out = pd.concat(out)\n out = out.set_index('dt')\n return out"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef extract_round_trips(transactions,\n portfolio_value=None):\n \"\"\"Group transactions into \"round trips\". First, transactions are\n grouped by day and directionality. Then, long and short\n transactions are matched to create round-trip round_trips for which\n PnL, duration and returns are computed. Crossings where a position\n changes from long to short and vice-versa are handled correctly.\n\n Under the hood, we reconstruct the individual shares in a\n portfolio over time and match round_trips in a FIFO-order.\n\n For example, the following transactions would constitute one round trip:\n index amount price symbol\n 2004-01-09 12:18:01 10 50 'AAPL'\n 2004-01-09 15:12:53 10 100 'AAPL'\n 2004-01-13 14:41:23 -10 100 'AAPL'\n 2004-01-13 15:23:34 -10 200 'AAPL'\n\n First, the first two and last two round_trips will be merged into a two\n single transactions (computing the price via vwap). Then, during\n the portfolio reconstruction, the two resulting transactions will\n be merged and result in 1 round-trip trade with a PnL of\n (150 * 20) - (75 * 20) = 1500.\n\n Note, that round trips do not have to close out positions\n completely. For example, we could have removed the last\n transaction in the example above and still generated a round-trip\n over 10 shares with 10 shares left in the portfolio to be matched\n with a later transaction.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Prices and amounts of executed round_trips. One row per trade.\n - See full explanation in tears.create_full_tear_sheet\n\n portfolio_value : pd.Series (optional)\n Portfolio value (all net assets including cash) over time.\n Note that portfolio_value needs to beginning of day, so either\n use .shift() or positions.sum(axis='columns') / (1+returns).\n\n Returns\n -------\n round_trips : pd.DataFrame\n DataFrame with one row per round trip. The returns column\n contains returns in respect to the portfolio value while\n rt_returns are the returns in regards to the invested capital\n into that partiulcar round-trip.\n \"\"\"\n\n transactions = _groupby_consecutive(transactions)\n roundtrips = []\n\n for sym, trans_sym in transactions.groupby('symbol'):\n trans_sym = trans_sym.sort_index()\n price_stack = deque()\n dt_stack = deque()\n trans_sym['signed_price'] = trans_sym.price * \\\n np.sign(trans_sym.amount)\n trans_sym['abs_amount'] = trans_sym.amount.abs().astype(int)\n for dt, t in trans_sym.iterrows():\n if t.price < 0:\n warnings.warn('Negative price detected, ignoring for'\n 'round-trip.')\n continue\n\n indiv_prices = [t.signed_price] * t.abs_amount\n if (len(price_stack) == 0) or \\\n (copysign(1, price_stack[-1]) == copysign(1, t.amount)):\n price_stack.extend(indiv_prices)\n dt_stack.extend([dt] * len(indiv_prices))\n else:\n # Close round-trip\n pnl = 0\n invested = 0\n cur_open_dts = []\n\n for price in indiv_prices:\n if len(price_stack) != 0 and \\\n (copysign(1, price_stack[-1]) != copysign(1, price)):\n # Retrieve first dt, stock-price pair from\n # stack\n prev_price = price_stack.popleft()\n prev_dt = dt_stack.popleft()\n\n pnl += -(price + prev_price)\n cur_open_dts.append(prev_dt)\n invested += abs(prev_price)\n\n else:\n # Push additional stock-prices onto stack\n price_stack.append(price)\n dt_stack.append(dt)\n\n roundtrips.append({'pnl': pnl,\n 'open_dt': cur_open_dts[0],\n 'close_dt': dt,\n 'long': price < 0,\n 'rt_returns': pnl / invested,\n 'symbol': sym,\n })\n\n roundtrips = pd.DataFrame(roundtrips)\n\n roundtrips['duration'] = roundtrips['close_dt'].sub(roundtrips['open_dt'])\n\n if portfolio_value is not None:\n # Need to normalize so that we can join\n pv = pd.DataFrame(portfolio_value,\n columns=['portfolio_value'])\\\n .assign(date=portfolio_value.index)\n\n roundtrips['date'] = roundtrips.close_dt.apply(lambda x:\n x.replace(hour=0,\n minute=0,\n second=0))\n\n tmp = roundtrips.join(pv, on='date', lsuffix='_')\n\n roundtrips['returns'] = tmp.pnl / tmp.portfolio_value\n roundtrips = roundtrips.drop('date', axis='columns')\n\n return roundtrips", "response": "Given a list of transactions and a portfolio value extract the round trips for that portfolio."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef add_closing_transactions(positions, transactions):\n\n closed_txns = transactions[['symbol', 'amount', 'price']]\n\n pos_at_end = positions.drop('cash', axis=1).iloc[-1]\n open_pos = pos_at_end.replace(0, np.nan).dropna()\n # Add closing round_trips one second after the close to be sure\n # they don't conflict with other round_trips executed at that time.\n end_dt = open_pos.name + pd.Timedelta(seconds=1)\n\n for sym, ending_val in open_pos.iteritems():\n txn_sym = transactions[transactions.symbol == sym]\n\n ending_amount = txn_sym.amount.sum()\n\n ending_price = ending_val / ending_amount\n closing_txn = {'symbol': sym,\n 'amount': -ending_amount,\n 'price': ending_price}\n\n closing_txn = pd.DataFrame(closing_txn, index=[end_dt])\n closed_txns = closed_txns.append(closing_txn)\n\n closed_txns = closed_txns[closed_txns.amount != 0]\n\n return closed_txns", "response": "Adds transactions that close out all positions at the end of the positions dataframe."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a DataFrame with the first row of round_trips translated to sector names.", "response": "def apply_sector_mappings_to_round_trips(round_trips, sector_mappings):\n \"\"\"\n Translates round trip symbols to sectors.\n\n Parameters\n ----------\n round_trips : pd.DataFrame\n DataFrame with one row per round trip trade.\n - See full explanation in round_trips.extract_round_trips\n sector_mappings : dict or pd.Series, optional\n Security identifier to sector mapping.\n Security ids as keys, sectors as values.\n\n Returns\n -------\n sector_round_trips : pd.DataFrame\n Round trips with symbol names replaced by sector names.\n \"\"\"\n\n sector_round_trips = round_trips.copy()\n sector_round_trips.symbol = sector_round_trips.symbol.apply(\n lambda x: sector_mappings.get(x, 'No Sector Mapping'))\n sector_round_trips = sector_round_trips.dropna(axis=0)\n\n return sector_round_trips"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates various round - trip statistics.", "response": "def gen_round_trip_stats(round_trips):\n \"\"\"Generate various round-trip statistics.\n\n Parameters\n ----------\n round_trips : pd.DataFrame\n DataFrame with one row per round trip trade.\n - See full explanation in round_trips.extract_round_trips\n\n Returns\n -------\n stats : dict\n A dictionary where each value is a pandas DataFrame containing\n various round-trip statistics.\n\n See also\n --------\n round_trips.print_round_trip_stats\n \"\"\"\n\n stats = {}\n stats['pnl'] = agg_all_long_short(round_trips, 'pnl', PNL_STATS)\n stats['summary'] = agg_all_long_short(round_trips, 'pnl',\n SUMMARY_STATS)\n stats['duration'] = agg_all_long_short(round_trips, 'duration',\n DURATION_STATS)\n stats['returns'] = agg_all_long_short(round_trips, 'returns',\n RETURN_STATS)\n\n stats['symbols'] = \\\n round_trips.groupby('symbol')['returns'].agg(RETURN_STATS).T\n\n return stats"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nprints various round - trip statistics. Tries to pretty - print tables with HTML output if run inside IPython NB.", "response": "def print_round_trip_stats(round_trips, hide_pos=False):\n \"\"\"Print various round-trip statistics. Tries to pretty-print tables\n with HTML output if run inside IPython NB.\n\n Parameters\n ----------\n round_trips : pd.DataFrame\n DataFrame with one row per round trip trade.\n - See full explanation in round_trips.extract_round_trips\n\n See also\n --------\n round_trips.gen_round_trip_stats\n \"\"\"\n\n stats = gen_round_trip_stats(round_trips)\n\n print_table(stats['summary'], float_format='{:.2f}'.format,\n name='Summary stats')\n print_table(stats['pnl'], float_format='${:.2f}'.format, name='PnL stats')\n print_table(stats['duration'], float_format='{:.2f}'.format,\n name='Duration stats')\n print_table(stats['returns'] * 100, float_format='{:.2f}%'.format,\n name='Return stats')\n\n if not hide_pos:\n stats['symbols'].columns = stats['symbols'].columns.map(format_asset)\n print_table(stats['symbols'] * 100,\n float_format='{:.2f}%'.format, name='Symbol stats')"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the performance of a single return stream.", "response": "def perf_attrib(returns,\n positions,\n factor_returns,\n factor_loadings,\n transactions=None,\n pos_in_dollars=True):\n \"\"\"\n Attributes the performance of a returns stream to a set of risk factors.\n\n Preprocesses inputs, and then calls empyrical.perf_attrib. See\n empyrical.perf_attrib for more info.\n\n Performance attribution determines how much each risk factor, e.g.,\n momentum, the technology sector, etc., contributed to total returns, as\n well as the daily exposure to each of the risk factors. The returns that\n can be attributed to one of the given risk factors are the\n `common_returns`, and the returns that _cannot_ be attributed to a risk\n factor are the `specific_returns`, or the alpha. The common_returns and\n specific_returns summed together will always equal the total returns.\n\n Parameters\n ----------\n returns : pd.Series\n Returns for each day in the date range.\n - Example:\n 2017-01-01 -0.017098\n 2017-01-02 0.002683\n 2017-01-03 -0.008669\n\n positions: pd.DataFrame\n Daily holdings (in dollars or percentages), indexed by date.\n Will be converted to percentages if positions are in dollars.\n Short positions show up as cash in the 'cash' column.\n - Examples:\n AAPL TLT XOM cash\n 2017-01-01 34 58 10 0\n 2017-01-02 22 77 18 0\n 2017-01-03 -15 27 30 15\n\n AAPL TLT XOM cash\n 2017-01-01 0.333333 0.568627 0.098039 0.0\n 2017-01-02 0.188034 0.658120 0.153846 0.0\n 2017-01-03 0.208333 0.375000 0.416667 0.0\n\n factor_returns : pd.DataFrame\n Returns by factor, with date as index and factors as columns\n - Example:\n momentum reversal\n 2017-01-01 0.002779 -0.005453\n 2017-01-02 0.001096 0.010290\n\n factor_loadings : pd.DataFrame\n Factor loadings for all days in the date range, with date and ticker as\n index, and factors as columns.\n - Example:\n momentum reversal\n dt ticker\n 2017-01-01 AAPL -1.592914 0.852830\n TLT 0.184864 0.895534\n XOM 0.993160 1.149353\n 2017-01-02 AAPL -0.140009 -0.524952\n TLT -1.066978 0.185435\n XOM -1.798401 0.761549\n\n\n transactions : pd.DataFrame, optional\n Executed trade volumes and fill prices. Used to check the turnover of\n the algorithm. Default is None, in which case the turnover check is\n skipped.\n\n - One row per trade.\n - Trades on different names that occur at the\n same time will have identical indicies.\n - Example:\n index amount price symbol\n 2004-01-09 12:18:01 483 324.12 'AAPL'\n 2004-01-09 12:18:01 122 83.10 'MSFT'\n 2004-01-13 14:12:23 -75 340.43 'AAPL'\n\n pos_in_dollars : bool\n Flag indicating whether `positions` are in dollars or percentages\n If True, positions are in dollars.\n\n Returns\n -------\n tuple of (risk_exposures_portfolio, perf_attribution)\n\n risk_exposures_portfolio : pd.DataFrame\n df indexed by datetime, with factors as columns\n - Example:\n momentum reversal\n dt\n 2017-01-01 -0.238655 0.077123\n 2017-01-02 0.821872 1.520515\n\n perf_attribution : pd.DataFrame\n df with factors, common returns, and specific returns as columns,\n and datetimes as index\n - Example:\n momentum reversal common_returns specific_returns\n dt\n 2017-01-01 0.249087 0.935925 1.185012 1.185012\n 2017-01-02 -0.003194 -0.400786 -0.403980 -0.403980\n \"\"\"\n (returns,\n positions,\n factor_returns,\n factor_loadings) = _align_and_warn(returns,\n positions,\n factor_returns,\n factor_loadings,\n transactions=transactions,\n pos_in_dollars=pos_in_dollars)\n\n # Note that we convert positions to percentages *after* the checks\n # above, since get_turnover() expects positions in dollars.\n positions = _stack_positions(positions, pos_in_dollars=pos_in_dollars)\n\n return ep.perf_attrib(returns, positions, factor_returns, factor_loadings)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute daily risk factor exposures for a single item in the archive.", "response": "def compute_exposures(positions, factor_loadings, stack_positions=True,\n pos_in_dollars=True):\n \"\"\"\n Compute daily risk factor exposures.\n\n Normalizes positions (if necessary) and calls ep.compute_exposures.\n See empyrical.compute_exposures for more info.\n\n Parameters\n ----------\n positions: pd.DataFrame or pd.Series\n Daily holdings (in dollars or percentages), indexed by date, OR\n a series of holdings indexed by date and ticker.\n - Examples:\n AAPL TLT XOM cash\n 2017-01-01 34 58 10 0\n 2017-01-02 22 77 18 0\n 2017-01-03 -15 27 30 15\n\n AAPL TLT XOM cash\n 2017-01-01 0.333333 0.568627 0.098039 0.0\n 2017-01-02 0.188034 0.658120 0.153846 0.0\n 2017-01-03 0.208333 0.375000 0.416667 0.0\n\n dt ticker\n 2017-01-01 AAPL 0.417582\n TLT 0.010989\n XOM 0.571429\n 2017-01-02 AAPL 0.202381\n TLT 0.535714\n XOM 0.261905\n\n factor_loadings : pd.DataFrame\n Factor loadings for all days in the date range, with date and ticker as\n index, and factors as columns.\n - Example:\n momentum reversal\n dt ticker\n 2017-01-01 AAPL -1.592914 0.852830\n TLT 0.184864 0.895534\n XOM 0.993160 1.149353\n 2017-01-02 AAPL -0.140009 -0.524952\n TLT -1.066978 0.185435\n XOM -1.798401 0.761549\n\n stack_positions : bool\n Flag indicating whether `positions` should be converted to long format.\n\n pos_in_dollars : bool\n Flag indicating whether `positions` are in dollars or percentages\n If True, positions are in dollars.\n\n Returns\n -------\n risk_exposures_portfolio : pd.DataFrame\n df indexed by datetime, with factors as columns.\n - Example:\n momentum reversal\n dt\n 2017-01-01 -0.238655 0.077123\n 2017-01-02 0.821872 1.520515\n \"\"\"\n if stack_positions:\n positions = _stack_positions(positions, pos_in_dollars=pos_in_dollars)\n\n return ep.compute_exposures(positions, factor_loadings)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a dataframe of statistics for a single perf attribute.", "response": "def create_perf_attrib_stats(perf_attrib, risk_exposures):\n \"\"\"\n Takes perf attribution data over a period of time and computes annualized\n multifactor alpha, multifactor sharpe, risk exposures.\n \"\"\"\n summary = OrderedDict()\n total_returns = perf_attrib['total_returns']\n specific_returns = perf_attrib['specific_returns']\n common_returns = perf_attrib['common_returns']\n\n summary['Annualized Specific Return'] =\\\n ep.annual_return(specific_returns)\n summary['Annualized Common Return'] =\\\n ep.annual_return(common_returns)\n summary['Annualized Total Return'] =\\\n ep.annual_return(total_returns)\n\n summary['Specific Sharpe Ratio'] =\\\n ep.sharpe_ratio(specific_returns)\n\n summary['Cumulative Specific Return'] =\\\n ep.cum_returns_final(specific_returns)\n summary['Cumulative Common Return'] =\\\n ep.cum_returns_final(common_returns)\n summary['Total Returns'] =\\\n ep.cum_returns_final(total_returns)\n\n summary = pd.Series(summary, name='')\n\n annualized_returns_by_factor = [ep.annual_return(perf_attrib[c])\n for c in risk_exposures.columns]\n cumulative_returns_by_factor = [ep.cum_returns_final(perf_attrib[c])\n for c in risk_exposures.columns]\n\n risk_exposure_summary = pd.DataFrame(\n data=OrderedDict([\n (\n 'Average Risk Factor Exposure',\n risk_exposures.mean(axis='rows')\n ),\n ('Annualized Return', annualized_returns_by_factor),\n ('Cumulative Return', cumulative_returns_by_factor),\n ]),\n index=risk_exposures.columns,\n )\n\n return summary, risk_exposure_summary"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef show_perf_attrib_stats(returns,\n positions,\n factor_returns,\n factor_loadings,\n transactions=None,\n pos_in_dollars=True):\n \"\"\"\n Calls `perf_attrib` using inputs, and displays outputs using\n `utils.print_table`.\n \"\"\"\n risk_exposures, perf_attrib_data = perf_attrib(\n returns,\n positions,\n factor_returns,\n factor_loadings,\n transactions,\n pos_in_dollars=pos_in_dollars,\n )\n\n perf_attrib_stats, risk_exposure_stats =\\\n create_perf_attrib_stats(perf_attrib_data, risk_exposures)\n\n percentage_formatter = '{:.2%}'.format\n float_formatter = '{:.2f}'.format\n\n summary_stats = perf_attrib_stats.loc[['Annualized Specific Return',\n 'Annualized Common Return',\n 'Annualized Total Return',\n 'Specific Sharpe Ratio']]\n\n # Format return rows in summary stats table as percentages.\n for col_name in (\n 'Annualized Specific Return',\n 'Annualized Common Return',\n 'Annualized Total Return',\n ):\n summary_stats[col_name] = percentage_formatter(summary_stats[col_name])\n\n # Display sharpe to two decimal places.\n summary_stats['Specific Sharpe Ratio'] = float_formatter(\n summary_stats['Specific Sharpe Ratio']\n )\n\n print_table(summary_stats, name='Summary Statistics')\n\n print_table(\n risk_exposure_stats,\n name='Exposures Summary',\n # In exposures table, format exposure column to 2 decimal places, and\n # return columns as percentages.\n formatters={\n 'Average Risk Factor Exposure': float_formatter,\n 'Annualized Return': percentage_formatter,\n 'Cumulative Return': percentage_formatter,\n },\n )", "response": "Displays summary statistics and exposures for a given set of returns."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_returns(perf_attrib_data, cost=None, ax=None):\n\n if ax is None:\n ax = plt.gca()\n\n returns = perf_attrib_data['total_returns']\n total_returns_label = 'Total returns'\n\n cumulative_returns_less_costs = _cumulative_returns_less_costs(\n returns,\n cost\n )\n if cost is not None:\n total_returns_label += ' (adjusted)'\n\n specific_returns = perf_attrib_data['specific_returns']\n common_returns = perf_attrib_data['common_returns']\n\n ax.plot(cumulative_returns_less_costs, color='b',\n label=total_returns_label)\n ax.plot(ep.cum_returns(specific_returns), color='g',\n label='Cumulative specific returns')\n ax.plot(ep.cum_returns(common_returns), color='r',\n label='Cumulative common returns')\n\n if cost is not None:\n ax.plot(-ep.cum_returns(cost), color='k',\n label='Cumulative cost spent')\n\n ax.set_title('Time series of cumulative returns')\n ax.set_ylabel('Returns')\n\n configure_legend(ax)\n\n return ax", "response": "Plots total specific common and specific returns for the current object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plot_alpha_returns(alpha_returns, ax=None):\n if ax is None:\n ax = plt.gca()\n\n ax.hist(alpha_returns, color='g', label='Multi-factor alpha')\n ax.set_title('Histogram of alphas')\n ax.axvline(0, color='k', linestyle='--', label='Zero')\n\n avg = alpha_returns.mean()\n ax.axvline(avg, color='b', label='Mean = {: 0.5f}'.format(avg))\n configure_legend(ax)\n\n return ax", "response": "Plots histogram of daily multi - factor alpha returns."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nplots each factor s contribution to performance.", "response": "def plot_factor_contribution_to_perf(\n perf_attrib_data,\n ax=None,\n title='Cumulative common returns attribution',\n):\n \"\"\"\n Plot each factor's contribution to performance.\n\n Parameters\n ----------\n perf_attrib_data : pd.DataFrame\n df with factors, common returns, and specific returns as columns,\n and datetimes as index\n - Example:\n momentum reversal common_returns specific_returns\n dt\n 2017-01-01 0.249087 0.935925 1.185012 1.185012\n 2017-01-02 -0.003194 -0.400786 -0.403980 -0.403980\n\n ax : matplotlib.axes.Axes\n axes on which plots are made. if None, current axes will be used\n\n title : str, optional\n title of plot\n\n Returns\n -------\n ax : matplotlib.axes.Axes\n \"\"\"\n if ax is None:\n ax = plt.gca()\n\n factors_to_plot = perf_attrib_data.drop(\n ['total_returns', 'common_returns'], axis='columns', errors='ignore'\n )\n\n factors_cumulative = pd.DataFrame()\n for factor in factors_to_plot:\n factors_cumulative[factor] = ep.cum_returns(factors_to_plot[factor])\n\n for col in factors_cumulative:\n ax.plot(factors_cumulative[col])\n\n ax.axhline(0, color='k')\n configure_legend(ax, change_colors=True)\n\n ax.set_ylabel('Cumulative returns by factor')\n ax.set_title(title)\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nplot the factor exposures of the daily risk factor.", "response": "def plot_risk_exposures(exposures, ax=None,\n title='Daily risk factor exposures'):\n \"\"\"\n Parameters\n ----------\n exposures : pd.DataFrame\n df indexed by datetime, with factors as columns\n - Example:\n momentum reversal\n dt\n 2017-01-01 -0.238655 0.077123\n 2017-01-02 0.821872 1.520515\n\n ax : matplotlib.axes.Axes\n axes on which plots are made. if None, current axes will be used\n\n Returns\n -------\n ax : matplotlib.axes.Axes\n \"\"\"\n if ax is None:\n ax = plt.gca()\n\n for col in exposures:\n ax.plot(exposures[col])\n\n configure_legend(ax, change_colors=True)\n ax.set_ylabel('Factor exposures')\n ax.set_title(title)\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _align_and_warn(returns,\n positions,\n factor_returns,\n factor_loadings,\n transactions=None,\n pos_in_dollars=True):\n \"\"\"\n Make sure that all inputs have matching dates and tickers,\n and raise warnings if necessary.\n \"\"\"\n missing_stocks = positions.columns.difference(\n factor_loadings.index.get_level_values(1).unique()\n )\n\n # cash will not be in factor_loadings\n num_stocks = len(positions.columns) - 1\n missing_stocks = missing_stocks.drop('cash')\n num_stocks_covered = num_stocks - len(missing_stocks)\n missing_ratio = round(len(missing_stocks) / num_stocks, ndigits=3)\n\n if num_stocks_covered == 0:\n raise ValueError(\"Could not perform performance attribution. \"\n \"No factor loadings were available for this \"\n \"algorithm's positions.\")\n\n if len(missing_stocks) > 0:\n\n if len(missing_stocks) > 5:\n\n missing_stocks_displayed = (\n \" {} assets were missing factor loadings, including: {}..{}\"\n ).format(len(missing_stocks),\n ', '.join(missing_stocks[:5].map(str)),\n missing_stocks[-1])\n avg_allocation_msg = \"selected missing assets\"\n\n else:\n missing_stocks_displayed = (\n \"The following assets were missing factor loadings: {}.\"\n ).format(list(missing_stocks))\n avg_allocation_msg = \"missing assets\"\n\n missing_stocks_warning_msg = (\n \"Could not determine risk exposures for some of this algorithm's \"\n \"positions. Returns from the missing assets will not be properly \"\n \"accounted for in performance attribution.\\n\"\n \"\\n\"\n \"{}. \"\n \"Ignoring for exposure calculation and performance attribution. \"\n \"Ratio of assets missing: {}. Average allocation of {}:\\n\"\n \"\\n\"\n \"{}.\\n\"\n ).format(\n missing_stocks_displayed,\n missing_ratio,\n avg_allocation_msg,\n positions[missing_stocks[:5].union(missing_stocks[[-1]])].mean(),\n )\n\n warnings.warn(missing_stocks_warning_msg)\n\n positions = positions.drop(missing_stocks, axis='columns',\n errors='ignore')\n\n missing_factor_loadings_index = positions.index.difference(\n factor_loadings.index.get_level_values(0).unique()\n )\n\n missing_factor_loadings_index = positions.index.difference(\n factor_loadings.index.get_level_values(0).unique()\n )\n\n if len(missing_factor_loadings_index) > 0:\n\n if len(missing_factor_loadings_index) > 5:\n missing_dates_displayed = (\n \"(first missing is {}, last missing is {})\"\n ).format(\n missing_factor_loadings_index[0],\n missing_factor_loadings_index[-1]\n )\n else:\n missing_dates_displayed = list(missing_factor_loadings_index)\n\n warning_msg = (\n \"Could not find factor loadings for {} dates: {}. \"\n \"Truncating date range for performance attribution. \"\n ).format(len(missing_factor_loadings_index), missing_dates_displayed)\n\n warnings.warn(warning_msg)\n\n positions = positions.drop(missing_factor_loadings_index,\n errors='ignore')\n returns = returns.drop(missing_factor_loadings_index, errors='ignore')\n factor_returns = factor_returns.drop(missing_factor_loadings_index,\n errors='ignore')\n\n if transactions is not None and pos_in_dollars:\n turnover = get_turnover(positions, transactions).mean()\n if turnover > PERF_ATTRIB_TURNOVER_THRESHOLD:\n warning_msg = (\n \"This algorithm has relatively high turnover of its \"\n \"positions. As a result, performance attribution might not be \"\n \"fully accurate.\\n\"\n \"\\n\"\n \"Performance attribution is calculated based \"\n \"on end-of-day holdings and does not account for intraday \"\n \"activity. Algorithms that derive a high percentage of \"\n \"returns from buying and selling within the same day may \"\n \"receive inaccurate performance attribution.\\n\"\n )\n warnings.warn(warning_msg)\n\n return (returns, positions, factor_returns, factor_loadings)", "response": "Aligns the returns and raises a warning if necessary."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts positions to percentages if necessary and change them to long format.", "response": "def _stack_positions(positions, pos_in_dollars=True):\n \"\"\"\n Convert positions to percentages if necessary, and change them\n to long format.\n\n Parameters\n ----------\n positions: pd.DataFrame\n Daily holdings (in dollars or percentages), indexed by date.\n Will be converted to percentages if positions are in dollars.\n Short positions show up as cash in the 'cash' column.\n\n pos_in_dollars : bool\n Flag indicating whether `positions` are in dollars or percentages\n If True, positions are in dollars.\n \"\"\"\n if pos_in_dollars:\n # convert holdings to percentages\n positions = get_percent_alloc(positions)\n\n # remove cash after normalizing positions\n positions = positions.drop('cash', axis='columns')\n\n # convert positions to long format\n positions = positions.stack()\n positions.index = positions.index.set_names(['dt', 'ticker'])\n\n return positions"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes cumulative returns less costs.", "response": "def _cumulative_returns_less_costs(returns, costs):\n \"\"\"\n Compute cumulative returns, less costs.\n \"\"\"\n if costs is None:\n return ep.cum_returns(returns)\n return ep.cum_returns(returns - costs)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nformatting the asset object to be displayed in the tear sheet.", "response": "def format_asset(asset):\n \"\"\"\n If zipline asset objects are used, we want to print them out prettily\n within the tear sheet. This function should only be applied directly\n before displaying.\n \"\"\"\n\n try:\n import zipline.assets\n except ImportError:\n return asset\n\n if isinstance(asset, zipline.assets.Asset):\n return asset.symbol\n else:\n return asset"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nextract returns positions transactions and leverage from the zipline dataframe.", "response": "def extract_rets_pos_txn_from_zipline(backtest):\n \"\"\"\n Extract returns, positions, transactions and leverage from the\n backtest data structure returned by zipline.TradingAlgorithm.run().\n\n The returned data structures are in a format compatible with the\n rest of pyfolio and can be directly passed to\n e.g. tears.create_full_tear_sheet().\n\n Parameters\n ----------\n backtest : pd.DataFrame\n DataFrame returned by zipline.TradingAlgorithm.run()\n\n Returns\n -------\n returns : pd.Series\n Daily returns of strategy.\n - See full explanation in tears.create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet.\n\n\n Example (on the Quantopian research platform)\n ---------------------------------------------\n >>> backtest = my_algo.run()\n >>> returns, positions, transactions =\n >>> pyfolio.utils.extract_rets_pos_txn_from_zipline(backtest)\n >>> pyfolio.tears.create_full_tear_sheet(returns,\n >>> positions, transactions)\n \"\"\"\n\n backtest.index = backtest.index.normalize()\n if backtest.index.tzinfo is None:\n backtest.index = backtest.index.tz_localize('UTC')\n returns = backtest.returns\n raw_positions = []\n for dt, pos_row in backtest.positions.iteritems():\n df = pd.DataFrame(pos_row)\n df.index = [dt] * len(df)\n raw_positions.append(df)\n if not raw_positions:\n raise ValueError(\"The backtest does not have any positions.\")\n positions = pd.concat(raw_positions)\n positions = pos.extract_pos(positions, backtest.ending_cash)\n transactions = txn.make_transaction_frame(backtest.transactions)\n if transactions.index.tzinfo is None:\n transactions.index = transactions.index.tz_localize('utc')\n\n return returns, positions, transactions"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndetect whether an intraday strategy is detected.", "response": "def detect_intraday(positions, transactions, threshold=0.25):\n \"\"\"\n Attempt to detect an intraday strategy. Get the number of\n positions held at the end of the day, and divide that by the\n number of unique stocks transacted every day. If the average quotient\n is below a threshold, then an intraday strategy is detected.\n\n Parameters\n ----------\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n\n Returns\n -------\n boolean\n True if an intraday strategy is detected.\n \"\"\"\n\n daily_txn = transactions.copy()\n daily_txn.index = daily_txn.index.date\n txn_count = daily_txn.groupby(level=0).symbol.nunique().sum()\n daily_pos = positions.drop('cash', axis=1).replace(0, np.nan)\n return daily_pos.count(axis=1).sum() / txn_count < threshold"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef check_intraday(estimate, returns, positions, transactions):\n\n if estimate == 'infer':\n if positions is not None and transactions is not None:\n if detect_intraday(positions, transactions):\n warnings.warn('Detected intraday strategy; inferring positi' +\n 'ons from transactions. Set estimate_intraday' +\n '=False to disable.')\n return estimate_intraday(returns, positions, transactions)\n else:\n return positions\n else:\n return positions\n\n elif estimate:\n if positions is not None and transactions is not None:\n return estimate_intraday(returns, positions, transactions)\n else:\n raise ValueError('Positions and txns needed to estimate intraday')\n else:\n return positions", "response": "Checks if a particular strategy is intraday and returns the result."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nestimates the intraday of the given returns positions and transactions.", "response": "def estimate_intraday(returns, positions, transactions, EOD_hour=23):\n \"\"\"\n Intraday strategies will often not hold positions at the day end.\n This attempts to find the point in the day that best represents\n the activity of the strategy on that day, and effectively resamples\n the end-of-day positions with the positions at this point of day.\n The point of day is found by detecting when our exposure in the\n market is at its maximum point. Note that this is an estimate.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in create_full_tear_sheet.\n\n Returns\n -------\n pd.DataFrame\n Daily net position values, resampled for intraday behavior.\n \"\"\"\n\n # Construct DataFrame of transaction amounts\n txn_val = transactions.copy()\n txn_val.index.names = ['date']\n txn_val['value'] = txn_val.amount * txn_val.price\n txn_val = txn_val.reset_index().pivot_table(\n index='date', values='value',\n columns='symbol').replace(np.nan, 0)\n\n # Cumulate transaction amounts each day\n txn_val['date'] = txn_val.index.date\n txn_val = txn_val.groupby('date').cumsum()\n\n # Calculate exposure, then take peak of exposure every day\n txn_val['exposure'] = txn_val.abs().sum(axis=1)\n condition = (txn_val['exposure'] == txn_val.groupby(\n pd.TimeGrouper('24H'))['exposure'].transform(max))\n txn_val = txn_val[condition].drop('exposure', axis=1)\n\n # Compute cash delta\n txn_val['cash'] = -txn_val.sum(axis=1)\n\n # Shift EOD positions to positions at start of next trading day\n positions_shifted = positions.copy().shift(1).fillna(0)\n starting_capital = positions.iloc[0].sum() / (1 + returns[0])\n positions_shifted.cash[0] = starting_capital\n\n # Format and add start positions to intraday position changes\n txn_val.index = txn_val.index.normalize()\n corrected_positions = positions_shifted.add(txn_val, fill_value=0)\n corrected_positions.index.name = 'period_close'\n corrected_positions.columns.name = 'sid'\n\n return corrected_positions"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef clip_returns_to_benchmark(rets, benchmark_rets):\n\n if (rets.index[0] < benchmark_rets.index[0]) \\\n or (rets.index[-1] > benchmark_rets.index[-1]):\n clipped_rets = rets[benchmark_rets.index]\n else:\n clipped_rets = rets\n\n return clipped_rets", "response": "Clip the returns of a benchmark to match the ones in the benchmark_rets."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts the DataFrame to UTC", "response": "def to_utc(df):\n \"\"\"\n For use in tests; applied UTC timestamp to DataFrame.\n \"\"\"\n\n try:\n df.index = df.index.tz_localize('UTC')\n except TypeError:\n df.index = df.index.tz_convert('UTC')\n\n return df"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_symbol_rets(symbol, start=None, end=None):\n\n return SETTINGS['returns_func'](symbol,\n start=start,\n end=end)", "response": "Returns the series of return data for the specified symbol."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconfigure legend for perf attribution plots", "response": "def configure_legend(ax, autofmt_xdate=True, change_colors=False,\n rotation=30, ha='right'):\n \"\"\"\n Format legend for perf attribution plots:\n - put legend to the right of plot instead of overlapping with it\n - make legend order match up with graph lines\n - set colors according to colormap\n \"\"\"\n chartBox = ax.get_position()\n ax.set_position([chartBox.x0, chartBox.y0,\n chartBox.width * 0.75, chartBox.height])\n\n # make legend order match graph lines\n handles, labels = ax.get_legend_handles_labels()\n handles_and_labels_sorted = sorted(zip(handles, labels),\n key=lambda x: x[0].get_ydata()[-1],\n reverse=True)\n\n handles_sorted = [h[0] for h in handles_and_labels_sorted]\n labels_sorted = [h[1] for h in handles_and_labels_sorted]\n\n if change_colors:\n for handle, color in zip(handles_sorted,\n cycle(COLORS)):\n\n handle.set_color(color)\n\n ax.legend(handles=handles_sorted,\n labels=labels_sorted,\n frameon=True,\n framealpha=0.5,\n loc='upper left',\n bbox_to_anchor=(1.05, 1),\n fontsize='large')\n\n # manually rotate xticklabels instead of using matplotlib's autofmt_xdate\n # because it disables xticklabels for all but the last plot\n if autofmt_xdate:\n for label in ax.get_xticklabels():\n label.set_ha(ha)\n label.set_rotation(rotation)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sample_colormap(cmap_name, n_samples):\n colors = []\n colormap = cm.cmap_d[cmap_name]\n for i in np.linspace(0, 1, n_samples):\n colors.append(colormap(i))\n\n return colors", "response": "Sample a colormap from matplotlib\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef customize(func):\n @wraps(func)\n def call_w_context(*args, **kwargs):\n set_context = kwargs.pop('set_context', True)\n if set_context:\n with plotting_context(), axes_style():\n return func(*args, **kwargs)\n else:\n return func(*args, **kwargs)\n return call_w_context", "response": "Decorator to customize the function call."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a new seaborn plotting style context.", "response": "def plotting_context(context='notebook', font_scale=1.5, rc=None):\n \"\"\"\n Create pyfolio default plotting style context.\n\n Under the hood, calls and returns seaborn.plotting_context() with\n some custom settings. Usually you would use in a with-context.\n\n Parameters\n ----------\n context : str, optional\n Name of seaborn context.\n font_scale : float, optional\n Scale font by factor font_scale.\n rc : dict, optional\n Config flags.\n By default, {'lines.linewidth': 1.5}\n is being used and will be added to any\n rc passed in, unless explicitly overriden.\n\n Returns\n -------\n seaborn plotting context\n\n Example\n -------\n >>> with pyfolio.plotting.plotting_context(font_scale=2):\n >>> pyfolio.create_full_tear_sheet(..., set_context=False)\n\n See also\n --------\n For more information, see seaborn.plotting_context().\n\n \"\"\"\n if rc is None:\n rc = {}\n\n rc_default = {'lines.linewidth': 1.5}\n\n # Add defaults if they do not exist\n for name, val in rc_default.items():\n rc.setdefault(name, val)\n\n return sns.plotting_context(context=context, font_scale=font_scale, rc=rc)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef axes_style(style='darkgrid', rc=None):\n if rc is None:\n rc = {}\n\n rc_default = {}\n\n # Add defaults if they do not exist\n for name, val in rc_default.items():\n rc.setdefault(name, val)\n\n return sns.axes_style(style=style, rc=rc)", "response": "Returns a new seaborn. Seaborn context with default axes style."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_monthly_returns_heatmap(returns, ax=None, **kwargs):\n\n if ax is None:\n ax = plt.gca()\n\n monthly_ret_table = ep.aggregate_returns(returns, 'monthly')\n monthly_ret_table = monthly_ret_table.unstack().round(3)\n\n sns.heatmap(\n monthly_ret_table.fillna(0) *\n 100.0,\n annot=True,\n annot_kws={\"size\": 9},\n alpha=1.0,\n center=0.0,\n cbar=False,\n cmap=matplotlib.cm.RdYlGn,\n ax=ax, **kwargs)\n ax.set_ylabel('Year')\n ax.set_xlabel('Month')\n ax.set_title(\"Monthly returns (%)\")\n return ax", "response": "Plots a heatmap of monthly returns."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nplot an annual returns for a single entry.", "response": "def plot_annual_returns(returns, ax=None, **kwargs):\n \"\"\"\n Plots a bar graph of returns by year.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n x_axis_formatter = FuncFormatter(utils.percentage)\n ax.xaxis.set_major_formatter(FuncFormatter(x_axis_formatter))\n ax.tick_params(axis='x', which='major')\n\n ann_ret_df = pd.DataFrame(\n ep.aggregate_returns(\n returns,\n 'yearly'))\n\n ax.axvline(\n 100 *\n ann_ret_df.values.mean(),\n color='steelblue',\n linestyle='--',\n lw=4,\n alpha=0.7)\n (100 * ann_ret_df.sort_index(ascending=False)\n ).plot(ax=ax, kind='barh', alpha=0.70, **kwargs)\n ax.axvline(0.0, color='black', linestyle='-', lw=3)\n\n ax.set_ylabel('Year')\n ax.set_xlabel('Returns')\n ax.set_title(\"Annual returns\")\n ax.legend(['Mean'], frameon=True, framealpha=0.5)\n return ax"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_monthly_returns_dist(returns, ax=None, **kwargs):\n\n if ax is None:\n ax = plt.gca()\n\n x_axis_formatter = FuncFormatter(utils.percentage)\n ax.xaxis.set_major_formatter(FuncFormatter(x_axis_formatter))\n ax.tick_params(axis='x', which='major')\n\n monthly_ret_table = ep.aggregate_returns(returns, 'monthly')\n\n ax.hist(\n 100 * monthly_ret_table,\n color='orangered',\n alpha=0.80,\n bins=20,\n **kwargs)\n\n ax.axvline(\n 100 * monthly_ret_table.mean(),\n color='gold',\n linestyle='--',\n lw=4,\n alpha=1.0)\n\n ax.axvline(0.0, color='black', linestyle='-', lw=3, alpha=0.75)\n ax.legend(['Mean'], frameon=True, framealpha=0.5)\n ax.set_ylabel('Number of months')\n ax.set_xlabel('Returns')\n ax.set_title(\"Distribution of monthly returns\")\n return ax", "response": "Plots a distribution of monthly returns."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nplots the daily total amount of stocks with an active position either short or long.", "response": "def plot_holdings(returns, positions, legend_loc='best', ax=None, **kwargs):\n \"\"\"\n Plots total amount of stocks with an active position, either short\n or long. Displays daily total, daily average per month, and\n all-time daily average.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n positions : pd.DataFrame, optional\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n positions = positions.copy().drop('cash', axis='columns')\n df_holdings = positions.replace(0, np.nan).count(axis=1)\n df_holdings_by_month = df_holdings.resample('1M').mean()\n df_holdings.plot(color='steelblue', alpha=0.6, lw=0.5, ax=ax, **kwargs)\n df_holdings_by_month.plot(\n color='orangered',\n lw=2,\n ax=ax,\n **kwargs)\n ax.axhline(\n df_holdings.values.mean(),\n color='steelblue',\n ls='--',\n lw=3)\n\n ax.set_xlim((returns.index[0], returns.index[-1]))\n\n leg = ax.legend(['Daily holdings',\n 'Average daily holdings, by month',\n 'Average daily holdings, overall'],\n loc=legend_loc, frameon=True,\n framealpha=0.5)\n leg.get_frame().set_edgecolor('black')\n\n ax.set_title('Total holdings')\n ax.set_ylabel('Holdings')\n ax.set_xlabel('')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nplot the long and short holdings of the given returns.", "response": "def plot_long_short_holdings(returns, positions,\n legend_loc='upper left', ax=None, **kwargs):\n \"\"\"\n Plots total amount of stocks with an active position, breaking out\n short and long into transparent filled regions.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n positions : pd.DataFrame, optional\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n positions = positions.drop('cash', axis='columns')\n positions = positions.replace(0, np.nan)\n df_longs = positions[positions > 0].count(axis=1)\n df_shorts = positions[positions < 0].count(axis=1)\n lf = ax.fill_between(df_longs.index, 0, df_longs.values,\n color='g', alpha=0.5, lw=2.0)\n sf = ax.fill_between(df_shorts.index, 0, df_shorts.values,\n color='r', alpha=0.5, lw=2.0)\n\n bf = patches.Rectangle([0, 0], 1, 1, color='darkgoldenrod')\n leg = ax.legend([lf, sf, bf],\n ['Long (max: %s, min: %s)' % (df_longs.max(),\n df_longs.min()),\n 'Short (max: %s, min: %s)' % (df_shorts.max(),\n df_shorts.min()),\n 'Overlap'], loc=legend_loc, frameon=True,\n framealpha=0.5)\n leg.get_frame().set_edgecolor('black')\n\n ax.set_xlim((returns.index[0], returns.index[-1]))\n ax.set_title('Long and short holdings')\n ax.set_ylabel('Holdings')\n ax.set_xlabel('')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nplot the cumulative returns of the strategy and noncumulative returns.", "response": "def plot_drawdown_periods(returns, top=10, ax=None, **kwargs):\n \"\"\"\n Plots cumulative returns highlighting top drawdown periods.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n top : int, optional\n Amount of top drawdowns periods to plot (default 10).\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n y_axis_formatter = FuncFormatter(utils.two_dec_places)\n ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))\n\n df_cum_rets = ep.cum_returns(returns, starting_value=1.0)\n df_drawdowns = timeseries.gen_drawdown_table(returns, top=top)\n\n df_cum_rets.plot(ax=ax, **kwargs)\n\n lim = ax.get_ylim()\n colors = sns.cubehelix_palette(len(df_drawdowns))[::-1]\n for i, (peak, recovery) in df_drawdowns[\n ['Peak date', 'Recovery date']].iterrows():\n if pd.isnull(recovery):\n recovery = returns.index[-1]\n ax.fill_between((peak, recovery),\n lim[0],\n lim[1],\n alpha=.4,\n color=colors[i])\n ax.set_ylim(lim)\n ax.set_title('Top %i drawdown periods' % top)\n ax.set_ylabel('Cumulative returns')\n ax.legend(['Portfolio'], loc='upper left',\n frameon=True, framealpha=0.5)\n ax.set_xlabel('')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nplotting the drawdown underwater of the current return for the given strategy.", "response": "def plot_drawdown_underwater(returns, ax=None, **kwargs):\n \"\"\"\n Plots how far underwaterr returns are over time, or plots current\n drawdown vs. date.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n y_axis_formatter = FuncFormatter(utils.percentage)\n ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))\n\n df_cum_rets = ep.cum_returns(returns, starting_value=1.0)\n running_max = np.maximum.accumulate(df_cum_rets)\n underwater = -100 * ((running_max - df_cum_rets) / running_max)\n (underwater).plot(ax=ax, kind='area', color='coral', alpha=0.7, **kwargs)\n ax.set_ylabel('Drawdown')\n ax.set_title('Underwater plot')\n ax.set_xlabel('')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef plot_perf_stats(returns, factor_returns, ax=None):\n\n if ax is None:\n ax = plt.gca()\n\n bootstrap_values = timeseries.perf_stats_bootstrap(returns,\n factor_returns,\n return_stats=False)\n bootstrap_values = bootstrap_values.drop('Kurtosis', axis='columns')\n\n sns.boxplot(data=bootstrap_values, orient='h', ax=ax)\n\n return ax", "response": "Plots some performance metrics of the strategy."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprints some performance metrics for a given strategy.", "response": "def show_perf_stats(returns, factor_returns=None, positions=None,\n transactions=None, turnover_denom='AGB',\n live_start_date=None, bootstrap=False,\n header_rows=None):\n \"\"\"\n Prints some performance metrics of the strategy.\n\n - Shows amount of time the strategy has been run in backtest and\n out-of-sample (in live trading).\n\n - Shows Omega ratio, max drawdown, Calmar ratio, annual return,\n stability, Sharpe ratio, annual volatility, alpha, and beta.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series, optional\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - This is in the same style as returns.\n positions : pd.DataFrame, optional\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n transactions : pd.DataFrame, optional\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet\n turnover_denom : str, optional\n Either AGB or portfolio_value, default AGB.\n - See full explanation in txn.get_turnover.\n live_start_date : datetime, optional\n The point in time when the strategy began live trading, after\n its backtest period.\n bootstrap : boolean, optional\n Whether to perform bootstrap analysis for the performance\n metrics.\n - For more information, see timeseries.perf_stats_bootstrap\n header_rows : dict or OrderedDict, optional\n Extra rows to display at the top of the displayed table.\n \"\"\"\n\n if bootstrap:\n perf_func = timeseries.perf_stats_bootstrap\n else:\n perf_func = timeseries.perf_stats\n\n perf_stats_all = perf_func(\n returns,\n factor_returns=factor_returns,\n positions=positions,\n transactions=transactions,\n turnover_denom=turnover_denom)\n\n date_rows = OrderedDict()\n if len(returns.index) > 0:\n date_rows['Start date'] = returns.index[0].strftime('%Y-%m-%d')\n date_rows['End date'] = returns.index[-1].strftime('%Y-%m-%d')\n\n if live_start_date is not None:\n live_start_date = ep.utils.get_utc_timestamp(live_start_date)\n returns_is = returns[returns.index < live_start_date]\n returns_oos = returns[returns.index >= live_start_date]\n\n positions_is = None\n positions_oos = None\n transactions_is = None\n transactions_oos = None\n\n if positions is not None:\n positions_is = positions[positions.index < live_start_date]\n positions_oos = positions[positions.index >= live_start_date]\n if transactions is not None:\n transactions_is = transactions[(transactions.index <\n live_start_date)]\n transactions_oos = transactions[(transactions.index >\n live_start_date)]\n\n perf_stats_is = perf_func(\n returns_is,\n factor_returns=factor_returns,\n positions=positions_is,\n transactions=transactions_is,\n turnover_denom=turnover_denom)\n\n perf_stats_oos = perf_func(\n returns_oos,\n factor_returns=factor_returns,\n positions=positions_oos,\n transactions=transactions_oos,\n turnover_denom=turnover_denom)\n if len(returns.index) > 0:\n date_rows['In-sample months'] = int(len(returns_is) /\n APPROX_BDAYS_PER_MONTH)\n date_rows['Out-of-sample months'] = int(len(returns_oos) /\n APPROX_BDAYS_PER_MONTH)\n\n perf_stats = pd.concat(OrderedDict([\n ('In-sample', perf_stats_is),\n ('Out-of-sample', perf_stats_oos),\n ('All', perf_stats_all),\n ]), axis=1)\n else:\n if len(returns.index) > 0:\n date_rows['Total months'] = int(len(returns) /\n APPROX_BDAYS_PER_MONTH)\n perf_stats = pd.DataFrame(perf_stats_all, columns=['Backtest'])\n\n for column in perf_stats.columns:\n for stat, value in perf_stats[column].iteritems():\n if stat in STAT_FUNCS_PCT:\n perf_stats.loc[stat, column] = str(np.round(value * 100,\n 1)) + '%'\n if header_rows is None:\n header_rows = date_rows\n else:\n header_rows = OrderedDict(header_rows)\n header_rows.update(date_rows)\n\n utils.print_table(\n perf_stats,\n float_format='{0:.2f}'.format,\n header_rows=header_rows,\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot_returns(returns,\n live_start_date=None,\n ax=None):\n \"\"\"\n Plots raw returns over time.\n\n Backtest returns are in green, and out-of-sample (live trading)\n returns are in red.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n live_start_date : datetime, optional\n The date when the strategy began live trading, after\n its backtest period. This date should be normalized.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n ax.set_label('')\n ax.set_ylabel('Returns')\n\n if live_start_date is not None:\n live_start_date = ep.utils.get_utc_timestamp(live_start_date)\n is_returns = returns.loc[returns.index < live_start_date]\n oos_returns = returns.loc[returns.index >= live_start_date]\n is_returns.plot(ax=ax, color='g')\n oos_returns.plot(ax=ax, color='r')\n\n else:\n returns.plot(ax=ax, color='g')\n\n return ax", "response": "Plots the raw returns over time."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_rolling_returns(returns,\n factor_returns=None,\n live_start_date=None,\n logy=False,\n cone_std=None,\n legend_loc='best',\n volatility_match=False,\n cone_function=timeseries.forecast_cone_bootstrap,\n ax=None, **kwargs):\n \"\"\"\n Plots cumulative rolling returns versus some benchmarks'.\n\n Backtest returns are in green, and out-of-sample (live trading)\n returns are in red.\n\n Additionally, a non-parametric cone plot may be added to the\n out-of-sample returns region.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series, optional\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - This is in the same style as returns.\n live_start_date : datetime, optional\n The date when the strategy began live trading, after\n its backtest period. This date should be normalized.\n logy : bool, optional\n Whether to log-scale the y-axis.\n cone_std : float, or tuple, optional\n If float, The standard deviation to use for the cone plots.\n If tuple, Tuple of standard deviation values to use for the cone plots\n - See timeseries.forecast_cone_bounds for more details.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n volatility_match : bool, optional\n Whether to normalize the volatility of the returns to those of the\n benchmark returns. This helps compare strategies with different\n volatilities. Requires passing of benchmark_rets.\n cone_function : function, optional\n Function to use when generating forecast probability cone.\n The function signiture must follow the form:\n def cone(in_sample_returns (pd.Series),\n days_to_project_forward (int),\n cone_std= (float, or tuple),\n starting_value= (int, or float))\n See timeseries.forecast_cone_bootstrap for an example.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n ax.set_xlabel('')\n ax.set_ylabel('Cumulative returns')\n ax.set_yscale('log' if logy else 'linear')\n\n if volatility_match and factor_returns is None:\n raise ValueError('volatility_match requires passing of '\n 'factor_returns.')\n elif volatility_match and factor_returns is not None:\n bmark_vol = factor_returns.loc[returns.index].std()\n returns = (returns / returns.std()) * bmark_vol\n\n cum_rets = ep.cum_returns(returns, 1.0)\n\n y_axis_formatter = FuncFormatter(utils.two_dec_places)\n ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))\n\n if factor_returns is not None:\n cum_factor_returns = ep.cum_returns(\n factor_returns[cum_rets.index], 1.0)\n cum_factor_returns.plot(lw=2, color='gray',\n label=factor_returns.name, alpha=0.60,\n ax=ax, **kwargs)\n\n if live_start_date is not None:\n live_start_date = ep.utils.get_utc_timestamp(live_start_date)\n is_cum_returns = cum_rets.loc[cum_rets.index < live_start_date]\n oos_cum_returns = cum_rets.loc[cum_rets.index >= live_start_date]\n else:\n is_cum_returns = cum_rets\n oos_cum_returns = pd.Series([])\n\n is_cum_returns.plot(lw=3, color='forestgreen', alpha=0.6,\n label='Backtest', ax=ax, **kwargs)\n\n if len(oos_cum_returns) > 0:\n oos_cum_returns.plot(lw=4, color='red', alpha=0.6,\n label='Live', ax=ax, **kwargs)\n\n if cone_std is not None:\n if isinstance(cone_std, (float, int)):\n cone_std = [cone_std]\n\n is_returns = returns.loc[returns.index < live_start_date]\n cone_bounds = cone_function(\n is_returns,\n len(oos_cum_returns),\n cone_std=cone_std,\n starting_value=is_cum_returns[-1])\n\n cone_bounds = cone_bounds.set_index(oos_cum_returns.index)\n for std in cone_std:\n ax.fill_between(cone_bounds.index,\n cone_bounds[float(std)],\n cone_bounds[float(-std)],\n color='steelblue', alpha=0.5)\n\n if legend_loc is not None:\n ax.legend(loc=legend_loc, frameon=True, framealpha=0.5)\n ax.axhline(1.0, linestyle='--', color='black', lw=2)\n\n return ax", "response": "Plots a series of cumulative rolling returns."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nplot the rolling beta of the given returns.", "response": "def plot_rolling_beta(returns, factor_returns, legend_loc='best',\n ax=None, **kwargs):\n \"\"\"\n Plots the rolling 6-month and 12-month beta versus date.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - This is in the same style as returns.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n y_axis_formatter = FuncFormatter(utils.two_dec_places)\n ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))\n\n ax.set_title(\"Rolling portfolio beta to \" + str(factor_returns.name))\n ax.set_ylabel('Beta')\n rb_1 = timeseries.rolling_beta(\n returns, factor_returns, rolling_window=APPROX_BDAYS_PER_MONTH * 6)\n rb_1.plot(color='steelblue', lw=3, alpha=0.6, ax=ax, **kwargs)\n rb_2 = timeseries.rolling_beta(\n returns, factor_returns, rolling_window=APPROX_BDAYS_PER_MONTH * 12)\n rb_2.plot(color='grey', lw=3, alpha=0.4, ax=ax, **kwargs)\n ax.axhline(rb_1.mean(), color='steelblue', linestyle='--', lw=3)\n ax.axhline(0.0, color='black', linestyle='-', lw=2)\n\n ax.set_xlabel('')\n ax.legend(['6-mo',\n '12-mo'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n ax.set_ylim((-1.0, 1.0))\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nplots the rolling volatility of the given returns.", "response": "def plot_rolling_volatility(returns, factor_returns=None,\n rolling_window=APPROX_BDAYS_PER_MONTH * 6,\n legend_loc='best', ax=None, **kwargs):\n \"\"\"\n Plots the rolling volatility versus date.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series, optional\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - This is in the same style as returns.\n rolling_window : int, optional\n The days window over which to compute the volatility.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n y_axis_formatter = FuncFormatter(utils.two_dec_places)\n ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))\n\n rolling_vol_ts = timeseries.rolling_volatility(\n returns, rolling_window)\n rolling_vol_ts.plot(alpha=.7, lw=3, color='orangered', ax=ax,\n **kwargs)\n if factor_returns is not None:\n rolling_vol_ts_factor = timeseries.rolling_volatility(\n factor_returns, rolling_window)\n rolling_vol_ts_factor.plot(alpha=.7, lw=3, color='grey', ax=ax,\n **kwargs)\n\n ax.set_title('Rolling volatility (6-month)')\n ax.axhline(\n rolling_vol_ts.mean(),\n color='steelblue',\n linestyle='--',\n lw=3)\n\n ax.axhline(0.0, color='black', linestyle='-', lw=2)\n\n ax.set_ylabel('Volatility')\n ax.set_xlabel('')\n if factor_returns is None:\n ax.legend(['Volatility', 'Average volatility'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n else:\n ax.legend(['Volatility', 'Benchmark volatility', 'Average volatility'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n return ax"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_rolling_sharpe(returns, factor_returns=None,\n rolling_window=APPROX_BDAYS_PER_MONTH * 6,\n legend_loc='best', ax=None, **kwargs):\n \"\"\"\n Plots the rolling Sharpe ratio versus date.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series, optional\n Daily noncumulative returns of the benchmark factor for\n which the benchmark rolling Sharpe is computed. Usually\n a benchmark such as market returns.\n - This is in the same style as returns.\n rolling_window : int, optional\n The days window over which to compute the sharpe ratio.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n y_axis_formatter = FuncFormatter(utils.two_dec_places)\n ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))\n\n rolling_sharpe_ts = timeseries.rolling_sharpe(\n returns, rolling_window)\n rolling_sharpe_ts.plot(alpha=.7, lw=3, color='orangered', ax=ax,\n **kwargs)\n\n if factor_returns is not None:\n rolling_sharpe_ts_factor = timeseries.rolling_sharpe(\n factor_returns, rolling_window)\n rolling_sharpe_ts_factor.plot(alpha=.7, lw=3, color='grey', ax=ax,\n **kwargs)\n\n ax.set_title('Rolling Sharpe ratio (6-month)')\n ax.axhline(\n rolling_sharpe_ts.mean(),\n color='steelblue',\n linestyle='--',\n lw=3)\n ax.axhline(0.0, color='black', linestyle='-', lw=3)\n\n ax.set_ylabel('Sharpe ratio')\n ax.set_xlabel('')\n if factor_returns is None:\n ax.legend(['Sharpe', 'Average'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n else:\n ax.legend(['Sharpe', 'Benchmark Sharpe', 'Average'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n\n return ax", "response": "Plots the rolling Sharpe ratio versus date."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nplots the gross leverage of the given returns and positions.", "response": "def plot_gross_leverage(returns, positions, ax=None, **kwargs):\n \"\"\"\n Plots gross leverage versus date.\n\n Gross leverage is the sum of long and short exposure per share\n divided by net asset value.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in create_full_tear_sheet.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n gl = timeseries.gross_lev(positions)\n gl.plot(lw=0.5, color='limegreen', legend=False, ax=ax, **kwargs)\n\n ax.axhline(gl.mean(), color='g', linestyle='--', lw=3)\n\n ax.set_title('Gross leverage')\n ax.set_ylabel('Gross leverage')\n ax.set_xlabel('')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nplots a cake chart of the long and short exposure.", "response": "def plot_exposures(returns, positions, ax=None, **kwargs):\n \"\"\"\n Plots a cake chart of the long and short exposure.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n positions_alloc : pd.DataFrame\n Portfolio allocation of positions. See\n pos.get_percent_alloc.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n pos_no_cash = positions.drop('cash', axis=1)\n l_exp = pos_no_cash[pos_no_cash > 0].sum(axis=1) / positions.sum(axis=1)\n s_exp = pos_no_cash[pos_no_cash < 0].sum(axis=1) / positions.sum(axis=1)\n net_exp = pos_no_cash.sum(axis=1) / positions.sum(axis=1)\n\n ax.fill_between(l_exp.index,\n 0,\n l_exp.values,\n label='Long', color='green', alpha=0.5)\n ax.fill_between(s_exp.index,\n 0,\n s_exp.values,\n label='Short', color='red', alpha=0.5)\n ax.plot(net_exp.index, net_exp.values,\n label='Net', color='black', linestyle='dotted')\n\n ax.set_xlim((returns.index[0], returns.index[-1]))\n ax.set_title(\"Exposure\")\n ax.set_ylabel('Exposure')\n ax.legend(loc='lower left', frameon=True, framealpha=0.5)\n ax.set_xlabel('')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprints and plots the exposures of the top 10 held positions of the current time.", "response": "def show_and_plot_top_positions(returns, positions_alloc,\n show_and_plot=2, hide_positions=False,\n legend_loc='real_best', ax=None,\n **kwargs):\n \"\"\"\n Prints and/or plots the exposures of the top 10 held positions of\n all time.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n positions_alloc : pd.DataFrame\n Portfolio allocation of positions. See pos.get_percent_alloc.\n show_and_plot : int, optional\n By default, this is 2, and both prints and plots.\n If this is 0, it will only plot; if 1, it will only print.\n hide_positions : bool, optional\n If True, will not output any symbol names.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n By default, the legend will display below the plot.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes, conditional\n The axes that were plotted on.\n\n \"\"\"\n positions_alloc = positions_alloc.copy()\n positions_alloc.columns = positions_alloc.columns.map(utils.format_asset)\n\n df_top_long, df_top_short, df_top_abs = pos.get_top_long_short_abs(\n positions_alloc)\n\n if show_and_plot == 1 or show_and_plot == 2:\n utils.print_table(pd.DataFrame(df_top_long * 100, columns=['max']),\n float_format='{0:.2f}%'.format,\n name='Top 10 long positions of all time')\n\n utils.print_table(pd.DataFrame(df_top_short * 100, columns=['max']),\n float_format='{0:.2f}%'.format,\n name='Top 10 short positions of all time')\n\n utils.print_table(pd.DataFrame(df_top_abs * 100, columns=['max']),\n float_format='{0:.2f}%'.format,\n name='Top 10 positions of all time')\n\n if show_and_plot == 0 or show_and_plot == 2:\n\n if ax is None:\n ax = plt.gca()\n\n positions_alloc[df_top_abs.index].plot(\n title='Portfolio allocation over time, only top 10 holdings',\n alpha=0.5, ax=ax, **kwargs)\n\n # Place legend below plot, shrink plot by 20%\n if legend_loc == 'real_best':\n box = ax.get_position()\n ax.set_position([box.x0, box.y0 + box.height * 0.1,\n box.width, box.height * 0.9])\n\n # Put a legend below current axis\n ax.legend(loc='upper center', frameon=True, framealpha=0.5,\n bbox_to_anchor=(0.5, -0.14), ncol=5)\n else:\n ax.legend(loc=legend_loc)\n\n ax.set_xlim((returns.index[0], returns.index[-1]))\n ax.set_ylabel('Exposure by holding')\n\n if hide_positions:\n ax.legend_.remove()\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nplot the max and median of long and short position concentrations over the time.", "response": "def plot_max_median_position_concentration(positions, ax=None, **kwargs):\n \"\"\"\n Plots the max and median of long and short position concentrations\n over the time.\n\n Parameters\n ----------\n positions : pd.DataFrame\n The positions that the strategy takes over time.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n alloc_summary = pos.get_max_median_position_concentration(positions)\n colors = ['mediumblue', 'steelblue', 'tomato', 'firebrick']\n alloc_summary.plot(linewidth=1, color=colors, alpha=0.6, ax=ax)\n\n ax.legend(loc='center left', frameon=True, framealpha=0.5)\n ax.set_ylabel('Exposure')\n ax.set_title('Long/short max and median position concentration')\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nplots the exposures of the portfolio over time.", "response": "def plot_sector_allocations(returns, sector_alloc, ax=None, **kwargs):\n \"\"\"\n Plots the sector exposures of the portfolio over time.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n sector_alloc : pd.DataFrame\n Portfolio allocation of positions. See pos.get_sector_alloc.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n sector_alloc.plot(title='Sector allocation over time',\n alpha=0.5, ax=ax, **kwargs)\n\n box = ax.get_position()\n ax.set_position([box.x0, box.y0 + box.height * 0.1,\n box.width, box.height * 0.9])\n\n # Put a legend below current axis\n ax.legend(loc='upper center', frameon=True, framealpha=0.5,\n bbox_to_anchor=(0.5, -0.14), ncol=5)\n\n ax.set_xlim((sector_alloc.index[0], sector_alloc.index[-1]))\n ax.set_ylabel('Exposure by sector')\n ax.set_xlabel('')\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nplot the quantiles of the daily weekly and monthly return - by - month returns.", "response": "def plot_return_quantiles(returns, live_start_date=None, ax=None, **kwargs):\n \"\"\"\n Creates a box plot of daily, weekly, and monthly return\n distributions.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n live_start_date : datetime, optional\n The point in time when the strategy began live trading, after\n its backtest period.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to seaborn plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n is_returns = returns if live_start_date is None \\\n else returns.loc[returns.index < live_start_date]\n is_weekly = ep.aggregate_returns(is_returns, 'weekly')\n is_monthly = ep.aggregate_returns(is_returns, 'monthly')\n sns.boxplot(data=[is_returns, is_weekly, is_monthly],\n palette=[\"#4c72B0\", \"#55A868\", \"#CCB974\"],\n ax=ax, **kwargs)\n\n if live_start_date is not None:\n oos_returns = returns.loc[returns.index >= live_start_date]\n oos_weekly = ep.aggregate_returns(oos_returns, 'weekly')\n oos_monthly = ep.aggregate_returns(oos_returns, 'monthly')\n\n sns.swarmplot(data=[oos_returns, oos_weekly, oos_monthly], ax=ax,\n color=\"red\",\n marker=\"d\", **kwargs)\n red_dots = matplotlib.lines.Line2D([], [], color=\"red\", marker=\"d\",\n label=\"Out-of-sample data\",\n linestyle='')\n ax.legend(handles=[red_dots], frameon=True, framealpha=0.5)\n ax.set_xticklabels(['Daily', 'Weekly', 'Monthly'])\n ax.set_title('Return quantiles')\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nplot the turnover of the given returns transactions and positions.", "response": "def plot_turnover(returns, transactions, positions,\n legend_loc='best', ax=None, **kwargs):\n \"\"\"\n Plots turnover vs. date.\n\n Turnover is the number of shares traded for a period as a fraction\n of total shares.\n\n Displays daily total, daily average per month, and all-time daily\n average.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n legend_loc : matplotlib.loc, optional\n The location of the legend on the plot.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n y_axis_formatter = FuncFormatter(utils.two_dec_places)\n ax.yaxis.set_major_formatter(FuncFormatter(y_axis_formatter))\n\n df_turnover = txn.get_turnover(positions, transactions)\n df_turnover_by_month = df_turnover.resample(\"M\").mean()\n df_turnover.plot(color='steelblue', alpha=1.0, lw=0.5, ax=ax, **kwargs)\n df_turnover_by_month.plot(\n color='orangered',\n alpha=0.5,\n lw=2,\n ax=ax,\n **kwargs)\n ax.axhline(\n df_turnover.mean(), color='steelblue', linestyle='--', lw=3, alpha=1.0)\n ax.legend(['Daily turnover',\n 'Average daily turnover, by month',\n 'Average daily turnover, net'],\n loc=legend_loc, frameon=True, framealpha=0.5)\n ax.set_title('Daily turnover')\n ax.set_xlim((returns.index[0], returns.index[-1]))\n ax.set_ylim((0, 2))\n ax.set_ylabel('Turnover')\n ax.set_xlabel('')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot_slippage_sweep(returns, positions, transactions,\n slippage_params=(3, 8, 10, 12, 15, 20, 50),\n ax=None, **kwargs):\n \"\"\"\n Plots equity curves at different per-dollar slippage assumptions.\n\n Parameters\n ----------\n returns : pd.Series\n Timeseries of portfolio returns to be adjusted for various\n degrees of slippage.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet.\n slippage_params: tuple\n Slippage pameters to apply to the return time series (in\n basis points).\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to seaborn plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n slippage_sweep = pd.DataFrame()\n for bps in slippage_params:\n adj_returns = txn.adjust_returns_for_slippage(returns, positions,\n transactions, bps)\n label = str(bps) + \" bps\"\n slippage_sweep[label] = ep.cum_returns(adj_returns, 1)\n\n slippage_sweep.plot(alpha=1.0, lw=0.5, ax=ax)\n\n ax.set_title('Cumulative returns given additional per-dollar slippage')\n ax.set_ylabel('')\n\n ax.legend(loc='center left', frameon=True, framealpha=0.5)\n\n return ax", "response": "Plots the slippage sweep curves at different per - dollar slippage assumptions."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nplotting the sensitivity of a single slippage.", "response": "def plot_slippage_sensitivity(returns, positions, transactions,\n ax=None, **kwargs):\n \"\"\"\n Plots curve relating per-dollar slippage to average annual returns.\n\n Parameters\n ----------\n returns : pd.Series\n Timeseries of portfolio returns to be adjusted for various\n degrees of slippage.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to seaborn plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n avg_returns_given_slippage = pd.Series()\n for bps in range(1, 100):\n adj_returns = txn.adjust_returns_for_slippage(returns, positions,\n transactions, bps)\n avg_returns = ep.annual_return(adj_returns)\n avg_returns_given_slippage.loc[bps] = avg_returns\n\n avg_returns_given_slippage.plot(alpha=1.0, lw=2, ax=ax)\n\n ax.set_title('Average annual returns given additional per-dollar slippage')\n ax.set_xticks(np.arange(0, 100, 10))\n ax.set_ylabel('Average annual return')\n ax.set_xlabel('Per-dollar slippage (bps)')\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nplot a histogram of daily turnover rates.", "response": "def plot_daily_turnover_hist(transactions, positions,\n ax=None, **kwargs):\n \"\"\"\n Plots a histogram of daily turnover rates.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to seaborn plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n turnover = txn.get_turnover(positions, transactions)\n sns.distplot(turnover, ax=ax, **kwargs)\n ax.set_title('Distribution of daily turnover rates')\n ax.set_xlabel('Turnover rate')\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef plot_daily_volume(returns, transactions, ax=None, **kwargs):\n\n if ax is None:\n ax = plt.gca()\n daily_txn = txn.get_txn_vol(transactions)\n daily_txn.txn_shares.plot(alpha=1.0, lw=0.5, ax=ax, **kwargs)\n ax.axhline(daily_txn.txn_shares.mean(), color='steelblue',\n linestyle='--', lw=3, alpha=1.0)\n ax.set_title('Daily trading volume')\n ax.set_xlim((returns.index[0], returns.index[-1]))\n ax.set_ylabel('Amount of shares traded')\n ax.set_xlabel('')\n return ax", "response": "Plots the daily trading volume of the given returns and transactions."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plot_txn_time_hist(transactions, bin_minutes=5, tz='America/New_York',\n ax=None, **kwargs):\n \"\"\"\n Plots a histogram of transaction times, binning the times into\n buckets of a given duration.\n\n Parameters\n ----------\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet.\n bin_minutes : float, optional\n Sizes of the bins in minutes, defaults to 5 minutes.\n tz : str, optional\n Time zone to plot against. Note that if the specified\n zone does not apply daylight savings, the distribution\n may be partially offset.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n **kwargs, optional\n Passed to plotting function.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n txn_time = transactions.copy()\n\n txn_time.index = txn_time.index.tz_convert(pytz.timezone(tz))\n txn_time.index = txn_time.index.map(lambda x: x.hour * 60 + x.minute)\n txn_time['trade_value'] = (txn_time.amount * txn_time.price).abs()\n txn_time = txn_time.groupby(level=0).sum().reindex(index=range(570, 961))\n txn_time.index = (txn_time.index / bin_minutes).astype(int) * bin_minutes\n txn_time = txn_time.groupby(level=0).sum()\n\n txn_time['time_str'] = txn_time.index.map(lambda x:\n str(datetime.time(int(x / 60),\n x % 60))[:-3])\n\n trade_value_sum = txn_time.trade_value.sum()\n txn_time.trade_value = txn_time.trade_value.fillna(0) / trade_value_sum\n\n ax.bar(txn_time.index, txn_time.trade_value, width=bin_minutes, **kwargs)\n\n ax.set_xlim(570, 960)\n ax.set_xticks(txn_time.index[::int(30 / bin_minutes)])\n ax.set_xticklabels(txn_time.time_str[::int(30 / bin_minutes)])\n ax.set_title('Transaction time distribution')\n ax.set_ylabel('Proportion')\n ax.set_xlabel('')\n return ax", "response": "Plots a histogram of transaction times."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprint information about the worst drawdown periods.", "response": "def show_worst_drawdown_periods(returns, top=5):\n \"\"\"\n Prints information about the worst drawdown periods.\n\n Prints peak dates, valley dates, recovery dates, and net\n drawdowns.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n top : int, optional\n Amount of top drawdowns periods to plot (default 5).\n \"\"\"\n\n drawdown_df = timeseries.gen_drawdown_table(returns, top=top)\n utils.print_table(\n drawdown_df.sort_values('Net drawdown in %', ascending=False),\n name='Worst drawdown periods',\n float_format='{0:.2f}'.format,\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef plot_monthly_returns_timeseries(returns, ax=None, **kwargs):\n\n def cumulate_returns(x):\n return ep.cum_returns(x)[-1]\n\n if ax is None:\n ax = plt.gca()\n\n monthly_rets = returns.resample('M').apply(lambda x: cumulate_returns(x))\n monthly_rets = monthly_rets.to_period()\n\n sns.barplot(x=monthly_rets.index,\n y=monthly_rets.values,\n color='steelblue')\n\n locs, labels = plt.xticks()\n plt.setp(labels, rotation=90)\n\n # only show x-labels on year boundary\n xticks_coord = []\n xticks_label = []\n count = 0\n for i in monthly_rets.index:\n if i.month == 1:\n xticks_label.append(i)\n xticks_coord.append(count)\n # plot yearly boundary line\n ax.axvline(count, color='gray', ls='--', alpha=0.3)\n\n count += 1\n\n ax.axhline(0.0, color='darkgray', ls='-')\n ax.set_xticks(xticks_coord)\n ax.set_xticklabels(xticks_label)\n\n return ax", "response": "Plots monthly returns as a timeseries."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plot_round_trip_lifetimes(round_trips, disp_amount=16, lsize=18, ax=None):\n\n if ax is None:\n ax = plt.subplot()\n\n symbols_sample = round_trips.symbol.unique()\n np.random.seed(1)\n sample = np.random.choice(round_trips.symbol.unique(), replace=False,\n size=min(disp_amount, len(symbols_sample)))\n sample_round_trips = round_trips[round_trips.symbol.isin(sample)]\n\n symbol_idx = pd.Series(np.arange(len(sample)), index=sample)\n\n for symbol, sym_round_trips in sample_round_trips.groupby('symbol'):\n for _, row in sym_round_trips.iterrows():\n c = 'b' if row.long else 'r'\n y_ix = symbol_idx[symbol] + 0.05\n ax.plot([row['open_dt'], row['close_dt']],\n [y_ix, y_ix], color=c,\n linewidth=lsize, solid_capstyle='butt')\n\n ax.set_yticks(range(disp_amount))\n ax.set_yticklabels([utils.format_asset(s) for s in sample])\n\n ax.set_ylim((-0.5, min(len(sample), disp_amount) - 0.5))\n blue = patches.Rectangle([0, 0], 1, 1, color='b', label='Long')\n red = patches.Rectangle([0, 0], 1, 1, color='r', label='Short')\n leg = ax.legend(handles=[blue, red], loc='lower left',\n frameon=True, framealpha=0.5)\n leg.get_frame().set_edgecolor('black')\n ax.grid(False)\n\n return ax", "response": "Plots the timespans and directions of a sample of round trades."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprints the share of total PnL contributed by each .", "response": "def show_profit_attribution(round_trips):\n \"\"\"\n Prints the share of total PnL contributed by each\n traded name.\n\n Parameters\n ----------\n round_trips : pd.DataFrame\n DataFrame with one row per round trip trade.\n - See full explanation in round_trips.extract_round_trips\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n total_pnl = round_trips['pnl'].sum()\n pnl_attribution = round_trips.groupby('symbol')['pnl'].sum() / total_pnl\n pnl_attribution.name = ''\n\n pnl_attribution.index = pnl_attribution.index.map(utils.format_asset)\n utils.print_table(\n pnl_attribution.sort_values(\n inplace=False,\n ascending=False,\n ),\n name='Profitability (PnL / PnL total) per name',\n float_format='{:.2%}'.format,\n )"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nplotting a probability distribution for the event of making a profitable trade.", "response": "def plot_prob_profit_trade(round_trips, ax=None):\n \"\"\"\n Plots a probability distribution for the event of making\n a profitable trade.\n\n Parameters\n ----------\n round_trips : pd.DataFrame\n DataFrame with one row per round trip trade.\n - See full explanation in round_trips.extract_round_trips\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n\n Returns\n -------\n ax : matplotlib.Axes\n The axes that were plotted on.\n \"\"\"\n\n x = np.linspace(0, 1., 500)\n\n round_trips['profitable'] = round_trips.pnl > 0\n\n dist = sp.stats.beta(round_trips.profitable.sum(),\n (~round_trips.profitable).sum())\n y = dist.pdf(x)\n lower_perc = dist.ppf(.025)\n upper_perc = dist.ppf(.975)\n\n lower_plot = dist.ppf(.001)\n upper_plot = dist.ppf(.999)\n\n if ax is None:\n ax = plt.subplot()\n\n ax.plot(x, y)\n ax.axvline(lower_perc, color='0.5')\n ax.axvline(upper_perc, color='0.5')\n\n ax.set_xlabel('Probability of making a profitable decision')\n ax.set_ylabel('Belief')\n ax.set_xlim(lower_plot, upper_plot)\n ax.set_ylim((0, y.max() + 1.))\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_cones(name, bounds, oos_returns, num_samples=1000, ax=None,\n cone_std=(1., 1.5, 2.), random_seed=None, num_strikes=3):\n \"\"\"\n Plots the upper and lower bounds of an n standard deviation\n cone of forecasted cumulative returns. Redraws a new cone when\n cumulative returns fall outside of last cone drawn.\n\n Parameters\n ----------\n name : str\n Account name to be used as figure title.\n bounds : pandas.core.frame.DataFrame\n Contains upper and lower cone boundaries. Column names are\n strings corresponding to the number of standard devations\n above (positive) or below (negative) the projected mean\n cumulative returns.\n oos_returns : pandas.core.frame.DataFrame\n Non-cumulative out-of-sample returns.\n num_samples : int\n Number of samples to draw from the in-sample daily returns.\n Each sample will be an array with length num_days.\n A higher number of samples will generate a more accurate\n bootstrap cone.\n ax : matplotlib.Axes, optional\n Axes upon which to plot.\n cone_std : list of int/float\n Number of standard devations to use in the boundaries of\n the cone. If multiple values are passed, cone bounds will\n be generated for each value.\n random_seed : int\n Seed for the pseudorandom number generator used by the pandas\n sample method.\n num_strikes : int\n Upper limit for number of cones drawn. Can be anything from 0 to 3.\n\n Returns\n -------\n Returns are either an ax or fig option, but not both. If a\n matplotlib.Axes instance is passed in as ax, then it will be modified\n and returned. This allows for users to plot interactively in jupyter\n notebook. When no ax object is passed in, a matplotlib.figure instance\n is generated and returned. This figure can then be used to save\n the plot as an image without viewing it.\n\n ax : matplotlib.Axes\n The axes that were plotted on.\n fig : matplotlib.figure\n The figure instance which contains all the plot elements.\n \"\"\"\n\n if ax is None:\n fig = figure.Figure(figsize=(10, 8))\n FigureCanvasAgg(fig)\n axes = fig.add_subplot(111)\n else:\n axes = ax\n\n returns = ep.cum_returns(oos_returns, starting_value=1.)\n bounds_tmp = bounds.copy()\n returns_tmp = returns.copy()\n cone_start = returns.index[0]\n colors = [\"green\", \"orange\", \"orangered\", \"darkred\"]\n\n for c in range(num_strikes + 1):\n if c > 0:\n tmp = returns.loc[cone_start:]\n bounds_tmp = bounds_tmp.iloc[0:len(tmp)]\n bounds_tmp = bounds_tmp.set_index(tmp.index)\n crossing = (tmp < bounds_tmp[float(-2.)].iloc[:len(tmp)])\n if crossing.sum() <= 0:\n break\n cone_start = crossing.loc[crossing].index[0]\n returns_tmp = returns.loc[cone_start:]\n bounds_tmp = (bounds - (1 - returns.loc[cone_start]))\n for std in cone_std:\n x = returns_tmp.index\n y1 = bounds_tmp[float(std)].iloc[:len(returns_tmp)]\n y2 = bounds_tmp[float(-std)].iloc[:len(returns_tmp)]\n axes.fill_between(x, y1, y2, color=colors[c], alpha=0.5)\n\n # Plot returns line graph\n label = 'Cumulative returns = {:.2f}%'.format((returns.iloc[-1] - 1) * 100)\n axes.plot(returns.index, returns.values, color='black', lw=3.,\n label=label)\n\n if name is not None:\n axes.set_title(name)\n axes.axhline(1, color='black', alpha=0.2)\n axes.legend(frameon=True, framealpha=0.5)\n\n if ax is None:\n return fig\n else:\n return axes", "response": "Plots a single bootstrap cone in a single plot."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sortino_ratio(returns, required_return=0, period=DAILY):\n\n return ep.sortino_ratio(returns, required_return=required_return)", "response": "Determines the Sortino ratio of a strategy."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the downside deviation below a threshold.", "response": "def downside_risk(returns, required_return=0, period=DAILY):\n \"\"\"\n Determines the downside deviation below a threshold\n\n Parameters\n ----------\n returns : pd.Series or pd.DataFrame\n Daily returns of the strategy, noncumulative.\n - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.\n required_return: float / series\n minimum acceptable return\n period : str, optional\n Defines the periodicity of the 'returns' data for purposes of\n annualizing. Can be 'monthly', 'weekly', or 'daily'.\n - Defaults to 'daily'.\n\n Returns\n -------\n depends on input type\n series ==> float\n DataFrame ==> np.array\n\n Annualized downside deviation\n \"\"\"\n\n return ep.downside_risk(returns,\n required_return=required_return,\n period=period)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sharpe_ratio(returns, risk_free=0, period=DAILY):\n\n return ep.sharpe_ratio(returns, risk_free=risk_free, period=period)", "response": "Returns the Sharpe ratio of a strategy."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef rolling_beta(returns, factor_returns,\n rolling_window=APPROX_BDAYS_PER_MONTH * 6):\n \"\"\"\n Determines the rolling beta of a strategy.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series or pd.DataFrame\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - If DataFrame is passed, computes rolling beta for each column.\n - This is in the same style as returns.\n rolling_window : int, optional\n The size of the rolling window, in days, over which to compute\n beta (default 6 months).\n\n Returns\n -------\n pd.Series\n Rolling beta.\n\n Note\n -----\n See https://en.wikipedia.org/wiki/Beta_(finance) for more details.\n \"\"\"\n\n if factor_returns.ndim > 1:\n # Apply column-wise\n return factor_returns.apply(partial(rolling_beta, returns),\n rolling_window=rolling_window)\n else:\n out = pd.Series(index=returns.index)\n for beg, end in zip(returns.index[0:-rolling_window],\n returns.index[rolling_window:]):\n out.loc[end] = ep.beta(\n returns.loc[beg:end],\n factor_returns.loc[beg:end])\n\n return out", "response": "Computes the rolling beta of a given strategy."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef rolling_regression(returns, factor_returns,\n rolling_window=APPROX_BDAYS_PER_MONTH * 6,\n nan_threshold=0.1):\n \"\"\"\n Computes rolling factor betas using a multivariate linear regression\n (separate linear regressions is problematic because the factors may be\n confounded).\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.DataFrame\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - Computes rolling beta for each column.\n - This is in the same style as returns.\n rolling_window : int, optional\n The days window over which to compute the beta. Defaults to 6 months.\n nan_threshold : float, optional\n If there are more than this fraction of NaNs, the rolling regression\n for the given date will be skipped.\n\n Returns\n -------\n pandas.DataFrame\n DataFrame containing rolling beta coefficients to SMB, HML and UMD\n \"\"\"\n\n # We need to drop NaNs to regress\n ret_no_na = returns.dropna()\n\n columns = ['alpha'] + factor_returns.columns.tolist()\n rolling_risk = pd.DataFrame(columns=columns,\n index=ret_no_na.index)\n\n rolling_risk.index.name = 'dt'\n\n for beg, end in zip(ret_no_na.index[:-rolling_window],\n ret_no_na.index[rolling_window:]):\n returns_period = ret_no_na[beg:end]\n factor_returns_period = factor_returns.loc[returns_period.index]\n\n if np.all(factor_returns_period.isnull().mean()) < nan_threshold:\n factor_returns_period_dnan = factor_returns_period.dropna()\n reg = linear_model.LinearRegression(fit_intercept=True).fit(\n factor_returns_period_dnan,\n returns_period.loc[factor_returns_period_dnan.index])\n rolling_risk.loc[end, factor_returns.columns] = reg.coef_\n rolling_risk.loc[end, 'alpha'] = reg.intercept_\n\n return rolling_risk", "response": "Computes a rolling regression of the given returns."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef gross_lev(positions):\n\n exposure = positions.drop('cash', axis=1).abs().sum(axis=1)\n return exposure / positions.sum(axis=1)", "response": "Calculates the gross leverage of a strategy."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef value_at_risk(returns, period=None, sigma=2.0):\n if period is not None:\n returns_agg = ep.aggregate_returns(returns, period)\n else:\n returns_agg = returns.copy()\n\n value_at_risk = returns_agg.mean() - sigma * returns_agg.std()\n return value_at_risk", "response": "Get value at risk ( VaR."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating various performance metrics for a given strategy.", "response": "def perf_stats(returns, factor_returns=None, positions=None,\n transactions=None, turnover_denom='AGB'):\n \"\"\"\n Calculates various performance metrics of a strategy, for use in\n plotting.show_perf_stats.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series, optional\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - This is in the same style as returns.\n - If None, do not compute alpha, beta, and information ratio.\n positions : pd.DataFrame\n Daily net position values.\n - See full explanation in tears.create_full_tear_sheet.\n transactions : pd.DataFrame\n Prices and amounts of executed trades. One row per trade.\n - See full explanation in tears.create_full_tear_sheet.\n turnover_denom : str\n Either AGB or portfolio_value, default AGB.\n - See full explanation in txn.get_turnover.\n\n Returns\n -------\n pd.Series\n Performance metrics.\n \"\"\"\n\n stats = pd.Series()\n for stat_func in SIMPLE_STAT_FUNCS:\n stats[STAT_FUNC_NAMES[stat_func.__name__]] = stat_func(returns)\n\n if positions is not None:\n stats['Gross leverage'] = gross_lev(positions).mean()\n if transactions is not None:\n stats['Daily turnover'] = get_turnover(positions,\n transactions,\n turnover_denom).mean()\n if factor_returns is not None:\n for stat_func in FACTOR_STAT_FUNCS:\n res = stat_func(returns, factor_returns)\n stats[STAT_FUNC_NAMES[stat_func.__name__]] = res\n\n return stats"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating various bootstrapped performance metrics for a particular strategy.", "response": "def perf_stats_bootstrap(returns, factor_returns=None, return_stats=True,\n **kwargs):\n \"\"\"Calculates various bootstrapped performance metrics of a strategy.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n factor_returns : pd.Series, optional\n Daily noncumulative returns of the benchmark factor to which betas are\n computed. Usually a benchmark such as market returns.\n - This is in the same style as returns.\n - If None, do not compute alpha, beta, and information ratio.\n return_stats : boolean (optional)\n If True, returns a DataFrame of mean, median, 5 and 95 percentiles\n for each perf metric.\n If False, returns a DataFrame with the bootstrap samples for\n each perf metric.\n\n Returns\n -------\n pd.DataFrame\n if return_stats is True:\n - Distributional statistics of bootstrapped sampling\n distribution of performance metrics.\n if return_stats is False:\n - Bootstrap samples for each performance metric.\n \"\"\"\n\n bootstrap_values = OrderedDict()\n\n for stat_func in SIMPLE_STAT_FUNCS:\n stat_name = STAT_FUNC_NAMES[stat_func.__name__]\n bootstrap_values[stat_name] = calc_bootstrap(stat_func,\n returns)\n\n if factor_returns is not None:\n for stat_func in FACTOR_STAT_FUNCS:\n stat_name = STAT_FUNC_NAMES[stat_func.__name__]\n bootstrap_values[stat_name] = calc_bootstrap(\n stat_func,\n returns,\n factor_returns=factor_returns)\n\n bootstrap_values = pd.DataFrame(bootstrap_values)\n\n if return_stats:\n stats = bootstrap_values.apply(calc_distribution_stats)\n return stats.T[['mean', 'median', '5%', '95%']]\n else:\n return bootstrap_values"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_bootstrap(func, returns, *args, **kwargs):\n\n n_samples = kwargs.pop('n_samples', 1000)\n out = np.empty(n_samples)\n\n factor_returns = kwargs.pop('factor_returns', None)\n\n for i in range(n_samples):\n idx = np.random.randint(len(returns), size=len(returns))\n returns_i = returns.iloc[idx].reset_index(drop=True)\n if factor_returns is not None:\n factor_returns_i = factor_returns.iloc[idx].reset_index(drop=True)\n out[i] = func(returns_i, factor_returns_i,\n *args, **kwargs)\n else:\n out[i] = func(returns_i,\n *args, **kwargs)\n\n return out", "response": "Calculates a bootstrap distribution of a user - defined function returning a new tree of items."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates various summary statistics of data.", "response": "def calc_distribution_stats(x):\n \"\"\"Calculate various summary statistics of data.\n\n Parameters\n ----------\n x : numpy.ndarray or pandas.Series\n Array to compute summary statistics for.\n\n Returns\n -------\n pandas.Series\n Series containing mean, median, std, as well as 5, 25, 75 and\n 95 percentiles of passed in values.\n \"\"\"\n\n return pd.Series({'mean': np.mean(x),\n 'median': np.median(x),\n 'std': np.std(x),\n '5%': np.percentile(x, 5),\n '25%': np.percentile(x, 25),\n '75%': np.percentile(x, 75),\n '95%': np.percentile(x, 95),\n 'IQR': np.subtract.reduce(\n np.percentile(x, [75, 25])),\n })"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_max_drawdown_underwater(underwater):\n\n valley = np.argmin(underwater) # end of the period\n # Find first 0\n peak = underwater[:valley][underwater[:valley] == 0].index[-1]\n # Find last 0\n try:\n recovery = underwater[valley:][underwater[valley:] == 0].index[0]\n except IndexError:\n recovery = np.nan # drawdown not recovered\n return peak, valley, recovery", "response": "Determines peak valley and recovery dates given an underwater DataFrame."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_max_drawdown(returns):\n\n returns = returns.copy()\n df_cum = cum_returns(returns, 1.0)\n running_max = np.maximum.accumulate(df_cum)\n underwater = df_cum / running_max - 1\n return get_max_drawdown_underwater(underwater)", "response": "Determines the maximum drawdown of a strategy."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_top_drawdowns(returns, top=10):\n\n returns = returns.copy()\n df_cum = ep.cum_returns(returns, 1.0)\n running_max = np.maximum.accumulate(df_cum)\n underwater = df_cum / running_max - 1\n\n drawdowns = []\n for t in range(top):\n peak, valley, recovery = get_max_drawdown_underwater(underwater)\n # Slice out draw-down period\n if not pd.isnull(recovery):\n underwater.drop(underwater[peak: recovery].index[1:-1],\n inplace=True)\n else:\n # drawdown has not ended yet\n underwater = underwater.loc[:peak]\n\n drawdowns.append((peak, valley, recovery))\n if (len(returns) == 0) or (len(underwater) == 0):\n break\n\n return drawdowns", "response": "Returns a list of top drawdowns sorted by drawdown amount."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate a table of top drawdowns.", "response": "def gen_drawdown_table(returns, top=10):\n \"\"\"\n Places top drawdowns in a table.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n top : int, optional\n The amount of top drawdowns to find (default 10).\n\n Returns\n -------\n df_drawdowns : pd.DataFrame\n Information about top drawdowns.\n \"\"\"\n\n df_cum = ep.cum_returns(returns, 1.0)\n drawdown_periods = get_top_drawdowns(returns, top=top)\n df_drawdowns = pd.DataFrame(index=list(range(top)),\n columns=['Net drawdown in %',\n 'Peak date',\n 'Valley date',\n 'Recovery date',\n 'Duration'])\n\n for i, (peak, valley, recovery) in enumerate(drawdown_periods):\n if pd.isnull(recovery):\n df_drawdowns.loc[i, 'Duration'] = np.nan\n else:\n df_drawdowns.loc[i, 'Duration'] = len(pd.date_range(peak,\n recovery,\n freq='B'))\n df_drawdowns.loc[i, 'Peak date'] = (peak.to_pydatetime()\n .strftime('%Y-%m-%d'))\n df_drawdowns.loc[i, 'Valley date'] = (valley.to_pydatetime()\n .strftime('%Y-%m-%d'))\n if isinstance(recovery, float):\n df_drawdowns.loc[i, 'Recovery date'] = recovery\n else:\n df_drawdowns.loc[i, 'Recovery date'] = (recovery.to_pydatetime()\n .strftime('%Y-%m-%d'))\n df_drawdowns.loc[i, 'Net drawdown in %'] = (\n (df_cum.loc[peak] - df_cum.loc[valley]) / df_cum.loc[peak]) * 100\n\n df_drawdowns['Peak date'] = pd.to_datetime(df_drawdowns['Peak date'])\n df_drawdowns['Valley date'] = pd.to_datetime(df_drawdowns['Valley date'])\n df_drawdowns['Recovery date'] = pd.to_datetime(\n df_drawdowns['Recovery date'])\n\n return df_drawdowns"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef rolling_volatility(returns, rolling_vol_window):\n\n return returns.rolling(rolling_vol_window).std() \\\n * np.sqrt(APPROX_BDAYS_PER_YEAR)", "response": "Determines the rolling volatility of a strategy."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndetermining the rolling Sharpe ratio of a strategy.", "response": "def rolling_sharpe(returns, rolling_sharpe_window):\n \"\"\"\n Determines the rolling Sharpe ratio of a strategy.\n\n Parameters\n ----------\n returns : pd.Series\n Daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n rolling_sharpe_window : int\n Length of rolling window, in days, over which to compute.\n\n Returns\n -------\n pd.Series\n Rolling Sharpe ratio.\n\n Note\n -----\n See https://en.wikipedia.org/wiki/Sharpe_ratio for more details.\n \"\"\"\n\n return returns.rolling(rolling_sharpe_window).mean() \\\n / returns.rolling(rolling_sharpe_window).std() \\\n * np.sqrt(APPROX_BDAYS_PER_YEAR)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef simulate_paths(is_returns, num_days,\n starting_value=1, num_samples=1000, random_seed=None):\n \"\"\"\n Gnerate alternate paths using available values from in-sample returns.\n\n Parameters\n ----------\n is_returns : pandas.core.frame.DataFrame\n Non-cumulative in-sample returns.\n num_days : int\n Number of days to project the probability cone forward.\n starting_value : int or float\n Starting value of the out of sample period.\n num_samples : int\n Number of samples to draw from the in-sample daily returns.\n Each sample will be an array with length num_days.\n A higher number of samples will generate a more accurate\n bootstrap cone.\n random_seed : int\n Seed for the pseudorandom number generator used by the pandas\n sample method.\n\n Returns\n -------\n samples : numpy.ndarray\n \"\"\"\n\n samples = np.empty((num_samples, num_days))\n seed = np.random.RandomState(seed=random_seed)\n for i in range(num_samples):\n samples[i, :] = is_returns.sample(num_days, replace=True,\n random_state=seed)\n\n return samples", "response": "Simulate alternate paths using the given number of days and a random number generator."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsummarizes the paths of a given set of possible outcomes.", "response": "def summarize_paths(samples, cone_std=(1., 1.5, 2.), starting_value=1.):\n \"\"\"\n Gnerate the upper and lower bounds of an n standard deviation\n cone of forecasted cumulative returns.\n\n Parameters\n ----------\n samples : numpy.ndarray\n Alternative paths, or series of possible outcomes.\n cone_std : list of int/float\n Number of standard devations to use in the boundaries of\n the cone. If multiple values are passed, cone bounds will\n be generated for each value.\n\n Returns\n -------\n samples : pandas.core.frame.DataFrame\n \"\"\"\n\n cum_samples = ep.cum_returns(samples.T,\n starting_value=starting_value).T\n\n cum_mean = cum_samples.mean(axis=0)\n cum_std = cum_samples.std(axis=0)\n\n if isinstance(cone_std, (float, int)):\n cone_std = [cone_std]\n\n cone_bounds = pd.DataFrame(columns=pd.Float64Index([]))\n for num_std in cone_std:\n cone_bounds.loc[:, float(num_std)] = cum_mean + cum_std * num_std\n cone_bounds.loc[:, float(-num_std)] = cum_mean - cum_std * num_std\n\n return cone_bounds"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef forecast_cone_bootstrap(is_returns, num_days, cone_std=(1., 1.5, 2.),\n starting_value=1, num_samples=1000,\n random_seed=None):\n \"\"\"\n Determines the upper and lower bounds of an n standard deviation\n cone of forecasted cumulative returns. Future cumulative mean and\n standard devation are computed by repeatedly sampling from the\n in-sample daily returns (i.e. bootstrap). This cone is non-parametric,\n meaning it does not assume that returns are normally distributed.\n\n Parameters\n ----------\n is_returns : pd.Series\n In-sample daily returns of the strategy, noncumulative.\n - See full explanation in tears.create_full_tear_sheet.\n num_days : int\n Number of days to project the probability cone forward.\n cone_std : int, float, or list of int/float\n Number of standard devations to use in the boundaries of\n the cone. If multiple values are passed, cone bounds will\n be generated for each value.\n starting_value : int or float\n Starting value of the out of sample period.\n num_samples : int\n Number of samples to draw from the in-sample daily returns.\n Each sample will be an array with length num_days.\n A higher number of samples will generate a more accurate\n bootstrap cone.\n random_seed : int\n Seed for the pseudorandom number generator used by the pandas\n sample method.\n\n Returns\n -------\n pd.DataFrame\n Contains upper and lower cone boundaries. Column names are\n strings corresponding to the number of standard devations\n above (positive) or below (negative) the projected mean\n cumulative returns.\n \"\"\"\n\n samples = simulate_paths(\n is_returns=is_returns,\n num_days=num_days,\n starting_value=starting_value,\n num_samples=num_samples,\n random_seed=random_seed\n )\n\n cone_bounds = summarize_paths(\n samples=samples,\n cone_std=cone_std,\n starting_value=starting_value\n )\n\n return cone_bounds", "response": "Returns a new DataFrame that contains the bootstrapped cumulative returns for a given cone."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef extract_interesting_date_ranges(returns):\n\n returns_dupe = returns.copy()\n returns_dupe.index = returns_dupe.index.map(pd.Timestamp)\n ranges = OrderedDict()\n for name, (start, end) in PERIODS.items():\n try:\n period = returns_dupe.loc[start:end]\n if len(period) == 0:\n continue\n ranges[name] = period\n except BaseException:\n continue\n\n return ranges", "response": "Extracts returns based on interesting events."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns Bayesian alpha - beta - model with T distributed returns.", "response": "def model_returns_t_alpha_beta(data, bmark, samples=2000, progressbar=True):\n \"\"\"\n Run Bayesian alpha-beta-model with T distributed returns.\n\n This model estimates intercept (alpha) and slope (beta) of two\n return sets. Usually, these will be algorithm returns and\n benchmark returns (e.g. S&P500). The data is assumed to be T\n distributed and thus is robust to outliers and takes tail events\n into account. If a pandas.DataFrame is passed as a benchmark, then\n multiple linear regression is used to estimate alpha and beta.\n\n Parameters\n ----------\n returns : pandas.Series\n Series of simple returns of an algorithm or stock.\n bmark : pandas.DataFrame\n DataFrame of benchmark returns (e.g., S&P500) or risk factors (e.g.,\n Fama-French SMB, HML, and UMD).\n If bmark has more recent returns than returns_train, these dates\n will be treated as missing values and predictions will be\n generated for them taking market correlations into account.\n samples : int (optional)\n Number of posterior samples to draw.\n\n Returns\n -------\n model : pymc.Model object\n PyMC3 model containing all random variables.\n trace : pymc3.sampling.BaseTrace object\n A PyMC3 trace object that contains samples for each parameter\n of the posterior.\n \"\"\"\n\n data_bmark = pd.concat([data, bmark], axis=1).dropna()\n\n with pm.Model() as model:\n sigma = pm.HalfCauchy(\n 'sigma',\n beta=1)\n nu = pm.Exponential('nu_minus_two', 1. / 10.)\n\n # alpha and beta\n X = data_bmark.iloc[:, 1]\n y = data_bmark.iloc[:, 0]\n\n alpha_reg = pm.Normal('alpha', mu=0, sd=.1)\n beta_reg = pm.Normal('beta', mu=0, sd=1)\n\n mu_reg = alpha_reg + beta_reg * X\n pm.StudentT('returns',\n nu=nu + 2,\n mu=mu_reg,\n sd=sigma,\n observed=y)\n trace = pm.sample(samples, progressbar=progressbar)\n\n return model, trace"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef model_returns_normal(data, samples=500, progressbar=True):\n\n with pm.Model() as model:\n mu = pm.Normal('mean returns', mu=0, sd=.01, testval=data.mean())\n sigma = pm.HalfCauchy('volatility', beta=1, testval=data.std())\n returns = pm.Normal('returns', mu=mu, sd=sigma, observed=data)\n pm.Deterministic(\n 'annual volatility',\n returns.distribution.variance**.5 *\n np.sqrt(252))\n pm.Deterministic(\n 'sharpe',\n returns.distribution.mean /\n returns.distribution.variance**.5 *\n np.sqrt(252))\n\n trace = pm.sample(samples, progressbar=progressbar)\n return model, trace", "response": "Run Bayesian model assuming returns are normally distributed."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef model_best(y1, y2, samples=1000, progressbar=True):\n\n y = np.concatenate((y1, y2))\n\n mu_m = np.mean(y)\n mu_p = 0.000001 * 1 / np.std(y)**2\n\n sigma_low = np.std(y) / 1000\n sigma_high = np.std(y) * 1000\n with pm.Model() as model:\n group1_mean = pm.Normal('group1_mean', mu=mu_m, tau=mu_p,\n testval=y1.mean())\n group2_mean = pm.Normal('group2_mean', mu=mu_m, tau=mu_p,\n testval=y2.mean())\n group1_std = pm.Uniform('group1_std', lower=sigma_low,\n upper=sigma_high, testval=y1.std())\n group2_std = pm.Uniform('group2_std', lower=sigma_low,\n upper=sigma_high, testval=y2.std())\n nu = pm.Exponential('nu_minus_two', 1 / 29., testval=4.) + 2.\n\n returns_group1 = pm.StudentT('group1', nu=nu, mu=group1_mean,\n lam=group1_std**-2, observed=y1)\n returns_group2 = pm.StudentT('group2', nu=nu, mu=group2_mean,\n lam=group2_std**-2, observed=y2)\n\n diff_of_means = pm.Deterministic('difference of means',\n group2_mean - group1_mean)\n pm.Deterministic('difference of stds',\n group2_std - group1_std)\n pm.Deterministic('effect size', diff_of_means /\n pm.math.sqrt((group1_std**2 +\n group2_std**2) / 2))\n\n pm.Deterministic('group1_annual_volatility',\n returns_group1.distribution.variance**.5 *\n np.sqrt(252))\n pm.Deterministic('group2_annual_volatility',\n returns_group2.distribution.variance**.5 *\n np.sqrt(252))\n\n pm.Deterministic('group1_sharpe', returns_group1.distribution.mean /\n returns_group1.distribution.variance**.5 *\n np.sqrt(252))\n pm.Deterministic('group2_sharpe', returns_group2.distribution.mean /\n returns_group2.distribution.variance**.5 *\n np.sqrt(252))\n\n trace = pm.sample(samples, progressbar=progressbar)\n return model, trace", "response": "This function runs a Bayesian hypothesis comparing if y1 and y2 come from the same distribution and returns the best fit of the T - Test."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nplot the BEST significance analysis of the current language.", "response": "def plot_best(trace=None, data_train=None, data_test=None,\n samples=1000, burn=200, axs=None):\n \"\"\"\n Plot BEST significance analysis.\n\n Parameters\n ----------\n trace : pymc3.sampling.BaseTrace, optional\n trace object as returned by model_best()\n If not passed, will run model_best(), for which\n data_train and data_test are required.\n data_train : pandas.Series, optional\n Returns of in-sample period.\n Required if trace=None.\n data_test : pandas.Series, optional\n Returns of out-of-sample period.\n Required if trace=None.\n samples : int, optional\n Posterior samples to draw.\n burn : int\n Posterior sampels to discard as burn-in.\n axs : array of matplotlib.axes objects, optional\n Plot into passed axes objects. Needs 6 axes.\n\n Returns\n -------\n None\n\n See Also\n --------\n model_best : Estimation of BEST model.\n \"\"\"\n\n if trace is None:\n if (data_train is not None) or (data_test is not None):\n raise ValueError('Either pass trace or data_train and data_test')\n trace = model_best(data_train, data_test, samples=samples)\n\n trace = trace[burn:]\n if axs is None:\n fig, axs = plt.subplots(ncols=2, nrows=3, figsize=(16, 4))\n\n def distplot_w_perc(trace, ax):\n sns.distplot(trace, ax=ax)\n ax.axvline(\n stats.scoreatpercentile(trace, 2.5),\n color='0.5', label='2.5 and 97.5 percentiles')\n ax.axvline(\n stats.scoreatpercentile(trace, 97.5),\n color='0.5')\n\n sns.distplot(trace['group1_mean'], ax=axs[0], label='Backtest')\n sns.distplot(trace['group2_mean'], ax=axs[0], label='Forward')\n axs[0].legend(loc=0, frameon=True, framealpha=0.5)\n axs[1].legend(loc=0, frameon=True, framealpha=0.5)\n\n distplot_w_perc(trace['difference of means'], axs[1])\n\n axs[0].set(xlabel='Mean', ylabel='Belief', yticklabels=[])\n axs[1].set(xlabel='Difference of means', yticklabels=[])\n\n sns.distplot(trace['group1_annual_volatility'], ax=axs[2],\n label='Backtest')\n sns.distplot(trace['group2_annual_volatility'], ax=axs[2],\n label='Forward')\n distplot_w_perc(trace['group2_annual_volatility'] -\n trace['group1_annual_volatility'], axs[3])\n axs[2].set(xlabel='Annual volatility', ylabel='Belief',\n yticklabels=[])\n axs[2].legend(loc=0, frameon=True, framealpha=0.5)\n axs[3].set(xlabel='Difference of volatility', yticklabels=[])\n\n sns.distplot(trace['group1_sharpe'], ax=axs[4], label='Backtest')\n sns.distplot(trace['group2_sharpe'], ax=axs[4], label='Forward')\n distplot_w_perc(trace['group2_sharpe'] - trace['group1_sharpe'],\n axs[5])\n axs[4].set(xlabel='Sharpe', ylabel='Belief', yticklabels=[])\n axs[4].legend(loc=0, frameon=True, framealpha=0.5)\n axs[5].set(xlabel='Difference of Sharpes', yticklabels=[])\n\n sns.distplot(trace['effect size'], ax=axs[6])\n axs[6].axvline(\n stats.scoreatpercentile(trace['effect size'], 2.5),\n color='0.5')\n axs[6].axvline(\n stats.scoreatpercentile(trace['effect size'], 97.5),\n color='0.5')\n axs[6].set(xlabel='Difference of means normalized by volatility',\n ylabel='Belief', yticklabels=[])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef model_stoch_vol(data, samples=2000, progressbar=True):\n\n from pymc3.distributions.timeseries import GaussianRandomWalk\n\n with pm.Model() as model:\n nu = pm.Exponential('nu', 1. / 10, testval=5.)\n sigma = pm.Exponential('sigma', 1. / .02, testval=.1)\n s = GaussianRandomWalk('s', sigma**-2, shape=len(data))\n volatility_process = pm.Deterministic('volatility_process',\n pm.math.exp(-2 * s))\n pm.StudentT('r', nu, lam=volatility_process, observed=data)\n\n trace = pm.sample(samples, progressbar=progressbar)\n\n return model, trace", "response": "Run stochastic volatility model."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot_stoch_vol(data, trace=None, ax=None):\n\n if trace is None:\n trace = model_stoch_vol(data)\n\n if ax is None:\n fig, ax = plt.subplots(figsize=(15, 8))\n\n data.abs().plot(ax=ax)\n ax.plot(data.index, np.exp(trace['s', ::30].T), 'r', alpha=.03)\n ax.set(title='Stochastic volatility', xlabel='Time', ylabel='Volatility')\n ax.legend(['Abs returns', 'Stochastic volatility process'],\n frameon=True, framealpha=0.5)\n\n return ax", "response": "Generate plot for stochastic volatility model."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef compute_bayes_cone(preds, starting_value=1.):\n\n def scoreatpercentile(cum_preds, p):\n return [stats.scoreatpercentile(\n c, p) for c in cum_preds.T]\n\n cum_preds = np.cumprod(preds + 1, 1) * starting_value\n perc = {p: scoreatpercentile(cum_preds, p) for p in (5, 25, 75, 95)}\n\n return perc", "response": "Compute 5 25 75 95 percentiles of cumulative returns used\n for the Bayesian cone."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef compute_consistency_score(returns_test, preds):\n\n returns_test_cum = cum_returns(returns_test, starting_value=1.)\n cum_preds = np.cumprod(preds + 1, 1)\n\n q = [sp.stats.percentileofscore(cum_preds[:, i],\n returns_test_cum.iloc[i],\n kind='weak')\n for i in range(len(returns_test_cum))]\n # normalize to be from 100 (perfect median line) to 0 (completely outside\n # of cone)\n return 100 - np.abs(50 - np.mean(q)) / .5", "response": "Compute Bayesian consistency score."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run_model(model, returns_train, returns_test=None,\n bmark=None, samples=500, ppc=False, progressbar=True):\n \"\"\"\n Run one of the Bayesian models.\n\n Parameters\n ----------\n model : {'alpha_beta', 't', 'normal', 'best'}\n Which model to run\n returns_train : pd.Series\n Timeseries of simple returns\n returns_test : pd.Series (optional)\n Out-of-sample returns. Datetimes in returns_test will be added to\n returns_train as missing values and predictions will be generated\n for them.\n bmark : pd.Series or pd.DataFrame (optional)\n Only used for alpha_beta to estimate regression coefficients.\n If bmark has more recent returns than returns_train, these dates\n will be treated as missing values and predictions will be\n generated for them taking market correlations into account.\n samples : int (optional)\n Number of posterior samples to draw.\n ppc : boolean (optional)\n Whether to run a posterior predictive check. Will generate\n samples of length returns_test. Returns a second argument\n that contains the PPC of shape samples x len(returns_test).\n\n Returns\n -------\n trace : pymc3.sampling.BaseTrace object\n A PyMC3 trace object that contains samples for each parameter\n of the posterior.\n\n ppc : numpy.array (if ppc==True)\n PPC of shape samples x len(returns_test).\n \"\"\"\n\n if model == 'alpha_beta':\n model, trace = model_returns_t_alpha_beta(returns_train,\n bmark, samples,\n progressbar=progressbar)\n elif model == 't':\n model, trace = model_returns_t(returns_train, samples,\n progressbar=progressbar)\n elif model == 'normal':\n model, trace = model_returns_normal(returns_train, samples,\n progressbar=progressbar)\n elif model == 'best':\n model, trace = model_best(returns_train, returns_test,\n samples=samples,\n progressbar=progressbar)\n else:\n raise NotImplementedError(\n 'Model {} not found.'\n 'Use alpha_beta, t, normal, or best.'.format(model))\n\n if ppc:\n ppc_samples = pm.sample_ppc(trace, samples=samples,\n model=model, size=len(returns_test),\n progressbar=progressbar)\n return trace, ppc_samples['returns']\n\n return trace", "response": "Run one of the Bayesian models."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating cumulative returns plot with Bayesian cone.", "response": "def plot_bayes_cone(returns_train, returns_test, ppc,\n plot_train_len=50, ax=None):\n \"\"\"\n Generate cumulative returns plot with Bayesian cone.\n\n Parameters\n ----------\n returns_train : pd.Series\n Timeseries of simple returns\n returns_test : pd.Series\n Out-of-sample returns. Datetimes in returns_test will be added to\n returns_train as missing values and predictions will be generated\n for them.\n ppc : np.array\n Posterior predictive samples of shape samples x\n len(returns_test).\n plot_train_len : int (optional)\n How many data points to plot of returns_train. Useful to zoom in on\n the prediction if there is a long backtest period.\n ax : matplotlib.Axis (optional)\n Axes upon which to plot.\n\n Returns\n -------\n score : float\n Consistency score (see compute_consistency_score)\n trace : pymc3.sampling.BaseTrace\n A PyMC3 trace object that contains samples for each parameter\n of the posterior.\n \"\"\"\n\n score = compute_consistency_score(returns_test,\n ppc)\n\n ax = _plot_bayes_cone(\n returns_train,\n returns_test,\n ppc,\n plot_train_len=plot_train_len,\n ax=ax)\n ax.text(\n 0.40,\n 0.90,\n 'Consistency score: %.1f' %\n score,\n verticalalignment='bottom',\n horizontalalignment='right',\n transform=ax.transAxes,\n )\n\n ax.set_ylabel('Cumulative returns')\n return score"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_voc_dataset(path='data', dataset='2012', contain_classes_in_person=False):\n path = os.path.join(path, 'VOC')\n\n def _recursive_parse_xml_to_dict(xml):\n \"\"\"Recursively parses XML contents to python dict.\n\n We assume that `object` tags are the only ones that can appear\n multiple times at the same level of a tree.\n\n Args:\n xml: xml tree obtained by parsing XML file contents using lxml.etree\n\n Returns:\n Python dictionary holding XML contents.\n\n \"\"\"\n if xml is not None:\n return {xml.tag: xml.text}\n result = {}\n for child in xml:\n child_result = _recursive_parse_xml_to_dict(child)\n if child.tag != 'object':\n result[child.tag] = child_result[child.tag]\n else:\n if child.tag not in result:\n result[child.tag] = []\n result[child.tag].append(child_result[child.tag])\n return {xml.tag: result}\n\n import xml.etree.ElementTree as ET\n\n if dataset == \"2012\":\n url = \"http://pjreddie.com/media/files/\"\n tar_filename = \"VOCtrainval_11-May-2012.tar\"\n extracted_filename = \"VOC2012\" #\"VOCdevkit/VOC2012\"\n logging.info(\" [============= VOC 2012 =============]\")\n elif dataset == \"2012test\":\n extracted_filename = \"VOC2012test\" #\"VOCdevkit/VOC2012\"\n logging.info(\" [============= VOC 2012 Test Set =============]\")\n logging.info(\n \" \\nAuthor: 2012test only have person annotation, so 2007test is highly recommended for testing !\\n\"\n )\n import time\n time.sleep(3)\n if os.path.isdir(os.path.join(path, extracted_filename)) is False:\n logging.info(\"For VOC 2012 Test data - online registration required\")\n logging.info(\n \" Please download VOC2012test.tar from: \\n register: http://host.robots.ox.ac.uk:8080 \\n voc2012 : http://host.robots.ox.ac.uk:8080/eval/challenges/voc2012/ \\ndownload: http://host.robots.ox.ac.uk:8080/eval/downloads/VOC2012test.tar\"\n )\n logging.info(\" unzip VOC2012test.tar,rename the folder to VOC2012test and put it into %s\" % path)\n exit()\n # # http://host.robots.ox.ac.uk:8080/eval/downloads/VOC2012test.tar\n # url = \"http://host.robots.ox.ac.uk:8080/eval/downloads/\"\n # tar_filename = \"VOC2012test.tar\"\n elif dataset == \"2007\":\n url = \"http://pjreddie.com/media/files/\"\n tar_filename = \"VOCtrainval_06-Nov-2007.tar\"\n extracted_filename = \"VOC2007\"\n logging.info(\" [============= VOC 2007 =============]\")\n elif dataset == \"2007test\":\n # http://host.robots.ox.ac.uk/pascal/VOC/voc2007/index.html#testdata\n # http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar\n url = \"http://pjreddie.com/media/files/\"\n tar_filename = \"VOCtest_06-Nov-2007.tar\"\n extracted_filename = \"VOC2007test\"\n logging.info(\" [============= VOC 2007 Test Set =============]\")\n else:\n raise Exception(\"Please set the dataset aug to 2012, 2012test or 2007.\")\n\n # download dataset\n if dataset != \"2012test\":\n from sys import platform as _platform\n if folder_exists(os.path.join(path, extracted_filename)) is False:\n logging.info(\"[VOC] {} is nonexistent in {}\".format(extracted_filename, path))\n maybe_download_and_extract(tar_filename, path, url, extract=True)\n del_file(os.path.join(path, tar_filename))\n if dataset == \"2012\":\n if _platform == \"win32\":\n os.system(\"move {}\\VOCdevkit\\VOC2012 {}\\VOC2012\".format(path, path))\n else:\n os.system(\"mv {}/VOCdevkit/VOC2012 {}/VOC2012\".format(path, path))\n elif dataset == \"2007\":\n if _platform == \"win32\":\n os.system(\"move {}\\VOCdevkit\\VOC2007 {}\\VOC2007\".format(path, path))\n else:\n os.system(\"mv {}/VOCdevkit/VOC2007 {}/VOC2007\".format(path, path))\n elif dataset == \"2007test\":\n if _platform == \"win32\":\n os.system(\"move {}\\VOCdevkit\\VOC2007 {}\\VOC2007test\".format(path, path))\n else:\n os.system(\"mv {}/VOCdevkit/VOC2007 {}/VOC2007test\".format(path, path))\n del_folder(os.path.join(path, 'VOCdevkit'))\n # object classes(labels) NOTE: YOU CAN CUSTOMIZE THIS LIST\n classes = [\n \"aeroplane\", \"bicycle\", \"bird\", \"boat\", \"bottle\", \"bus\", \"car\", \"cat\", \"chair\", \"cow\", \"diningtable\", \"dog\",\n \"horse\", \"motorbike\", \"person\", \"pottedplant\", \"sheep\", \"sofa\", \"train\", \"tvmonitor\"\n ]\n if contain_classes_in_person:\n classes_in_person = [\"head\", \"hand\", \"foot\"]\n else:\n classes_in_person = []\n\n classes += classes_in_person # use extra 3 classes for person\n\n classes_dict = utils.list_string_to_dict(classes)\n logging.info(\"[VOC] object classes {}\".format(classes_dict))\n\n # 1. image path list\n # folder_imgs = path+\"/\"+extracted_filename+\"/JPEGImages/\"\n folder_imgs = os.path.join(path, extracted_filename, \"JPEGImages\")\n imgs_file_list = load_file_list(path=folder_imgs, regx='\\\\.jpg', printable=False)\n logging.info(\"[VOC] {} images found\".format(len(imgs_file_list)))\n\n imgs_file_list.sort(\n key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])\n ) # 2007_000027.jpg --> 2007000027\n\n imgs_file_list = [os.path.join(folder_imgs, s) for s in imgs_file_list]\n # logging.info('IM',imgs_file_list[0::3333], imgs_file_list[-1])\n if dataset != \"2012test\":\n ##======== 2. semantic segmentation maps path list\n # folder_semseg = path+\"/\"+extracted_filename+\"/SegmentationClass/\"\n folder_semseg = os.path.join(path, extracted_filename, \"SegmentationClass\")\n imgs_semseg_file_list = load_file_list(path=folder_semseg, regx='\\\\.png', printable=False)\n logging.info(\"[VOC] {} maps for semantic segmentation found\".format(len(imgs_semseg_file_list)))\n imgs_semseg_file_list.sort(\n key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])\n ) # 2007_000032.png --> 2007000032\n imgs_semseg_file_list = [os.path.join(folder_semseg, s) for s in imgs_semseg_file_list]\n # logging.info('Semantic Seg IM',imgs_semseg_file_list[0::333], imgs_semseg_file_list[-1])\n ##======== 3. instance segmentation maps path list\n # folder_insseg = path+\"/\"+extracted_filename+\"/SegmentationObject/\"\n folder_insseg = os.path.join(path, extracted_filename, \"SegmentationObject\")\n imgs_insseg_file_list = load_file_list(path=folder_insseg, regx='\\\\.png', printable=False)\n logging.info(\"[VOC] {} maps for instance segmentation found\".format(len(imgs_semseg_file_list)))\n imgs_insseg_file_list.sort(\n key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])\n ) # 2007_000032.png --> 2007000032\n imgs_insseg_file_list = [os.path.join(folder_insseg, s) for s in imgs_insseg_file_list]\n # logging.info('Instance Seg IM',imgs_insseg_file_list[0::333], imgs_insseg_file_list[-1])\n else:\n imgs_semseg_file_list = []\n imgs_insseg_file_list = []\n # 4. annotations for bounding box and object class\n # folder_ann = path+\"/\"+extracted_filename+\"/Annotations/\"\n folder_ann = os.path.join(path, extracted_filename, \"Annotations\")\n imgs_ann_file_list = load_file_list(path=folder_ann, regx='\\\\.xml', printable=False)\n logging.info(\n \"[VOC] {} XML annotation files for bounding box and object class found\".format(len(imgs_ann_file_list))\n )\n imgs_ann_file_list.sort(\n key=lambda s: int(s.replace('.', ' ').replace('_', '').split(' ')[-2])\n ) # 2007_000027.xml --> 2007000027\n imgs_ann_file_list = [os.path.join(folder_ann, s) for s in imgs_ann_file_list]\n # logging.info('ANN',imgs_ann_file_list[0::3333], imgs_ann_file_list[-1])\n\n if dataset == \"2012test\": # remove unused images in JPEG folder\n imgs_file_list_new = []\n for ann in imgs_ann_file_list:\n ann = os.path.split(ann)[-1].split('.')[0]\n for im in imgs_file_list:\n if ann in im:\n imgs_file_list_new.append(im)\n break\n imgs_file_list = imgs_file_list_new\n logging.info(\"[VOC] keep %d images\" % len(imgs_file_list_new))\n\n # parse XML annotations\n def convert(size, box):\n dw = 1. / size[0]\n dh = 1. / size[1]\n x = (box[0] + box[1]) / 2.0\n y = (box[2] + box[3]) / 2.0\n w = box[1] - box[0]\n h = box[3] - box[2]\n x = x * dw\n w = w * dw\n y = y * dh\n h = h * dh\n return x, y, w, h\n\n def convert_annotation(file_name):\n \"\"\"Given VOC2012 XML Annotations, returns number of objects and info.\"\"\"\n in_file = open(file_name)\n out_file = \"\"\n tree = ET.parse(in_file)\n root = tree.getroot()\n size = root.find('size')\n w = int(size.find('width').text)\n h = int(size.find('height').text)\n n_objs = 0\n\n for obj in root.iter('object'):\n if dataset != \"2012test\":\n difficult = obj.find('difficult').text\n cls = obj.find('name').text\n if cls not in classes or int(difficult) == 1:\n continue\n else:\n cls = obj.find('name').text\n if cls not in classes:\n continue\n cls_id = classes.index(cls)\n xmlbox = obj.find('bndbox')\n b = (\n float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),\n float(xmlbox.find('ymax').text)\n )\n bb = convert((w, h), b)\n\n out_file += str(cls_id) + \" \" + \" \".join([str(a) for a in bb]) + '\\n'\n n_objs += 1\n if cls in \"person\":\n for part in obj.iter('part'):\n cls = part.find('name').text\n if cls not in classes_in_person:\n continue\n cls_id = classes.index(cls)\n xmlbox = part.find('bndbox')\n b = (\n float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text),\n float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text)\n )\n bb = convert((w, h), b)\n # out_file.write(str(cls_id) + \" \" + \" \".join([str(a) for a in bb]) + '\\n')\n out_file += str(cls_id) + \" \" + \" \".join([str(a) for a in bb]) + '\\n'\n n_objs += 1\n in_file.close()\n return n_objs, out_file\n\n logging.info(\"[VOC] Parsing xml annotations files\")\n n_objs_list = []\n objs_info_list = [] # Darknet Format list of string\n objs_info_dicts = {}\n for idx, ann_file in enumerate(imgs_ann_file_list):\n n_objs, objs_info = convert_annotation(ann_file)\n n_objs_list.append(n_objs)\n objs_info_list.append(objs_info)\n with tf.gfile.GFile(ann_file, 'r') as fid:\n xml_str = fid.read()\n xml = etree.fromstring(xml_str)\n data = _recursive_parse_xml_to_dict(xml)['annotation']\n objs_info_dicts.update({imgs_file_list[idx]: data})\n\n return imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list, classes, classes_in_person, classes_dict, n_objs_list, objs_info_list, objs_info_dicts", "response": "Load a Pascal VOC 2007 dataset."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef private_method(func):\n\n def func_wrapper(*args, **kwargs):\n \"\"\"Decorator wrapper function.\"\"\"\n outer_frame = inspect.stack()[1][0]\n if 'self' not in outer_frame.f_locals or outer_frame.f_locals['self'] is not args[0]:\n raise RuntimeError('%s.%s is a private method' % (args[0].__class__.__name__, func.__name__))\n\n return func(*args, **kwargs)\n\n return func_wrapper", "response": "Decorator for making an instance method private."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef protected_method(func):\n\n def func_wrapper(*args, **kwargs):\n \"\"\"Decorator wrapper function.\"\"\"\n outer_frame = inspect.stack()[1][0]\n\n caller = inspect.getmro(outer_frame.f_locals['self'].__class__)[:-1]\n target = inspect.getmro(args[0].__class__)[:-1]\n\n share_subsclass = False\n\n for cls_ in target:\n if issubclass(caller[0], cls_) or caller[0] is cls_:\n share_subsclass = True\n break\n\n if ('self' not in outer_frame.f_locals or\n outer_frame.f_locals['self'] is not args[0]) and (not share_subsclass):\n raise RuntimeError('%s.%s is a protected method' % (args[0].__class__.__name__, func.__name__))\n\n return func(*args, **kwargs)\n\n return func_wrapper", "response": "Decorator for making an instance method private."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsimplify version of Conv1dLayer.", "response": "def atrous_conv1d(\n prev_layer,\n n_filter=32,\n filter_size=2,\n stride=1,\n dilation=1,\n act=None,\n padding='SAME',\n data_format='NWC',\n W_init=tf.truncated_normal_initializer(stddev=0.02),\n b_init=tf.constant_initializer(value=0.0),\n W_init_args=None,\n b_init_args=None,\n name='atrous_1d',\n):\n \"\"\"Simplified version of :class:`AtrousConv1dLayer`.\n\n Parameters\n ----------\n prev_layer : :class:`Layer`\n Previous layer.\n n_filter : int\n The number of filters.\n filter_size : int\n The filter size.\n stride : tuple of int\n The strides: (height, width).\n dilation : int\n The filter dilation size.\n act : activation function\n The activation function of this layer.\n padding : str\n The padding algorithm type: \"SAME\" or \"VALID\".\n data_format : str\n Default is 'NWC' as it is a 1D CNN.\n W_init : initializer\n The initializer for the weight matrix.\n b_init : initializer or None\n The initializer for the bias vector. If None, skip biases.\n W_init_args : dictionary\n The arguments for the weight matrix initializer.\n b_init_args : dictionary\n The arguments for the bias vector initializer.\n name : str\n A unique layer name.\n\n Returns\n -------\n :class:`Layer`\n A :class:`AtrousConv1dLayer` object\n\n \"\"\"\n return Conv1dLayer(\n prev_layer=prev_layer,\n act=act,\n shape=(filter_size, int(prev_layer.outputs.get_shape()[-1]), n_filter),\n stride=stride,\n padding=padding,\n dilation_rate=dilation,\n data_format=data_format,\n W_init=W_init,\n b_init=b_init,\n W_init_args=W_init_args,\n b_init_args=b_init_args,\n name=name,\n )"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwrap for _log_counter_per_token. Args: token: The token for which to look up the count. Returns: The number of times this function has been called with *token* as an argument (starting at 0)", "response": "def _GetNextLogCountPerToken(token):\n \"\"\"Wrapper for _log_counter_per_token.\n\n Args:\n token: The token for which to look up the count.\n\n Returns:\n The number of times this function has been called with\n *token* as an argument (starting at 0)\n \"\"\"\n global _log_counter_per_token # pylint: disable=global-variable-not-assigned\n _log_counter_per_token[token] = 1 + _log_counter_per_token.get(token, -1)\n return _log_counter_per_token[token]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef log_every_n(level, msg, n, *args):\n count = _GetNextLogCountPerToken(_GetFileAndLine())\n log_if(level, msg, not (count % n), *args)", "response": "Logs msg % args at level once per n times."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef log_if(level, msg, condition, *args):\n if condition:\n vlog(level, msg, *args)", "response": "Log msg % args at level only if condition is fulfilled."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _GetFileAndLine():\n # Use sys._getframe(). This avoids creating a traceback object.\n # pylint: disable=protected-access\n f = _sys._getframe()\n # pylint: enable=protected-access\n our_file = f.f_code.co_filename\n f = f.f_back\n while f:\n code = f.f_code\n if code.co_filename != our_file:\n return (code.co_filename, f.f_lineno)\n f = f.f_back\n return ('', 0)", "response": "Returns the filename and line number for the stack frame."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef google2_log_prefix(level, timestamp=None, file_and_line=None):\n # pylint: disable=global-variable-not-assigned\n global _level_names\n # pylint: enable=global-variable-not-assigned\n\n # Record current time\n now = timestamp or _time.time()\n now_tuple = _time.localtime(now)\n now_microsecond = int(1e6 * (now % 1.0))\n\n (filename, line) = file_and_line or _GetFileAndLine()\n basename = _os.path.basename(filename)\n\n # Severity string\n severity = 'I'\n if level in _level_names:\n severity = _level_names[level][0]\n\n s = '%c%02d%02d %02d: %02d: %02d.%06d %5d %s: %d] ' % (\n severity,\n now_tuple[1], # month\n now_tuple[2], # day\n now_tuple[3], # hour\n now_tuple[4], # min\n now_tuple[5], # sec\n now_microsecond,\n _get_thread_id(),\n basename,\n line\n )\n\n return s", "response": "Assemble a logline prefix using the google2 format."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload MPII Human Pose Dataset.", "response": "def load_mpii_pose_dataset(path='data', is_16_pos_only=False):\n \"\"\"Load MPII Human Pose Dataset.\n\n Parameters\n -----------\n path : str\n The path that the data is downloaded to.\n is_16_pos_only : boolean\n If True, only return the peoples contain 16 pose keypoints. (Usually be used for single person pose estimation)\n\n Returns\n ----------\n img_train_list : list of str\n The image directories of training data.\n ann_train_list : list of dict\n The annotations of training data.\n img_test_list : list of str\n The image directories of testing data.\n ann_test_list : list of dict\n The annotations of testing data.\n\n Examples\n --------\n >>> import pprint\n >>> import tensorlayer as tl\n >>> img_train_list, ann_train_list, img_test_list, ann_test_list = tl.files.load_mpii_pose_dataset()\n >>> image = tl.vis.read_image(img_train_list[0])\n >>> tl.vis.draw_mpii_pose_to_image(image, ann_train_list[0], 'image.png')\n >>> pprint.pprint(ann_train_list[0])\n\n References\n -----------\n - `MPII Human Pose Dataset. CVPR 14 `__\n - `MPII Human Pose Models. CVPR 16 `__\n - `MPII Human Shape, Poselet Conditioned Pictorial Structures and etc `__\n - `MPII Keyponts and ID `__\n \"\"\"\n path = os.path.join(path, 'mpii_human_pose')\n logging.info(\"Load or Download MPII Human Pose > {}\".format(path))\n\n # annotation\n url = \"http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/\"\n tar_filename = \"mpii_human_pose_v1_u12_2.zip\"\n extracted_filename = \"mpii_human_pose_v1_u12_2\"\n if folder_exists(os.path.join(path, extracted_filename)) is False:\n logging.info(\"[MPII] (annotation) {} is nonexistent in {}\".format(extracted_filename, path))\n maybe_download_and_extract(tar_filename, path, url, extract=True)\n del_file(os.path.join(path, tar_filename))\n\n # images\n url = \"http://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/\"\n tar_filename = \"mpii_human_pose_v1.tar.gz\"\n extracted_filename2 = \"images\"\n if folder_exists(os.path.join(path, extracted_filename2)) is False:\n logging.info(\"[MPII] (images) {} is nonexistent in {}\".format(extracted_filename, path))\n maybe_download_and_extract(tar_filename, path, url, extract=True)\n del_file(os.path.join(path, tar_filename))\n\n # parse annotation, format see http://human-pose.mpi-inf.mpg.de/#download\n import scipy.io as sio\n logging.info(\"reading annotations from mat file ...\")\n # mat = sio.loadmat(os.path.join(path, extracted_filename, \"mpii_human_pose_v1_u12_1.mat\"))\n\n # def fix_wrong_joints(joint): # https://github.com/mitmul/deeppose/blob/master/datasets/mpii_dataset.py\n # if '12' in joint and '13' in joint and '2' in joint and '3' in joint:\n # if ((joint['12'][0] < joint['13'][0]) and\n # (joint['3'][0] < joint['2'][0])):\n # joint['2'], joint['3'] = joint['3'], joint['2']\n # if ((joint['12'][0] > joint['13'][0]) and\n # (joint['3'][0] > joint['2'][0])):\n # joint['2'], joint['3'] = joint['3'], joint['2']\n # return joint\n\n ann_train_list = []\n ann_test_list = []\n img_train_list = []\n img_test_list = []\n\n def save_joints():\n # joint_data_fn = os.path.join(path, 'data.json')\n # fp = open(joint_data_fn, 'w')\n mat = sio.loadmat(os.path.join(path, extracted_filename, \"mpii_human_pose_v1_u12_1.mat\"))\n\n for _, (anno, train_flag) in enumerate( # all images\n zip(mat['RELEASE']['annolist'][0, 0][0], mat['RELEASE']['img_train'][0, 0][0])):\n\n img_fn = anno['image']['name'][0, 0][0]\n train_flag = int(train_flag)\n\n # print(i, img_fn, train_flag) # DEBUG print all images\n\n if train_flag:\n img_train_list.append(img_fn)\n ann_train_list.append([])\n else:\n img_test_list.append(img_fn)\n ann_test_list.append([])\n\n head_rect = []\n if 'x1' in str(anno['annorect'].dtype):\n head_rect = zip(\n [x1[0, 0] for x1 in anno['annorect']['x1'][0]], [y1[0, 0] for y1 in anno['annorect']['y1'][0]],\n [x2[0, 0] for x2 in anno['annorect']['x2'][0]], [y2[0, 0] for y2 in anno['annorect']['y2'][0]]\n )\n else:\n head_rect = [] # TODO\n\n if 'annopoints' in str(anno['annorect'].dtype):\n annopoints = anno['annorect']['annopoints'][0]\n head_x1s = anno['annorect']['x1'][0]\n head_y1s = anno['annorect']['y1'][0]\n head_x2s = anno['annorect']['x2'][0]\n head_y2s = anno['annorect']['y2'][0]\n\n for annopoint, head_x1, head_y1, head_x2, head_y2 in zip(annopoints, head_x1s, head_y1s, head_x2s,\n head_y2s):\n # if annopoint != []:\n # if len(annopoint) != 0:\n if annopoint.size:\n head_rect = [\n float(head_x1[0, 0]),\n float(head_y1[0, 0]),\n float(head_x2[0, 0]),\n float(head_y2[0, 0])\n ]\n\n # joint coordinates\n annopoint = annopoint['point'][0, 0]\n j_id = [str(j_i[0, 0]) for j_i in annopoint['id'][0]]\n x = [x[0, 0] for x in annopoint['x'][0]]\n y = [y[0, 0] for y in annopoint['y'][0]]\n joint_pos = {}\n for _j_id, (_x, _y) in zip(j_id, zip(x, y)):\n joint_pos[int(_j_id)] = [float(_x), float(_y)]\n # joint_pos = fix_wrong_joints(joint_pos)\n\n # visibility list\n if 'is_visible' in str(annopoint.dtype):\n vis = [v[0] if v.size > 0 else [0] for v in annopoint['is_visible'][0]]\n vis = dict([(k, int(v[0])) if len(v) > 0 else v for k, v in zip(j_id, vis)])\n else:\n vis = None\n\n # if len(joint_pos) == 16:\n if ((is_16_pos_only ==True) and (len(joint_pos) == 16)) or (is_16_pos_only == False):\n # only use image with 16 key points / or use all\n data = {\n 'filename': img_fn,\n 'train': train_flag,\n 'head_rect': head_rect,\n 'is_visible': vis,\n 'joint_pos': joint_pos\n }\n # print(json.dumps(data), file=fp) # py3\n if train_flag:\n ann_train_list[-1].append(data)\n else:\n ann_test_list[-1].append(data)\n\n # def write_line(datum, fp):\n # joints = sorted([[int(k), v] for k, v in datum['joint_pos'].items()])\n # joints = np.array([j for i, j in joints]).flatten()\n #\n # out = [datum['filename']]\n # out.extend(joints)\n # out = [str(o) for o in out]\n # out = ','.join(out)\n #\n # print(out, file=fp)\n\n # def split_train_test():\n # # fp_test = open('data/mpii/test_joints.csv', 'w')\n # fp_test = open(os.path.join(path, 'test_joints.csv'), 'w')\n # # fp_train = open('data/mpii/train_joints.csv', 'w')\n # fp_train = open(os.path.join(path, 'train_joints.csv'), 'w')\n # # all_data = open('data/mpii/data.json').readlines()\n # all_data = open(os.path.join(path, 'data.json')).readlines()\n # N = len(all_data)\n # N_test = int(N * 0.1)\n # N_train = N - N_test\n #\n # print('N:{}'.format(N))\n # print('N_train:{}'.format(N_train))\n # print('N_test:{}'.format(N_test))\n #\n # np.random.seed(1701)\n # perm = np.random.permutation(N)\n # test_indices = perm[:N_test]\n # train_indices = perm[N_test:]\n #\n # print('train_indices:{}'.format(len(train_indices)))\n # print('test_indices:{}'.format(len(test_indices)))\n #\n # for i in train_indices:\n # datum = json.loads(all_data[i].strip())\n # write_line(datum, fp_train)\n #\n # for i in test_indices:\n # datum = json.loads(all_data[i].strip())\n # write_line(datum, fp_test)\n\n save_joints()\n # split_train_test() #\n\n ## read images dir\n logging.info(\"reading images list ...\")\n img_dir = os.path.join(path, extracted_filename2)\n _img_list = load_file_list(path=os.path.join(path, extracted_filename2), regx='\\\\.jpg', printable=False)\n # ann_list = json.load(open(os.path.join(path, 'data.json')))\n for i, im in enumerate(img_train_list):\n if im not in _img_list:\n print('missing training image {} in {} (remove from img(ann)_train_list)'.format(im, img_dir))\n # img_train_list.remove(im)\n del img_train_list[i]\n del ann_train_list[i]\n for i, im in enumerate(img_test_list):\n if im not in _img_list:\n print('missing testing image {} in {} (remove from img(ann)_test_list)'.format(im, img_dir))\n # img_test_list.remove(im)\n del img_train_list[i]\n del ann_train_list[i]\n\n ## check annotation and images\n n_train_images = len(img_train_list)\n n_test_images = len(img_test_list)\n n_images = n_train_images + n_test_images\n logging.info(\"n_images: {} n_train_images: {} n_test_images: {}\".format(n_images, n_train_images, n_test_images))\n n_train_ann = len(ann_train_list)\n n_test_ann = len(ann_test_list)\n n_ann = n_train_ann + n_test_ann\n logging.info(\"n_ann: {} n_train_ann: {} n_test_ann: {}\".format(n_ann, n_train_ann, n_test_ann))\n n_train_people = len(sum(ann_train_list, []))\n n_test_people = len(sum(ann_test_list, []))\n n_people = n_train_people + n_test_people\n logging.info(\"n_people: {} n_train_people: {} n_test_people: {}\".format(n_people, n_train_people, n_test_people))\n # add path to all image file name\n for i, value in enumerate(img_train_list):\n img_train_list[i] = os.path.join(img_dir, value)\n for i, value in enumerate(img_test_list):\n img_test_list[i] = os.path.join(img_dir, value)\n return img_train_list, ann_train_list, img_test_list, ann_test_list"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef batch_transformer(U, thetas, out_size, name='BatchSpatialTransformer2dAffine'):\n with tf.variable_scope(name):\n num_batch, num_transforms = map(int, thetas.get_shape().as_list()[:2])\n indices = [[i] * num_transforms for i in xrange(num_batch)]\n input_repeated = tf.gather(U, tf.reshape(indices, [-1]))\n return transformer(input_repeated, thetas, out_size)", "response": "Batch Spatial Transformer function for 2D Affine Transformation ."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_task_spec_def():\n if 'TF_CONFIG' in os.environ:\n # TF_CONFIG is used in ML-engine\n env = json.loads(os.environ.get('TF_CONFIG', '{}'))\n task_data = env.get('task', None) or {'type': 'master', 'index': 0}\n cluster_data = env.get('cluster', None) or {'ps': None, 'worker': None, 'master': None}\n return TaskSpecDef(\n task_type=task_data['type'], index=task_data['index'],\n trial=task_data['trial'] if 'trial' in task_data else None, ps_hosts=cluster_data['ps'],\n worker_hosts=cluster_data['worker'], master=cluster_data['master'] if 'master' in cluster_data else None\n )\n elif 'JOB_NAME' in os.environ:\n # JOB_NAME, TASK_INDEX, PS_HOSTS, WORKER_HOSTS and MASTER_HOST are used in TensorPort\n return TaskSpecDef(\n task_type=os.environ['JOB_NAME'], index=os.environ['TASK_INDEX'], ps_hosts=os.environ.get('PS_HOSTS', None),\n worker_hosts=os.environ.get('WORKER_HOSTS', None), master=os.environ.get('MASTER_HOST', None)\n )\n else:\n raise Exception('You need to setup TF_CONFIG or JOB_NAME to define the task.')", "response": "Returns a TaskSpecDef based on the environment variables for distributed training."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a distributed session.", "response": "def create_distributed_session(\n task_spec=None, checkpoint_dir=None, scaffold=None, hooks=None, chief_only_hooks=None, save_checkpoint_secs=600,\n save_summaries_steps=object(), save_summaries_secs=object(), config=None, stop_grace_period_secs=120,\n log_step_count_steps=100\n):\n \"\"\"Creates a distributed session.\n\n It calls `MonitoredTrainingSession` to create a :class:`MonitoredSession` for distributed training.\n\n Parameters\n ----------\n task_spec : :class:`TaskSpecDef`.\n The task spec definition from create_task_spec_def()\n checkpoint_dir : str.\n Optional path to a directory where to restore variables.\n scaffold : ``Scaffold``\n A `Scaffold` used for gathering or building supportive ops.\n If not specified, a default one is created. It's used to finalize the graph.\n hooks : list of ``SessionRunHook`` objects.\n Optional\n chief_only_hooks : list of ``SessionRunHook`` objects.\n Activate these hooks if `is_chief==True`, ignore otherwise.\n save_checkpoint_secs : int\n The frequency, in seconds, that a checkpoint is saved\n using a default checkpoint saver. If `save_checkpoint_secs` is set to\n `None`, then the default checkpoint saver isn't used.\n save_summaries_steps : int\n The frequency, in number of global steps, that the\n summaries are written to disk using a default summary saver. If both\n `save_summaries_steps` and `save_summaries_secs` are set to `None`, then\n the default summary saver isn't used. Default 100.\n save_summaries_secs : int\n The frequency, in secs, that the summaries are written\n to disk using a default summary saver. If both `save_summaries_steps` and\n `save_summaries_secs` are set to `None`, then the default summary saver\n isn't used. Default not enabled.\n config : ``tf.ConfigProto``\n an instance of `tf.ConfigProto` proto used to configure the session.\n It's the `config` argument of constructor of `tf.Session`.\n stop_grace_period_secs : int\n Number of seconds given to threads to stop after\n `close()` has been called.\n log_step_count_steps : int\n The frequency, in number of global steps, that the\n global step/sec is logged.\n\n Examples\n --------\n A simple example for distributed training where all the workers use the same dataset:\n\n >>> task_spec = TaskSpec()\n >>> with tf.device(task_spec.device_fn()):\n >>> tensors = create_graph()\n >>> with tl.DistributedSession(task_spec=task_spec,\n ... checkpoint_dir='/tmp/ckpt') as session:\n >>> while not session.should_stop():\n >>> session.run(tensors)\n\n An example where the dataset is shared among the workers\n (see https://www.tensorflow.org/programmers_guide/datasets):\n\n >>> task_spec = TaskSpec()\n >>> # dataset is a :class:`tf.data.Dataset` with the raw data\n >>> dataset = create_dataset()\n >>> if task_spec is not None:\n >>> dataset = dataset.shard(task_spec.num_workers, task_spec.shard_index)\n >>> # shuffle or apply a map function to the new sharded dataset, for example:\n >>> dataset = dataset.shuffle(buffer_size=10000)\n >>> dataset = dataset.batch(batch_size)\n >>> dataset = dataset.repeat(num_epochs)\n >>> # create the iterator for the dataset and the input tensor\n >>> iterator = dataset.make_one_shot_iterator()\n >>> next_element = iterator.get_next()\n >>> with tf.device(task_spec.device_fn()):\n >>> # next_element is the input for the graph\n >>> tensors = create_graph(next_element)\n >>> with tl.DistributedSession(task_spec=task_spec,\n ... checkpoint_dir='/tmp/ckpt') as session:\n >>> while not session.should_stop():\n >>> session.run(tensors)\n\n References\n ----------\n - `MonitoredTrainingSession `__\n\n \"\"\"\n target = task_spec.target() if task_spec is not None else None\n is_chief = task_spec.is_master() if task_spec is not None else True\n return tf.train.MonitoredTrainingSession(\n master=target, is_chief=is_chief, checkpoint_dir=checkpoint_dir, scaffold=scaffold,\n save_checkpoint_secs=save_checkpoint_secs, save_summaries_steps=save_summaries_steps,\n save_summaries_secs=save_summaries_secs, log_step_count_steps=log_step_count_steps,\n stop_grace_period_secs=stop_grace_period_secs, config=config, hooks=hooks, chief_only_hooks=chief_only_hooks\n )"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef validation_metrics(self):\n\n if (self._validation_iterator is None) or (self._validation_metrics is None):\n raise AttributeError('Validation is not setup.')\n\n n = 0.0\n metric_sums = [0.0] * len(self._validation_metrics)\n self._sess.run(self._validation_iterator.initializer)\n while True:\n try:\n metrics = self._sess.run(self._validation_metrics)\n for i, m in enumerate(metrics):\n metric_sums[i] += m\n n += 1.0\n except tf.errors.OutOfRangeError:\n break\n for i, m in enumerate(metric_sums):\n metric_sums[i] = metric_sums[i] / n\n return zip(self._validation_metrics, metric_sums)", "response": "A helper function to compute validation related metrics"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef train_and_validate_to_end(self, validate_step_size=50):\n while not self._sess.should_stop():\n self.train_on_batch() # Run a training step synchronously.\n if self.global_step % validate_step_size == 0:\n # logging.info(\"Average loss for validation dataset: %s\" % self.get_validation_metrics())\n log_str = 'step: %d, ' % self.global_step\n for n, m in self.validation_metrics:\n log_str += '%s: %f, ' % (n.name, m)\n logging.info(log_str)", "response": "This function is used to train and validate a model at the same time."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/'):\n path = os.path.join(path, name)\n\n # Define functions for loading mnist-like data's images and labels.\n # For convenience, they also download the requested files if needed.\n def load_mnist_images(path, filename):\n filepath = maybe_download_and_extract(filename, path, url)\n\n logging.info(filepath)\n # Read the inputs in Yann LeCun's binary format.\n with gzip.open(filepath, 'rb') as f:\n data = np.frombuffer(f.read(), np.uint8, offset=16)\n # The inputs are vectors now, we reshape them to monochrome 2D images,\n # following the shape convention: (examples, channels, rows, columns)\n data = data.reshape(shape)\n # The inputs come as bytes, we convert them to float32 in range [0,1].\n # (Actually to range [0, 255/256], for compatibility to the version\n # provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)\n return data / np.float32(256)\n\n def load_mnist_labels(path, filename):\n filepath = maybe_download_and_extract(filename, path, url)\n # Read the labels in Yann LeCun's binary format.\n with gzip.open(filepath, 'rb') as f:\n data = np.frombuffer(f.read(), np.uint8, offset=8)\n # The labels are vectors of integers now, that's exactly what we want.\n return data\n\n # Download and read the training and test set images and labels.\n logging.info(\"Load or Download {0} > {1}\".format(name.upper(), path))\n X_train = load_mnist_images(path, 'train-images-idx3-ubyte.gz')\n y_train = load_mnist_labels(path, 'train-labels-idx1-ubyte.gz')\n X_test = load_mnist_images(path, 't10k-images-idx3-ubyte.gz')\n y_test = load_mnist_labels(path, 't10k-labels-idx1-ubyte.gz')\n\n # We reserve the last 10000 training examples for validation.\n X_train, X_val = X_train[:-10000], X_train[-10000:]\n y_train, y_val = y_train[:-10000], y_train[-10000:]\n\n # We just return all the arrays in order, as expected in main().\n # (It doesn't matter how we do this as long as we can read them again.)\n X_train = np.asarray(X_train, dtype=np.float32)\n y_train = np.asarray(y_train, dtype=np.int32)\n X_val = np.asarray(X_val, dtype=np.float32)\n y_val = np.asarray(y_val, dtype=np.int32)\n X_test = np.asarray(X_test, dtype=np.float32)\n y_test = np.asarray(y_test, dtype=np.int32)\n return X_train, y_train, X_val, y_val, X_test, y_test", "response": "Load mnist - like dataset."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nload CIFAR - 10 dataset.", "response": "def load_cifar10_dataset(shape=(-1, 32, 32, 3), path='data', plotable=False):\n \"\"\"Load CIFAR-10 dataset.\n\n It consists of 60000 32x32 colour images in 10 classes, with\n 6000 images per class. There are 50000 training images and 10000 test images.\n\n The dataset is divided into five training batches and one test batch, each with\n 10000 images. The test batch contains exactly 1000 randomly-selected images from\n each class. The training batches contain the remaining images in random order,\n but some training batches may contain more images from one class than another.\n Between them, the training batches contain exactly 5000 images from each class.\n\n Parameters\n ----------\n shape : tupe\n The shape of digit images e.g. (-1, 3, 32, 32) and (-1, 32, 32, 3).\n path : str\n The path that the data is downloaded to, defaults is ``data/cifar10/``.\n plotable : boolean\n Whether to plot some image examples, False as default.\n\n Examples\n --------\n >>> X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))\n\n References\n ----------\n - `CIFAR website `__\n - `Data download link `__\n - ``__\n\n \"\"\"\n path = os.path.join(path, 'cifar10')\n logging.info(\"Load or Download cifar10 > {}\".format(path))\n\n # Helper function to unpickle the data\n def unpickle(file):\n fp = open(file, 'rb')\n if sys.version_info.major == 2:\n data = pickle.load(fp)\n elif sys.version_info.major == 3:\n data = pickle.load(fp, encoding='latin-1')\n fp.close()\n return data\n\n filename = 'cifar-10-python.tar.gz'\n url = 'https://www.cs.toronto.edu/~kriz/'\n # Download and uncompress file\n maybe_download_and_extract(filename, path, url, extract=True)\n\n # Unpickle file and fill in data\n X_train = None\n y_train = []\n for i in range(1, 6):\n data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', \"data_batch_{}\".format(i)))\n if i == 1:\n X_train = data_dic['data']\n else:\n X_train = np.vstack((X_train, data_dic['data']))\n y_train += data_dic['labels']\n\n test_data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', \"test_batch\"))\n X_test = test_data_dic['data']\n y_test = np.array(test_data_dic['labels'])\n\n if shape == (-1, 3, 32, 32):\n X_test = X_test.reshape(shape)\n X_train = X_train.reshape(shape)\n elif shape == (-1, 32, 32, 3):\n X_test = X_test.reshape(shape, order='F')\n X_train = X_train.reshape(shape, order='F')\n X_test = np.transpose(X_test, (0, 2, 1, 3))\n X_train = np.transpose(X_train, (0, 2, 1, 3))\n else:\n X_test = X_test.reshape(shape)\n X_train = X_train.reshape(shape)\n\n y_train = np.array(y_train)\n\n if plotable:\n logging.info('\\nCIFAR-10')\n fig = plt.figure(1)\n\n logging.info('Shape of a training image: X_train[0] %s' % X_train[0].shape)\n\n plt.ion() # interactive mode\n count = 1\n for _ in range(10): # each row\n for _ in range(10): # each column\n _ = fig.add_subplot(10, 10, count)\n if shape == (-1, 3, 32, 32):\n # plt.imshow(X_train[count-1], interpolation='nearest')\n plt.imshow(np.transpose(X_train[count - 1], (1, 2, 0)), interpolation='nearest')\n # plt.imshow(np.transpose(X_train[count-1], (2, 1, 0)), interpolation='nearest')\n elif shape == (-1, 32, 32, 3):\n plt.imshow(X_train[count - 1], interpolation='nearest')\n # plt.imshow(np.transpose(X_train[count-1], (1, 0, 2)), interpolation='nearest')\n else:\n raise Exception(\"Do not support the given 'shape' to plot the image examples\")\n plt.gca().xaxis.set_major_locator(plt.NullLocator()) # \u4e0d\u663e\u793a\u523b\u5ea6(tick)\n plt.gca().yaxis.set_major_locator(plt.NullLocator())\n count = count + 1\n plt.draw() # interactive mode\n plt.pause(3) # interactive mode\n\n logging.info(\"X_train: %s\" % X_train.shape)\n logging.info(\"y_train: %s\" % y_train.shape)\n logging.info(\"X_test: %s\" % X_test.shape)\n logging.info(\"y_test: %s\" % y_test.shape)\n\n X_train = np.asarray(X_train, dtype=np.float32)\n X_test = np.asarray(X_test, dtype=np.float32)\n y_train = np.asarray(y_train, dtype=np.int32)\n y_test = np.asarray(y_test, dtype=np.int32)\n\n return X_train, y_train, X_test, y_test"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef load_cropped_svhn(path='data', include_extra=True):\n start_time = time.time()\n\n path = os.path.join(path, 'cropped_svhn')\n logging.info(\"Load or Download Cropped SVHN > {} | include extra images: {}\".format(path, include_extra))\n url = \"http://ufldl.stanford.edu/housenumbers/\"\n\n np_file = os.path.join(path, \"train_32x32.npz\")\n if file_exists(np_file) is False:\n filename = \"train_32x32.mat\"\n filepath = maybe_download_and_extract(filename, path, url)\n mat = sio.loadmat(filepath)\n X_train = mat['X'] / 255.0 # to [0, 1]\n X_train = np.transpose(X_train, (3, 0, 1, 2))\n y_train = np.squeeze(mat['y'], axis=1)\n y_train[y_train == 10] = 0 # replace 10 to 0\n np.savez(np_file, X=X_train, y=y_train)\n del_file(filepath)\n else:\n v = np.load(np_file)\n X_train = v['X']\n y_train = v['y']\n logging.info(\" n_train: {}\".format(len(y_train)))\n\n np_file = os.path.join(path, \"test_32x32.npz\")\n if file_exists(np_file) is False:\n filename = \"test_32x32.mat\"\n filepath = maybe_download_and_extract(filename, path, url)\n mat = sio.loadmat(filepath)\n X_test = mat['X'] / 255.0\n X_test = np.transpose(X_test, (3, 0, 1, 2))\n y_test = np.squeeze(mat['y'], axis=1)\n y_test[y_test == 10] = 0\n np.savez(np_file, X=X_test, y=y_test)\n del_file(filepath)\n else:\n v = np.load(np_file)\n X_test = v['X']\n y_test = v['y']\n logging.info(\" n_test: {}\".format(len(y_test)))\n\n if include_extra:\n logging.info(\" getting extra 531131 images, please wait ...\")\n np_file = os.path.join(path, \"extra_32x32.npz\")\n if file_exists(np_file) is False:\n logging.info(\" the first time to load extra images will take long time to convert the file format ...\")\n filename = \"extra_32x32.mat\"\n filepath = maybe_download_and_extract(filename, path, url)\n mat = sio.loadmat(filepath)\n X_extra = mat['X'] / 255.0\n X_extra = np.transpose(X_extra, (3, 0, 1, 2))\n y_extra = np.squeeze(mat['y'], axis=1)\n y_extra[y_extra == 10] = 0\n np.savez(np_file, X=X_extra, y=y_extra)\n del_file(filepath)\n else:\n v = np.load(np_file)\n X_extra = v['X']\n y_extra = v['y']\n # print(X_train.shape, X_extra.shape)\n logging.info(\" adding n_extra {} to n_train {}\".format(len(y_extra), len(y_train)))\n t = time.time()\n X_train = np.concatenate((X_train, X_extra), 0)\n y_train = np.concatenate((y_train, y_extra), 0)\n # X_train = np.append(X_train, X_extra, axis=0)\n # y_train = np.append(y_train, y_extra, axis=0)\n logging.info(\" added n_extra {} to n_train {} took {}s\".format(len(y_extra), len(y_train), time.time() - t))\n else:\n logging.info(\" no extra images are included\")\n logging.info(\" image size: %s n_train: %d n_test: %d\" % (str(X_train.shape[1:4]), len(y_train), len(y_test)))\n logging.info(\" took: {}s\".format(int(time.time() - start_time)))\n return X_train, y_train, X_test, y_test", "response": "Load Cropped SVHN dataset."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_ptb_dataset(path='data'):\n path = os.path.join(path, 'ptb')\n logging.info(\"Load or Download Penn TreeBank (PTB) dataset > {}\".format(path))\n\n # Maybe dowload and uncompress tar, or load exsisting files\n filename = 'simple-examples.tgz'\n url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'\n maybe_download_and_extract(filename, path, url, extract=True)\n\n data_path = os.path.join(path, 'simple-examples', 'data')\n train_path = os.path.join(data_path, \"ptb.train.txt\")\n valid_path = os.path.join(data_path, \"ptb.valid.txt\")\n test_path = os.path.join(data_path, \"ptb.test.txt\")\n\n word_to_id = nlp.build_vocab(nlp.read_words(train_path))\n\n train_data = nlp.words_to_word_ids(nlp.read_words(train_path), word_to_id)\n valid_data = nlp.words_to_word_ids(nlp.read_words(valid_path), word_to_id)\n test_data = nlp.words_to_word_ids(nlp.read_words(test_path), word_to_id)\n vocab_size = len(word_to_id)\n\n # logging.info(nlp.read_words(train_path)) # ... 'according', 'to', 'mr.', '', '']\n # logging.info(train_data) # ... 214, 5, 23, 1, 2]\n # logging.info(word_to_id) # ... 'beyond': 1295, 'anti-nuclear': 9599, 'trouble': 1520, '': 2 ... }\n # logging.info(vocabulary) # 10000\n # exit()\n return train_data, valid_data, test_data, vocab_size", "response": "Loads a Penn TreeBank dataset."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load_matt_mahoney_text8_dataset(path='data'):\n path = os.path.join(path, 'mm_test8')\n logging.info(\"Load or Download matt_mahoney_text8 Dataset> {}\".format(path))\n\n filename = 'text8.zip'\n url = 'http://mattmahoney.net/dc/'\n maybe_download_and_extract(filename, path, url, expected_bytes=31344016)\n\n with zipfile.ZipFile(os.path.join(path, filename)) as f:\n word_list = f.read(f.namelist()[0]).split()\n for idx, _ in enumerate(word_list):\n word_list[idx] = word_list[idx].decode()\n return word_list", "response": "Load Matt Mahoney s dataset."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nload IMDB dataset. Parameters ---------- path : str The path that the data is downloaded to, defaults is ``data/imdb/``. nb_words : int Number of words to get. skip_top : int Top most frequent words to ignore (they will appear as oov_char value in the sequence data). maxlen : int Maximum sequence length. Any longer sequence will be truncated. seed : int Seed for reproducible data shuffling. start_char : int The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character. oov_char : int Words that were cut out because of the num_words or skip_top limit will be replaced with this character. index_from : int Index actual words with this index and higher. Examples -------- >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset( ... nb_words=20000, test_split=0.2) >>> print('X_train.shape', X_train.shape) (20000,) [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..] >>> print('y_train.shape', y_train.shape) (20000,) [1 0 0 ..., 1 0 1] References ----------- - `Modified from keras. `__", "response": "def load_imdb_dataset(\n path='data', nb_words=None, skip_top=0, maxlen=None, test_split=0.2, seed=113, start_char=1, oov_char=2,\n index_from=3\n):\n \"\"\"Load IMDB dataset.\n\n Parameters\n ----------\n path : str\n The path that the data is downloaded to, defaults is ``data/imdb/``.\n nb_words : int\n Number of words to get.\n skip_top : int\n Top most frequent words to ignore (they will appear as oov_char value in the sequence data).\n maxlen : int\n Maximum sequence length. Any longer sequence will be truncated.\n seed : int\n Seed for reproducible data shuffling.\n start_char : int\n The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character.\n oov_char : int\n Words that were cut out because of the num_words or skip_top limit will be replaced with this character.\n index_from : int\n Index actual words with this index and higher.\n\n Examples\n --------\n >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(\n ... nb_words=20000, test_split=0.2)\n >>> print('X_train.shape', X_train.shape)\n (20000,) [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]\n >>> print('y_train.shape', y_train.shape)\n (20000,) [1 0 0 ..., 1 0 1]\n\n References\n -----------\n - `Modified from keras. `__\n\n \"\"\"\n path = os.path.join(path, 'imdb')\n\n filename = \"imdb.pkl\"\n url = 'https://s3.amazonaws.com/text-datasets/'\n maybe_download_and_extract(filename, path, url)\n\n if filename.endswith(\".gz\"):\n f = gzip.open(os.path.join(path, filename), 'rb')\n else:\n f = open(os.path.join(path, filename), 'rb')\n\n X, labels = cPickle.load(f)\n f.close()\n\n np.random.seed(seed)\n np.random.shuffle(X)\n np.random.seed(seed)\n np.random.shuffle(labels)\n\n if start_char is not None:\n X = [[start_char] + [w + index_from for w in x] for x in X]\n elif index_from:\n X = [[w + index_from for w in x] for x in X]\n\n if maxlen:\n new_X = []\n new_labels = []\n for x, y in zip(X, labels):\n if len(x) < maxlen:\n new_X.append(x)\n new_labels.append(y)\n X = new_X\n labels = new_labels\n if not X:\n raise Exception(\n 'After filtering for sequences shorter than maxlen=' + str(maxlen) + ', no sequence was kept. '\n 'Increase maxlen.'\n )\n if not nb_words:\n nb_words = max([max(x) for x in X])\n\n # by convention, use 2 as OOV word\n # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)\n if oov_char is not None:\n X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]\n else:\n nX = []\n for x in X:\n nx = []\n for w in x:\n if (w >= nb_words or w < skip_top):\n nx.append(w)\n nX.append(nx)\n X = nX\n\n X_train = np.array(X[:int(len(X) * (1 - test_split))])\n y_train = np.array(labels[:int(len(X) * (1 - test_split))])\n\n X_test = np.array(X[int(len(X) * (1 - test_split)):])\n y_test = np.array(labels[int(len(X) * (1 - test_split)):])\n\n return X_train, y_train, X_test, y_test"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_nietzsche_dataset(path='data'):\n logging.info(\"Load or Download nietzsche dataset > {}\".format(path))\n path = os.path.join(path, 'nietzsche')\n\n filename = \"nietzsche.txt\"\n url = 'https://s3.amazonaws.com/text-datasets/'\n filepath = maybe_download_and_extract(filename, path, url)\n\n with open(filepath, \"r\") as f:\n words = f.read()\n return words", "response": "Load Nietzsche dataset.\n\n Parameters\n ----------\n path : str\n The path that the data is downloaded to, defaults is ``data/nietzsche/``.\n\n Returns\n --------\n str\n The content.\n\n Examples\n --------\n >>> see tutorial_generate_text.py\n >>> words = tl.files.load_nietzsche_dataset()\n >>> words = basic_clean_str(words)\n >>> words = words.split()"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload the WMT en - fr dataset.", "response": "def load_wmt_en_fr_dataset(path='data'):\n \"\"\"Load WMT'15 English-to-French translation dataset.\n\n It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.\n Returns the directories of training data and test data.\n\n Parameters\n ----------\n path : str\n The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.\n\n References\n ----------\n - Code modified from /tensorflow/models/rnn/translation/data_utils.py\n\n Notes\n -----\n Usually, it will take a long time to download this dataset.\n\n \"\"\"\n path = os.path.join(path, 'wmt_en_fr')\n # URLs for WMT data.\n _WMT_ENFR_TRAIN_URL = \"http://www.statmt.org/wmt10/\"\n _WMT_ENFR_DEV_URL = \"http://www.statmt.org/wmt15/\"\n\n def gunzip_file(gz_path, new_path):\n \"\"\"Unzips from gz_path into new_path.\"\"\"\n logging.info(\"Unpacking %s to %s\" % (gz_path, new_path))\n with gzip.open(gz_path, \"rb\") as gz_file:\n with open(new_path, \"wb\") as new_file:\n for line in gz_file:\n new_file.write(line)\n\n def get_wmt_enfr_train_set(path):\n \"\"\"Download the WMT en-fr training corpus to directory unless it's there.\"\"\"\n filename = \"training-giga-fren.tar\"\n maybe_download_and_extract(filename, path, _WMT_ENFR_TRAIN_URL, extract=True)\n train_path = os.path.join(path, \"giga-fren.release2.fixed\")\n gunzip_file(train_path + \".fr.gz\", train_path + \".fr\")\n gunzip_file(train_path + \".en.gz\", train_path + \".en\")\n return train_path\n\n def get_wmt_enfr_dev_set(path):\n \"\"\"Download the WMT en-fr training corpus to directory unless it's there.\"\"\"\n filename = \"dev-v2.tgz\"\n dev_file = maybe_download_and_extract(filename, path, _WMT_ENFR_DEV_URL, extract=False)\n dev_name = \"newstest2013\"\n dev_path = os.path.join(path, \"newstest2013\")\n if not (gfile.Exists(dev_path + \".fr\") and gfile.Exists(dev_path + \".en\")):\n logging.info(\"Extracting tgz file %s\" % dev_file)\n with tarfile.open(dev_file, \"r:gz\") as dev_tar:\n fr_dev_file = dev_tar.getmember(\"dev/\" + dev_name + \".fr\")\n en_dev_file = dev_tar.getmember(\"dev/\" + dev_name + \".en\")\n fr_dev_file.name = dev_name + \".fr\" # Extract without \"dev/\" prefix.\n en_dev_file.name = dev_name + \".en\"\n dev_tar.extract(fr_dev_file, path)\n dev_tar.extract(en_dev_file, path)\n return dev_path\n\n logging.info(\"Load or Download WMT English-to-French translation > {}\".format(path))\n\n train_path = get_wmt_enfr_train_set(path)\n dev_path = get_wmt_enfr_dev_set(path)\n\n return train_path, dev_path"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef load_flickr25k_dataset(tag='sky', path=\"data\", n_threads=50, printable=False):\n path = os.path.join(path, 'flickr25k')\n\n filename = 'mirflickr25k.zip'\n url = 'http://press.liacs.nl/mirflickr/mirflickr25k/'\n\n # download dataset\n if folder_exists(os.path.join(path, \"mirflickr\")) is False:\n logging.info(\"[*] Flickr25k is nonexistent in {}\".format(path))\n maybe_download_and_extract(filename, path, url, extract=True)\n del_file(os.path.join(path, filename))\n\n # return images by the given tag.\n # 1. image path list\n folder_imgs = os.path.join(path, \"mirflickr\")\n path_imgs = load_file_list(path=folder_imgs, regx='\\\\.jpg', printable=False)\n path_imgs.sort(key=natural_keys)\n\n # 2. tag path list\n folder_tags = os.path.join(path, \"mirflickr\", \"meta\", \"tags\")\n path_tags = load_file_list(path=folder_tags, regx='\\\\.txt', printable=False)\n path_tags.sort(key=natural_keys)\n\n # 3. select images\n if tag is None:\n logging.info(\"[Flickr25k] reading all images\")\n else:\n logging.info(\"[Flickr25k] reading images with tag: {}\".format(tag))\n images_list = []\n for idx, _v in enumerate(path_tags):\n tags = read_file(os.path.join(folder_tags, path_tags[idx])).split('\\n')\n # logging.info(idx+1, tags)\n if tag is None or tag in tags:\n images_list.append(path_imgs[idx])\n\n images = visualize.read_images(images_list, folder_imgs, n_threads=n_threads, printable=printable)\n return images", "response": "Load a list of images by a given tag from Flick25k dataset."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads Flick1M dataset. Returns a list of images by a given tag from Flickr1M dataset, it will download Flickr1M from `the official website `__ at the first time you use it. Parameters ------------ tag : str or None What images to return. - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search `__. - If you want to get all images, set to ``None``. size : int integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10. path : str The path that the data is downloaded to, defaults is ``data/flickr25k/``. n_threads : int The number of thread to read image. printable : boolean Whether to print infomation when reading images, default is ``False``. Examples ---------- Use 200k images >>> images = tl.files.load_flickr1M_dataset(tag='zebra', size=2) Use 1 Million images >>> images = tl.files.load_flickr1M_dataset(tag='zebra')", "response": "def load_flickr1M_dataset(tag='sky', size=10, path=\"data\", n_threads=50, printable=False):\n \"\"\"Load Flick1M dataset.\n\n Returns a list of images by a given tag from Flickr1M dataset,\n it will download Flickr1M from `the official website `__\n at the first time you use it.\n\n Parameters\n ------------\n tag : str or None\n What images to return.\n - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search `__.\n - If you want to get all images, set to ``None``.\n\n size : int\n integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10.\n path : str\n The path that the data is downloaded to, defaults is ``data/flickr25k/``.\n n_threads : int\n The number of thread to read image.\n printable : boolean\n Whether to print infomation when reading images, default is ``False``.\n\n Examples\n ----------\n Use 200k images\n\n >>> images = tl.files.load_flickr1M_dataset(tag='zebra', size=2)\n\n Use 1 Million images\n\n >>> images = tl.files.load_flickr1M_dataset(tag='zebra')\n\n \"\"\"\n path = os.path.join(path, 'flickr1M')\n logging.info(\"[Flickr1M] using {}% of images = {}\".format(size * 10, size * 100000))\n images_zip = [\n 'images0.zip', 'images1.zip', 'images2.zip', 'images3.zip', 'images4.zip', 'images5.zip', 'images6.zip',\n 'images7.zip', 'images8.zip', 'images9.zip'\n ]\n tag_zip = 'tags.zip'\n url = 'http://press.liacs.nl/mirflickr/mirflickr1m/'\n\n # download dataset\n for image_zip in images_zip[0:size]:\n image_folder = image_zip.split(\".\")[0]\n # logging.info(path+\"/\"+image_folder)\n if folder_exists(os.path.join(path, image_folder)) is False:\n # logging.info(image_zip)\n logging.info(\"[Flickr1M] {} is missing in {}\".format(image_folder, path))\n maybe_download_and_extract(image_zip, path, url, extract=True)\n del_file(os.path.join(path, image_zip))\n # os.system(\"mv {} {}\".format(os.path.join(path, 'images'), os.path.join(path, image_folder)))\n shutil.move(os.path.join(path, 'images'), os.path.join(path, image_folder))\n else:\n logging.info(\"[Flickr1M] {} exists in {}\".format(image_folder, path))\n\n # download tag\n if folder_exists(os.path.join(path, \"tags\")) is False:\n logging.info(\"[Flickr1M] tag files is nonexistent in {}\".format(path))\n maybe_download_and_extract(tag_zip, path, url, extract=True)\n del_file(os.path.join(path, tag_zip))\n else:\n logging.info(\"[Flickr1M] tags exists in {}\".format(path))\n\n # 1. image path list\n images_list = []\n images_folder_list = []\n for i in range(0, size):\n images_folder_list += load_folder_list(path=os.path.join(path, 'images%d' % i))\n images_folder_list.sort(key=lambda s: int(s.split('/')[-1])) # folder/images/ddd\n\n for folder in images_folder_list[0:size * 10]:\n tmp = load_file_list(path=folder, regx='\\\\.jpg', printable=False)\n tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.jpg\n images_list.extend([os.path.join(folder, x) for x in tmp])\n\n # 2. tag path list\n tag_list = []\n tag_folder_list = load_folder_list(os.path.join(path, \"tags\"))\n\n # tag_folder_list.sort(key=lambda s: int(s.split(\"/\")[-1])) # folder/images/ddd\n tag_folder_list.sort(key=lambda s: int(os.path.basename(s)))\n\n for folder in tag_folder_list[0:size * 10]:\n tmp = load_file_list(path=folder, regx='\\\\.txt', printable=False)\n tmp.sort(key=lambda s: int(s.split('.')[-2])) # ddd.txt\n tmp = [os.path.join(folder, s) for s in tmp]\n tag_list += tmp\n\n # 3. select images\n logging.info(\"[Flickr1M] searching tag: {}\".format(tag))\n select_images_list = []\n for idx, _val in enumerate(tag_list):\n tags = read_file(tag_list[idx]).split('\\n')\n if tag in tags:\n select_images_list.append(images_list[idx])\n\n logging.info(\"[Flickr1M] reading images with tag: {}\".format(tag))\n images = visualize.read_images(select_images_list, '', n_threads=n_threads, printable=printable)\n return images"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load_cyclegan_dataset(filename='summer2winter_yosemite', path='data'):\n path = os.path.join(path, 'cyclegan')\n url = 'https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/'\n\n if folder_exists(os.path.join(path, filename)) is False:\n logging.info(\"[*] {} is nonexistent in {}\".format(filename, path))\n maybe_download_and_extract(filename + '.zip', path, url, extract=True)\n del_file(os.path.join(path, filename + '.zip'))\n\n def load_image_from_folder(path):\n path_imgs = load_file_list(path=path, regx='\\\\.jpg', printable=False)\n return visualize.read_images(path_imgs, path=path, n_threads=10, printable=False)\n\n im_train_A = load_image_from_folder(os.path.join(path, filename, \"trainA\"))\n im_train_B = load_image_from_folder(os.path.join(path, filename, \"trainB\"))\n im_test_A = load_image_from_folder(os.path.join(path, filename, \"testA\"))\n im_test_B = load_image_from_folder(os.path.join(path, filename, \"testB\"))\n\n def if_2d_to_3d(images): # [h, w] --> [h, w, 3]\n for i, _v in enumerate(images):\n if len(images[i].shape) == 2:\n images[i] = images[i][:, :, np.newaxis]\n images[i] = np.tile(images[i], (1, 1, 3))\n return images\n\n im_train_A = if_2d_to_3d(im_train_A)\n im_train_B = if_2d_to_3d(im_train_B)\n im_test_A = if_2d_to_3d(im_test_A)\n im_test_B = if_2d_to_3d(im_test_B)\n\n return im_train_A, im_train_B, im_test_A, im_test_B", "response": "Loads the CycleGAN dataset from the CycleGAN s database."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef download_file_from_google_drive(ID, destination):\n\n def save_response_content(response, destination, chunk_size=32 * 1024):\n total_size = int(response.headers.get('content-length', 0))\n with open(destination, \"wb\") as f:\n for chunk in tqdm(response.iter_content(chunk_size), total=total_size, unit='B', unit_scale=True,\n desc=destination):\n if chunk: # filter out keep-alive new chunks\n f.write(chunk)\n\n def get_confirm_token(response):\n for key, value in response.cookies.items():\n if key.startswith('download_warning'):\n return value\n return None\n\n URL = \"https://docs.google.com/uc?export=download\"\n session = requests.Session()\n\n response = session.get(URL, params={'id': ID}, stream=True)\n token = get_confirm_token(response)\n\n if token:\n params = {'id': ID, 'confirm': token}\n response = session.get(URL, params=params, stream=True)\n save_response_content(response, destination)", "response": "Download file from Google Drive."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads CelebA dataset Return a list of image path. Parameters ----------- path : str The path that the data is downloaded to, defaults is ``data/celebA/``.", "response": "def load_celebA_dataset(path='data'):\n \"\"\"Load CelebA dataset\n\n Return a list of image path.\n\n Parameters\n -----------\n path : str\n The path that the data is downloaded to, defaults is ``data/celebA/``.\n\n \"\"\"\n data_dir = 'celebA'\n filename, drive_id = \"img_align_celeba.zip\", \"0B7EVK8r0v71pZjFTYXZWM3FlRnM\"\n save_path = os.path.join(path, filename)\n image_path = os.path.join(path, data_dir)\n if os.path.exists(image_path):\n logging.info('[*] {} already exists'.format(save_path))\n else:\n exists_or_mkdir(path)\n download_file_from_google_drive(drive_id, save_path)\n zip_dir = ''\n with zipfile.ZipFile(save_path) as zf:\n zip_dir = zf.namelist()[0]\n zf.extractall(path)\n os.remove(save_path)\n os.rename(os.path.join(path, zip_dir), image_path)\n\n data_files = load_file_list(path=image_path, regx='\\\\.jpg', printable=False)\n for i, _v in enumerate(data_files):\n data_files[i] = os.path.join(image_path, data_files[i])\n return data_files"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsave parameters and the file name save parameters into. npz file.", "response": "def save_npz(save_list=None, name='model.npz', sess=None):\n \"\"\"Input parameters and the file name, save parameters into .npz file. Use tl.utils.load_npz() to restore.\n\n Parameters\n ----------\n save_list : list of tensor\n A list of parameters (tensor) to be saved.\n name : str\n The name of the `.npz` file.\n sess : None or Session\n Session may be required in some case.\n\n Examples\n --------\n Save model to npz\n\n >>> tl.files.save_npz(network.all_params, name='model.npz', sess=sess)\n\n Load model from npz (Method 1)\n\n >>> load_params = tl.files.load_npz(name='model.npz')\n >>> tl.files.assign_params(sess, load_params, network)\n\n Load model from npz (Method 2)\n\n >>> tl.files.load_and_assign_npz(sess=sess, name='model.npz', network=network)\n\n Notes\n -----\n If you got session issues, you can change the value.eval() to value.eval(session=sess)\n\n References\n ----------\n `Saving dictionary using numpy `__\n\n \"\"\"\n logging.info(\"[*] Saving TL params into %s\" % name)\n if save_list is None:\n save_list = []\n\n save_list_var = []\n if sess:\n save_list_var = sess.run(save_list)\n else:\n try:\n save_list_var.extend([v.eval() for v in save_list])\n except Exception:\n logging.info(\n \" Fail to save model, Hint: pass the session into this function, tl.files.save_npz(network.all_params, name='model.npz', sess=sess)\"\n )\n np.savez(name, params=save_list_var)\n save_list_var = None\n del save_list_var\n logging.info(\"[*] Saved\")"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_npz(path='', name='model.npz'):\n d = np.load(os.path.join(path, name))\n return d['params']", "response": "Loads the parameters of a Model saved by tl. files. save_npz."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nassign the given parameters to the TensorFlow Session.", "response": "def assign_params(sess, params, network):\n \"\"\"Assign the given parameters to the TensorLayer network.\n\n Parameters\n ----------\n sess : Session\n TensorFlow Session.\n params : list of array\n A list of parameters (array) in order.\n network : :class:`Layer`\n The network to be assigned.\n\n Returns\n --------\n list of operations\n A list of tf ops in order that assign params. Support sess.run(ops) manually.\n\n Examples\n --------\n - See ``tl.files.save_npz``\n\n References\n ----------\n - `Assign value to a TensorFlow variable `__\n\n \"\"\"\n ops = []\n for idx, param in enumerate(params):\n ops.append(network.all_params[idx].assign(param))\n if sess is not None:\n sess.run(ops)\n return ops"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading a model from npz and assign it to a network.", "response": "def load_and_assign_npz(sess=None, name=None, network=None):\n \"\"\"Load model from npz and assign to a network.\n\n Parameters\n -------------\n sess : Session\n TensorFlow Session.\n name : str\n The name of the `.npz` file.\n network : :class:`Layer`\n The network to be assigned.\n\n Returns\n --------\n False or network\n Returns False, if the model is not exist.\n\n Examples\n --------\n - See ``tl.files.save_npz``\n\n \"\"\"\n if network is None:\n raise ValueError(\"network is None.\")\n if sess is None:\n raise ValueError(\"session is None.\")\n if not os.path.exists(name):\n logging.error(\"file {} doesn't exist.\".format(name))\n return False\n else:\n params = load_npz(name=name)\n assign_params(sess, params, network)\n logging.info(\"[*] Load {} SUCCESS!\".format(name))\n return network"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsaves parameters and the file name into. npz file.", "response": "def save_npz_dict(save_list=None, name='model.npz', sess=None):\n \"\"\"Input parameters and the file name, save parameters as a dictionary into .npz file.\n\n Use ``tl.files.load_and_assign_npz_dict()`` to restore.\n\n Parameters\n ----------\n save_list : list of parameters\n A list of parameters (tensor) to be saved.\n name : str\n The name of the `.npz` file.\n sess : Session\n TensorFlow Session.\n\n \"\"\"\n if sess is None:\n raise ValueError(\"session is None.\")\n if save_list is None:\n save_list = []\n\n save_list_names = [tensor.name for tensor in save_list]\n save_list_var = sess.run(save_list)\n save_var_dict = {save_list_names[idx]: val for idx, val in enumerate(save_list_var)}\n np.savez(name, **save_var_dict)\n save_list_var = None\n save_var_dict = None\n del save_list_var\n del save_var_dict\n logging.info(\"[*] Model saved in npz_dict %s\" % name)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nload and assign the parameters saved by save_npz_dict.", "response": "def load_and_assign_npz_dict(name='model.npz', sess=None):\n \"\"\"Restore the parameters saved by ``tl.files.save_npz_dict()``.\n\n Parameters\n ----------\n name : str\n The name of the `.npz` file.\n sess : Session\n TensorFlow Session.\n\n \"\"\"\n if sess is None:\n raise ValueError(\"session is None.\")\n\n if not os.path.exists(name):\n logging.error(\"file {} doesn't exist.\".format(name))\n return False\n\n params = np.load(name)\n if len(params.keys()) != len(set(params.keys())):\n raise Exception(\"Duplication in model npz_dict %s\" % name)\n ops = list()\n for key in params.keys():\n try:\n # tensor = tf.get_default_graph().get_tensor_by_name(key)\n # varlist = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=key)\n varlist = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=key)\n if len(varlist) > 1:\n raise Exception(\"[!] Multiple candidate variables to be assigned for name %s\" % key)\n elif len(varlist) == 0:\n raise KeyError\n else:\n ops.append(varlist[0].assign(params[key]))\n logging.info(\"[*] params restored: %s\" % key)\n except KeyError:\n logging.info(\"[!] Warning: Tensor named %s not found in network.\" % key)\n\n sess.run(ops)\n logging.info(\"[*] Model restored from npz_dict %s\" % name)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_ckpt(\n sess=None, mode_name='model.ckpt', save_dir='checkpoint', var_list=None, global_step=None, printable=False\n):\n \"\"\"Save parameters into `ckpt` file.\n\n Parameters\n ------------\n sess : Session\n TensorFlow Session.\n mode_name : str\n The name of the model, default is ``model.ckpt``.\n save_dir : str\n The path / file directory to the `ckpt`, default is ``checkpoint``.\n var_list : list of tensor\n The parameters / variables (tensor) to be saved. If empty, save all global variables (default).\n global_step : int or None\n Step number.\n printable : boolean\n Whether to print all parameters information.\n\n See Also\n --------\n load_ckpt\n\n \"\"\"\n if sess is None:\n raise ValueError(\"session is None.\")\n if var_list is None:\n var_list = []\n\n ckpt_file = os.path.join(save_dir, mode_name)\n if var_list == []:\n var_list = tf.global_variables()\n\n logging.info(\"[*] save %s n_params: %d\" % (ckpt_file, len(var_list)))\n\n if printable:\n for idx, v in enumerate(var_list):\n logging.info(\" param {:3}: {:15} {}\".format(idx, v.name, str(v.get_shape())))\n\n saver = tf.train.Saver(var_list)\n saver.save(sess, ckpt_file, global_step=global_step)", "response": "Save parameters into CKPT file."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads parameters from the CKPT file.", "response": "def load_ckpt(sess=None, mode_name='model.ckpt', save_dir='checkpoint', var_list=None, is_latest=True, printable=False):\n \"\"\"Load parameters from `ckpt` file.\n\n Parameters\n ------------\n sess : Session\n TensorFlow Session.\n mode_name : str\n The name of the model, default is ``model.ckpt``.\n save_dir : str\n The path / file directory to the `ckpt`, default is ``checkpoint``.\n var_list : list of tensor\n The parameters / variables (tensor) to be saved. If empty, save all global variables (default).\n is_latest : boolean\n Whether to load the latest `ckpt`, if False, load the `ckpt` with the name of ```mode_name``.\n printable : boolean\n Whether to print all parameters information.\n\n Examples\n ----------\n - Save all global parameters.\n\n >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', save_dir='model', printable=True)\n\n - Save specific parameters.\n\n >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', printable=True)\n\n - Load latest ckpt.\n\n >>> tl.files.load_ckpt(sess=sess, var_list=net.all_params, save_dir='model', printable=True)\n\n - Load specific ckpt.\n\n >>> tl.files.load_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', is_latest=False, printable=True)\n\n \"\"\"\n if sess is None:\n raise ValueError(\"session is None.\")\n if var_list is None:\n var_list = []\n\n if is_latest:\n ckpt_file = tf.train.latest_checkpoint(save_dir)\n else:\n ckpt_file = os.path.join(save_dir, mode_name)\n\n if not var_list:\n var_list = tf.global_variables()\n\n logging.info(\"[*] load %s n_params: %d\" % (ckpt_file, len(var_list)))\n\n if printable:\n for idx, v in enumerate(var_list):\n logging.info(\" param {:3}: {:15} {}\".format(idx, v.name, str(v.get_shape())))\n\n try:\n saver = tf.train.Saver(var_list)\n saver.restore(sess, ckpt_file)\n except Exception as e:\n logging.info(e)\n logging.info(\"[*] load ckpt fail ...\")"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading any. npy file.", "response": "def load_npy_to_any(path='', name='file.npy'):\n \"\"\"Load `.npy` file.\n\n Parameters\n ------------\n path : str\n Path to the file (optional).\n name : str\n File name.\n\n Examples\n ---------\n - see tl.files.save_any_to_npy()\n\n \"\"\"\n file_path = os.path.join(path, name)\n try:\n return np.load(file_path).item()\n except Exception:\n return np.load(file_path)\n raise Exception(\"[!] Fail to load %s\" % file_path)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_folder_list(path=\"\"):\n return [os.path.join(path, o) for o in os.listdir(path) if os.path.isdir(os.path.join(path, o))]", "response": "Return a list of folder names in a folder by given a folder path."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck a folder by given name and creates it if it does not exist.", "response": "def exists_or_mkdir(path, verbose=True):\n \"\"\"Check a folder by given name, if not exist, create the folder and return False,\n if directory exists, return True.\n\n Parameters\n ----------\n path : str\n A folder path.\n verbose : boolean\n If True (default), prints results.\n\n Returns\n --------\n boolean\n True if folder already exist, otherwise, returns False and create the folder.\n\n Examples\n --------\n >>> tl.files.exists_or_mkdir(\"checkpoints/train\")\n\n \"\"\"\n if not os.path.exists(path):\n if verbose:\n logging.info(\"[*] creates %s ...\" % path)\n os.makedirs(path)\n return False\n else:\n if verbose:\n logging.info(\"[!] %s exists ...\" % path)\n return True"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndownloading and extract the file from the given URL.", "response": "def maybe_download_and_extract(filename, working_directory, url_source, extract=False, expected_bytes=None):\n \"\"\"Checks if file exists in working_directory otherwise tries to dowload the file,\n and optionally also tries to extract the file if format is \".zip\" or \".tar\"\n\n Parameters\n -----------\n filename : str\n The name of the (to be) dowloaded file.\n working_directory : str\n A folder path to search for the file in and dowload the file to\n url : str\n The URL to download the file from\n extract : boolean\n If True, tries to uncompress the dowloaded file is \".tar.gz/.tar.bz2\" or \".zip\" file, default is False.\n expected_bytes : int or None\n If set tries to verify that the downloaded file is of the specified size, otherwise raises an Exception, defaults is None which corresponds to no check being performed.\n\n Returns\n ----------\n str\n File path of the dowloaded (uncompressed) file.\n\n Examples\n --------\n >>> down_file = tl.files.maybe_download_and_extract(filename='train-images-idx3-ubyte.gz',\n ... working_directory='data/',\n ... url_source='http://yann.lecun.com/exdb/mnist/')\n >>> tl.files.maybe_download_and_extract(filename='ADEChallengeData2016.zip',\n ... working_directory='data/',\n ... url_source='http://sceneparsing.csail.mit.edu/data/',\n ... extract=True)\n\n \"\"\"\n\n # We first define a download function, supporting both Python 2 and 3.\n def _download(filename, working_directory, url_source):\n\n progress_bar = progressbar.ProgressBar()\n\n def _dlProgress(count, blockSize, totalSize, pbar=progress_bar):\n if (totalSize != 0):\n\n if not pbar.max_value:\n totalBlocks = math.ceil(float(totalSize) / float(blockSize))\n pbar.max_value = int(totalBlocks)\n\n pbar.update(count, force=True)\n\n filepath = os.path.join(working_directory, filename)\n\n logging.info('Downloading %s...\\n' % filename)\n\n urlretrieve(url_source + filename, filepath, reporthook=_dlProgress)\n\n exists_or_mkdir(working_directory, verbose=False)\n filepath = os.path.join(working_directory, filename)\n\n if not os.path.exists(filepath):\n\n _download(filename, working_directory, url_source)\n statinfo = os.stat(filepath)\n logging.info('Succesfully downloaded %s %s bytes.' % (filename, statinfo.st_size)) # , 'bytes.')\n if (not (expected_bytes is None) and (expected_bytes != statinfo.st_size)):\n raise Exception('Failed to verify ' + filename + '. Can you get to it with a browser?')\n if (extract):\n if tarfile.is_tarfile(filepath):\n logging.info('Trying to extract tar file')\n tarfile.open(filepath, 'r').extractall(working_directory)\n logging.info('... Success!')\n elif zipfile.is_zipfile(filepath):\n logging.info('Trying to extract zip file')\n with zipfile.ZipFile(filepath) as zf:\n zf.extractall(working_directory)\n logging.info('... Success!')\n else:\n logging.info(\"Unknown compression_format only .tar.gz/.tar.bz2/.tar and .zip supported\")\n return filepath"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsorting list of string with number in human order.", "response": "def natural_keys(text):\n \"\"\"Sort list of string with number in human order.\n\n Examples\n ----------\n >>> l = ['im1.jpg', 'im31.jpg', 'im11.jpg', 'im21.jpg', 'im03.jpg', 'im05.jpg']\n >>> l.sort(key=tl.files.natural_keys)\n ['im1.jpg', 'im03.jpg', 'im05', 'im11.jpg', 'im21.jpg', 'im31.jpg']\n >>> l.sort() # that is what we dont want\n ['im03.jpg', 'im05', 'im1.jpg', 'im11.jpg', 'im21.jpg', 'im31.jpg']\n\n References\n ----------\n - `link `__\n\n \"\"\"\n\n # - alist.sort(key=natural_keys) sorts in human order\n # http://nedbatchelder.com/blog/200712/human_sorting.html\n # (See Toothy's implementation in the comments)\n def atoi(text):\n return int(text) if text.isdigit() else text\n\n return [atoi(c) for c in re.split('(\\d+)', text)]"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef npz_to_W_pdf(path=None, regx='w1pre_[0-9]+\\.(npz)'):\n file_list = load_file_list(path=path, regx=regx)\n for f in file_list:\n W = load_npz(path, f)[0]\n logging.info(\"%s --> %s\" % (f, f.split('.')[0] + '.pdf'))\n visualize.draw_weights(W, second=10, saveable=True, name=f.split('.')[0], fig_idx=2012)", "response": "r Converts the first weight matrix of. npz file to. pdf by using tl. visualize. W."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprocesses a batch of data by given function by threading.", "response": "def threading_data(data=None, fn=None, thread_count=None, **kwargs):\n \"\"\"Process a batch of data by given function by threading.\n\n Usually be used for data augmentation.\n\n Parameters\n -----------\n data : numpy.array or others\n The data to be processed.\n thread_count : int\n The number of threads to use.\n fn : function\n The function for data processing.\n more args : the args for `fn`\n Ssee Examples below.\n\n Examples\n --------\n Process images.\n\n >>> images, _, _, _ = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))\n >>> images = tl.prepro.threading_data(images[0:32], tl.prepro.zoom, zoom_range=[0.5, 1])\n\n Customized image preprocessing function.\n\n >>> def distort_img(x):\n >>> x = tl.prepro.flip_axis(x, axis=0, is_random=True)\n >>> x = tl.prepro.flip_axis(x, axis=1, is_random=True)\n >>> x = tl.prepro.crop(x, 100, 100, is_random=True)\n >>> return x\n >>> images = tl.prepro.threading_data(images, distort_img)\n\n Process images and masks together (Usually be used for image segmentation).\n\n >>> X, Y --> [batch_size, row, col, 1]\n >>> data = tl.prepro.threading_data([_ for _ in zip(X, Y)], tl.prepro.zoom_multi, zoom_range=[0.5, 1], is_random=True)\n data --> [batch_size, 2, row, col, 1]\n >>> X_, Y_ = data.transpose((1,0,2,3,4))\n X_, Y_ --> [batch_size, row, col, 1]\n >>> tl.vis.save_image(X_, 'images.png')\n >>> tl.vis.save_image(Y_, 'masks.png')\n\n Process images and masks together by using ``thread_count``.\n\n >>> X, Y --> [batch_size, row, col, 1]\n >>> data = tl.prepro.threading_data(X, tl.prepro.zoom_multi, 8, zoom_range=[0.5, 1], is_random=True)\n data --> [batch_size, 2, row, col, 1]\n >>> X_, Y_ = data.transpose((1,0,2,3,4))\n X_, Y_ --> [batch_size, row, col, 1]\n >>> tl.vis.save_image(X_, 'after.png')\n >>> tl.vis.save_image(Y_, 'before.png')\n\n Customized function for processing images and masks together.\n\n >>> def distort_img(data):\n >>> x, y = data\n >>> x, y = tl.prepro.flip_axis_multi([x, y], axis=0, is_random=True)\n >>> x, y = tl.prepro.flip_axis_multi([x, y], axis=1, is_random=True)\n >>> x, y = tl.prepro.crop_multi([x, y], 100, 100, is_random=True)\n >>> return x, y\n\n >>> X, Y --> [batch_size, row, col, channel]\n >>> data = tl.prepro.threading_data([_ for _ in zip(X, Y)], distort_img)\n >>> X_, Y_ = data.transpose((1,0,2,3,4))\n\n Returns\n -------\n list or numpyarray\n The processed results.\n\n References\n ----------\n - `python queue `__\n - `run with limited queue `__\n\n \"\"\"\n\n def apply_fn(results, i, data, kwargs):\n results[i] = fn(data, **kwargs)\n\n if thread_count is None:\n results = [None] * len(data)\n threads = []\n # for i in range(len(data)):\n # t = threading.Thread(name='threading_and_return', target=apply_fn, args=(results, i, data[i], kwargs))\n for i, d in enumerate(data):\n t = threading.Thread(name='threading_and_return', target=apply_fn, args=(results, i, d, kwargs))\n t.start()\n threads.append(t)\n else:\n divs = np.linspace(0, len(data), thread_count + 1)\n divs = np.round(divs).astype(int)\n results = [None] * thread_count\n threads = []\n for i in range(thread_count):\n t = threading.Thread(\n name='threading_and_return', target=apply_fn, args=(results, i, data[divs[i]:divs[i + 1]], kwargs)\n )\n t.start()\n threads.append(t)\n\n for t in threads:\n t.join()\n\n if thread_count is None:\n try:\n return np.asarray(results)\n except Exception:\n return results\n else:\n return np.concatenate(results)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate an affine transform matrix for image rotation.", "response": "def affine_rotation_matrix(angle=(-20, 20)):\n \"\"\"Create an affine transform matrix for image rotation.\n NOTE: In OpenCV, x is width and y is height.\n\n Parameters\n -----------\n angle : int/float or tuple of two int/float\n Degree to rotate, usually -180 ~ 180.\n - int/float, a fixed angle.\n - tuple of 2 floats/ints, randomly sample a value as the angle between these 2 values.\n\n Returns\n -------\n numpy.array\n An affine transform matrix.\n\n \"\"\"\n if isinstance(angle, tuple):\n theta = np.pi / 180 * np.random.uniform(angle[0], angle[1])\n else:\n theta = np.pi / 180 * angle\n rotation_matrix = np.array([[np.cos(theta), np.sin(theta), 0], \\\n [-np.sin(theta), np.cos(theta), 0], \\\n [0, 0, 1]])\n return rotation_matrix"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating an affine transformation matrix for image horizontal flipping.", "response": "def affine_horizontal_flip_matrix(prob=0.5):\n \"\"\"Create an affine transformation matrix for image horizontal flipping.\n NOTE: In OpenCV, x is width and y is height.\n\n Parameters\n ----------\n prob : float\n Probability to flip the image. 1.0 means always flip.\n\n Returns\n -------\n numpy.array\n An affine transform matrix.\n\n \"\"\"\n factor = np.random.uniform(0, 1)\n if prob >= factor:\n filp_matrix = np.array([[ -1. , 0., 0. ], \\\n [ 0., 1., 0. ], \\\n [ 0., 0., 1. ]])\n return filp_matrix\n else:\n filp_matrix = np.array([[ 1. , 0., 0. ], \\\n [ 0., 1., 0. ], \\\n [ 0., 0., 1. ]])\n return filp_matrix"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef affine_vertical_flip_matrix(prob=0.5):\n factor = np.random.uniform(0, 1)\n if prob >= factor:\n filp_matrix = np.array([[ 1. , 0., 0. ], \\\n [ 0., -1., 0. ], \\\n [ 0., 0., 1. ]])\n return filp_matrix\n else:\n filp_matrix = np.array([[ 1. , 0., 0. ], \\\n [ 0., 1., 0. ], \\\n [ 0., 0., 1. ]])\n return filp_matrix", "response": "Create an affine transformation for image vertical flipping."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate an affine transform matrix for image shifting.", "response": "def affine_shift_matrix(wrg=(-0.1, 0.1), hrg=(-0.1, 0.1), w=200, h=200):\n \"\"\"Create an affine transform matrix for image shifting.\n NOTE: In OpenCV, x is width and y is height.\n\n Parameters\n -----------\n wrg : float or tuple of floats\n Range to shift on width axis, -1 ~ 1.\n - float, a fixed distance.\n - tuple of 2 floats, randomly sample a value as the distance between these 2 values.\n hrg : float or tuple of floats\n Range to shift on height axis, -1 ~ 1.\n - float, a fixed distance.\n - tuple of 2 floats, randomly sample a value as the distance between these 2 values.\n w, h : int\n The width and height of the image.\n\n Returns\n -------\n numpy.array\n An affine transform matrix.\n\n \"\"\"\n if isinstance(wrg, tuple):\n tx = np.random.uniform(wrg[0], wrg[1]) * w\n else:\n tx = wrg * w\n if isinstance(hrg, tuple):\n ty = np.random.uniform(hrg[0], hrg[1]) * h\n else:\n ty = hrg * h\n shift_matrix = np.array([[1, 0, tx], \\\n [0, 1, ty], \\\n [0, 0, 1]])\n return shift_matrix"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef affine_shear_matrix(x_shear=(-0.1, 0.1), y_shear=(-0.1, 0.1)):\n # if len(shear) != 2:\n # raise AssertionError(\n # \"shear should be tuple of 2 floats, or you want to use tl.prepro.shear rather than tl.prepro.shear2 ?\"\n # )\n # if isinstance(shear, tuple):\n # shear = list(shear)\n # if is_random:\n # shear[0] = np.random.uniform(-shear[0], shear[0])\n # shear[1] = np.random.uniform(-shear[1], shear[1])\n if isinstance(x_shear, tuple):\n x_shear = np.random.uniform(x_shear[0], x_shear[1])\n if isinstance(y_shear, tuple):\n y_shear = np.random.uniform(y_shear[0], y_shear[1])\n\n shear_matrix = np.array([[1, x_shear, 0], \\\n [y_shear, 1, 0], \\\n [0, 0, 1]])\n return shear_matrix", "response": "Create an affine transform matrix for image shearing."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef affine_zoom_matrix(zoom_range=(0.8, 1.1)):\n\n if isinstance(zoom_range, (float, int)):\n scale = zoom_range\n elif isinstance(zoom_range, tuple):\n scale = np.random.uniform(zoom_range[0], zoom_range[1])\n else:\n raise Exception(\"zoom_range: float or tuple of 2 floats\")\n\n zoom_matrix = np.array([[scale, 0, 0], \\\n [0, scale, 0], \\\n [0, 0, 1]])\n return zoom_matrix", "response": "Create an affine transform matrix for zooming and scaling an image s height and width."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets affine transform matrix for zooming and scaling that height and width are changed independently.", "response": "def affine_respective_zoom_matrix(w_range=0.8, h_range=1.1):\n \"\"\"Get affine transform matrix for zooming/scaling that height and width are changed independently.\n OpenCV format, x is width.\n\n Parameters\n -----------\n w_range : float or tuple of 2 floats\n The zooming/scaling ratio of width, greater than 1 means larger.\n - float, a fixed ratio.\n - tuple of 2 floats, randomly sample a value as the ratio between 2 values.\n h_range : float or tuple of 2 floats\n The zooming/scaling ratio of height, greater than 1 means larger.\n - float, a fixed ratio.\n - tuple of 2 floats, randomly sample a value as the ratio between 2 values.\n\n Returns\n -------\n numpy.array\n An affine transform matrix.\n\n \"\"\"\n\n if isinstance(h_range, (float, int)):\n zy = h_range\n elif isinstance(h_range, tuple):\n zy = np.random.uniform(h_range[0], h_range[1])\n else:\n raise Exception(\"h_range: float or tuple of 2 floats\")\n\n if isinstance(w_range, (float, int)):\n zx = w_range\n elif isinstance(w_range, tuple):\n zx = np.random.uniform(w_range[0], w_range[1])\n else:\n raise Exception(\"w_range: float or tuple of 2 floats\")\n\n zoom_matrix = np.array([[zx, 0, 0], \\\n [0, zy, 0], \\\n [0, 0, 1]])\n return zoom_matrix"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts the matrix from Cartesian coordinates to Image coordinates.", "response": "def transform_matrix_offset_center(matrix, y, x):\n \"\"\"Convert the matrix from Cartesian coordinates (the origin in the middle of image) to Image coordinates (the origin on the top-left of image).\n\n Parameters\n ----------\n matrix : numpy.array\n Transform matrix.\n x and y : 2 int\n Size of image.\n\n Returns\n -------\n numpy.array\n The transform matrix.\n\n Examples\n --------\n - See ``tl.prepro.rotation``, ``tl.prepro.shear``, ``tl.prepro.zoom``.\n \"\"\"\n o_x = (x - 1) / 2.0\n o_y = (y - 1) / 2.0\n offset_matrix = np.array([[1, 0, o_x], [0, 1, o_y], [0, 0, 1]])\n reset_matrix = np.array([[1, 0, -o_x], [0, 1, -o_y], [0, 0, 1]])\n transform_matrix = np.dot(np.dot(offset_matrix, matrix), reset_matrix)\n return transform_matrix"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef affine_transform(x, transform_matrix, channel_index=2, fill_mode='nearest', cval=0., order=1):\n # transform_matrix = transform_matrix_offset_center()\n # asdihasid\n # asd\n\n x = np.rollaxis(x, channel_index, 0)\n final_affine_matrix = transform_matrix[:2, :2]\n final_offset = transform_matrix[:2, 2]\n channel_images = [\n ndi.interpolation.\n affine_transform(x_channel, final_affine_matrix, final_offset, order=order, mode=fill_mode, cval=cval)\n for x_channel in x\n ]\n x = np.stack(channel_images, axis=0)\n x = np.rollaxis(x, 0, channel_index + 1)\n return x", "response": "Return transformed images by given an affine matrix in Scipy format."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns transformed images by given an affine matrix in OpenCV2 format.", "response": "def affine_transform_cv2(x, transform_matrix, flags=None, border_mode='constant'):\n \"\"\"Return transformed images by given an affine matrix in OpenCV format (x is width). (Powered by OpenCV2, faster than ``tl.prepro.affine_transform``)\n\n Parameters\n ----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n transform_matrix : numpy.array\n A transform matrix, OpenCV format.\n border_mode : str\n - `constant`, pad the image with a constant value (i.e. black or 0)\n - `replicate`, the row or column at the very edge of the original is replicated to the extra border.\n\n Examples\n --------\n >>> M_shear = tl.prepro.affine_shear_matrix(intensity=0.2, is_random=False)\n >>> M_zoom = tl.prepro.affine_zoom_matrix(zoom_range=0.8)\n >>> M_combined = M_shear.dot(M_zoom)\n >>> result = tl.prepro.affine_transform_cv2(image, M_combined)\n \"\"\"\n rows, cols = x.shape[0], x.shape[1]\n if flags is None:\n flags = cv2.INTER_AREA\n if border_mode is 'constant':\n border_mode = cv2.BORDER_CONSTANT\n elif border_mode is 'replicate':\n border_mode = cv2.BORDER_REPLICATE\n else:\n raise Exception(\"unsupport border_mode, check cv.BORDER_ for more details.\")\n return cv2.warpAffine(x, transform_matrix[0:2,:], \\\n (cols,rows), flags=flags, borderMode=border_mode)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef affine_transform_keypoints(coords_list, transform_matrix):\n coords_result_list = []\n for coords in coords_list:\n coords = np.asarray(coords)\n coords = coords.transpose([1, 0])\n coords = np.insert(coords, 2, 1, axis=0)\n # print(coords)\n # print(transform_matrix)\n coords_result = np.matmul(transform_matrix, coords)\n coords_result = coords_result[0:2, :].transpose([1, 0])\n coords_result_list.append(coords_result)\n return coords_result_list", "response": "Transform keypoint coordinates according to a given affine transform matrix."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef projective_transform_by_points(\n x, src, dst, map_args=None, output_shape=None, order=1, mode='constant', cval=0.0, clip=True,\n preserve_range=False\n):\n \"\"\"Projective transform by given coordinates, usually 4 coordinates.\n\n see `scikit-image `__.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n src : list or numpy\n The original coordinates, usually 4 coordinates of (width, height).\n dst : list or numpy\n The coordinates after transformation, the number of coordinates is the same with src.\n map_args : dictionary or None\n Keyword arguments passed to inverse map.\n output_shape : tuple of 2 int\n Shape of the output image generated. By default the shape of the input image is preserved. Note that, even for multi-band images, only rows and columns need to be specified.\n order : int\n The order of interpolation. The order has to be in the range 0-5:\n - 0 Nearest-neighbor\n - 1 Bi-linear (default)\n - 2 Bi-quadratic\n - 3 Bi-cubic\n - 4 Bi-quartic\n - 5 Bi-quintic\n mode : str\n One of `constant` (default), `edge`, `symmetric`, `reflect` or `wrap`.\n Points outside the boundaries of the input are filled according to the given mode. Modes match the behaviour of numpy.pad.\n cval : float\n Used in conjunction with mode `constant`, the value outside the image boundaries.\n clip : boolean\n Whether to clip the output to the range of values of the input image. This is enabled by default, since higher order interpolation may produce values outside the given input range.\n preserve_range : boolean\n Whether to keep the original range of values. Otherwise, the input image is converted according to the conventions of img_as_float.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n Examples\n --------\n Assume X is an image from CIFAR-10, i.e. shape == (32, 32, 3)\n\n >>> src = [[0,0],[0,32],[32,0],[32,32]] # [w, h]\n >>> dst = [[10,10],[0,32],[32,0],[32,32]]\n >>> x = tl.prepro.projective_transform_by_points(X, src, dst)\n\n References\n -----------\n - `scikit-image : geometric transformations `__\n - `scikit-image : examples `__\n\n \"\"\"\n if map_args is None:\n map_args = {}\n # if type(src) is list:\n if isinstance(src, list): # convert to numpy\n src = np.array(src)\n # if type(dst) is list:\n if isinstance(dst, list):\n dst = np.array(dst)\n if np.max(x) > 1: # convert to [0, 1]\n x = x / 255\n\n m = transform.ProjectiveTransform()\n m.estimate(dst, src)\n warped = transform.warp(\n x, m, map_args=map_args, output_shape=output_shape, order=order, mode=mode, cval=cval, clip=clip,\n preserve_range=preserve_range\n )\n return warped", "response": "Projective transform by given points."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef rotation(\n x, rg=20, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0., order=1\n):\n \"\"\"Rotate an image randomly or non-randomly.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n rg : int or float\n Degree to rotate, usually 0 ~ 180.\n is_random : boolean\n If True, randomly rotate. Default is False\n row_index col_index and channel_index : int\n Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).\n fill_mode : str\n Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see `scipy ndimage affine_transform `__\n cval : float\n Value used for points outside the boundaries of the input if mode=`constant`. Default is 0.0\n order : int\n The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform `__\n\n Returns\n -------\n numpy.array\n A processed image.\n\n Examples\n ---------\n >>> x --> [row, col, 1]\n >>> x = tl.prepro.rotation(x, rg=40, is_random=False)\n >>> tl.vis.save_image(x, 'im.png')\n\n \"\"\"\n if is_random:\n theta = np.pi / 180 * np.random.uniform(-rg, rg)\n else:\n theta = np.pi / 180 * rg\n rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0], [np.sin(theta), np.cos(theta), 0], [0, 0, 1]])\n\n h, w = x.shape[row_index], x.shape[col_index]\n transform_matrix = transform_matrix_offset_center(rotation_matrix, h, w)\n x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)\n return x", "response": "Rotate an image randomly or non - randomly."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef crop_multi(x, wrg, hrg, is_random=False, row_index=0, col_index=1):\n h, w = x[0].shape[row_index], x[0].shape[col_index]\n\n if (h < hrg) or (w < wrg):\n raise AssertionError(\"The size of cropping should smaller than or equal to the original image\")\n\n if is_random:\n h_offset = int(np.random.uniform(0, h - hrg))\n w_offset = int(np.random.uniform(0, w - wrg))\n results = []\n for data in x:\n results.append(data[h_offset:hrg + h_offset, w_offset:wrg + w_offset])\n return np.asarray(results)\n else:\n # central crop\n h_offset = (h - hrg) / 2\n w_offset = (w - wrg) / 2\n results = []\n for data in x:\n results.append(data[h_offset:h - h_offset, w_offset:w - w_offset])\n return np.asarray(results)", "response": "Randomly or centrally crop multiple images.\n\n Parameters\n ----------\n x : list of numpy.array\n List of images with dimension of [n_images, row, col, channel] (default).\n others : args\n See ``tl.prepro.crop``.\n\n Returns\n -------\n numpy.array\n A list of processed images."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nflipping the axis of an image in a random order.", "response": "def flip_axis(x, axis=1, is_random=False):\n \"\"\"Flip the axis of an image, such as flip left and right, up and down, randomly or non-randomly,\n\n Parameters\n ----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n axis : int\n Which axis to flip.\n - 0, flip up and down\n - 1, flip left and right\n - 2, flip channel\n is_random : boolean\n If True, randomly flip. Default is False.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n \"\"\"\n if is_random:\n factor = np.random.uniform(-1, 1)\n if factor > 0:\n x = np.asarray(x).swapaxes(axis, 0)\n x = x[::-1, ...]\n x = x.swapaxes(0, axis)\n return x\n else:\n return x\n else:\n x = np.asarray(x).swapaxes(axis, 0)\n x = x[::-1, ...]\n x = x.swapaxes(0, axis)\n return x"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nflip the axises of multiple images together.", "response": "def flip_axis_multi(x, axis, is_random=False):\n \"\"\"Flip the axises of multiple images together, such as flip left and right, up and down, randomly or non-randomly,\n\n Parameters\n -----------\n x : list of numpy.array\n List of images with dimension of [n_images, row, col, channel] (default).\n others : args\n See ``tl.prepro.flip_axis``.\n\n Returns\n -------\n numpy.array\n A list of processed images.\n\n \"\"\"\n if is_random:\n factor = np.random.uniform(-1, 1)\n if factor > 0:\n # x = np.asarray(x).swapaxes(axis, 0)\n # x = x[::-1, ...]\n # x = x.swapaxes(0, axis)\n # return x\n results = []\n for data in x:\n data = np.asarray(data).swapaxes(axis, 0)\n data = data[::-1, ...]\n data = data.swapaxes(0, axis)\n results.append(data)\n return np.asarray(results)\n else:\n return np.asarray(x)\n else:\n # x = np.asarray(x).swapaxes(axis, 0)\n # x = x[::-1, ...]\n # x = x.swapaxes(0, axis)\n # return x\n results = []\n for data in x:\n data = np.asarray(data).swapaxes(axis, 0)\n data = data[::-1, ...]\n data = data.swapaxes(0, axis)\n results.append(data)\n return np.asarray(results)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nshifts an image randomly or non - randomly.", "response": "def shift(\n x, wrg=0.1, hrg=0.1, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,\n order=1\n):\n \"\"\"Shift an image randomly or non-randomly.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n wrg : float\n Percentage of shift in axis x, usually -0.25 ~ 0.25.\n hrg : float\n Percentage of shift in axis y, usually -0.25 ~ 0.25.\n is_random : boolean\n If True, randomly shift. Default is False.\n row_index col_index and channel_index : int\n Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).\n fill_mode : str\n Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see `scipy ndimage affine_transform `__\n cval : float\n Value used for points outside the boundaries of the input if mode='constant'. Default is 0.0.\n order : int\n The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform `__\n\n Returns\n -------\n numpy.array\n A processed image.\n\n \"\"\"\n h, w = x.shape[row_index], x.shape[col_index]\n if is_random:\n tx = np.random.uniform(-hrg, hrg) * h\n ty = np.random.uniform(-wrg, wrg) * w\n else:\n tx, ty = hrg * h, wrg * w\n translation_matrix = np.array([[1, 0, tx], [0, 1, ty], [0, 0, 1]])\n\n transform_matrix = translation_matrix # no need to do offset\n x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)\n return x"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nshifts images with the same arguments randomly or non - randomly.", "response": "def shift_multi(\n x, wrg=0.1, hrg=0.1, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,\n order=1\n):\n \"\"\"Shift images with the same arguments, randomly or non-randomly.\n Usually be used for image segmentation which x=[X, Y], X and Y should be matched.\n\n Parameters\n -----------\n x : list of numpy.array\n List of images with dimension of [n_images, row, col, channel] (default).\n others : args\n See ``tl.prepro.shift``.\n\n Returns\n -------\n numpy.array\n A list of processed images.\n\n \"\"\"\n h, w = x[0].shape[row_index], x[0].shape[col_index]\n if is_random:\n tx = np.random.uniform(-hrg, hrg) * h\n ty = np.random.uniform(-wrg, wrg) * w\n else:\n tx, ty = hrg * h, wrg * w\n translation_matrix = np.array([[1, 0, tx], [0, 1, ty], [0, 0, 1]])\n\n transform_matrix = translation_matrix # no need to do offset\n results = []\n for data in x:\n results.append(affine_transform(data, transform_matrix, channel_index, fill_mode, cval, order))\n return np.asarray(results)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nshearing an image randomly or non - randomly.", "response": "def shear(\n x, intensity=0.1, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,\n order=1\n):\n \"\"\"Shear an image randomly or non-randomly.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n intensity : float\n Percentage of shear, usually -0.5 ~ 0.5 (is_random==True), 0 ~ 0.5 (is_random==False),\n you can have a quick try by shear(X, 1).\n is_random : boolean\n If True, randomly shear. Default is False.\n row_index col_index and channel_index : int\n Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).\n fill_mode : str\n Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see and `scipy ndimage affine_transform `__\n cval : float\n Value used for points outside the boundaries of the input if mode='constant'. Default is 0.0.\n order : int\n The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform `__\n\n Returns\n -------\n numpy.array\n A processed image.\n\n References\n -----------\n - `Affine transformation `__\n\n \"\"\"\n if is_random:\n shear = np.random.uniform(-intensity, intensity)\n else:\n shear = intensity\n shear_matrix = np.array([[1, -np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])\n\n h, w = x.shape[row_index], x.shape[col_index]\n transform_matrix = transform_matrix_offset_center(shear_matrix, h, w)\n x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)\n return x"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nshear an image randomly or non - randomly.", "response": "def shear2(\n x, shear=(0.1, 0.1), is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,\n order=1\n):\n \"\"\"Shear an image randomly or non-randomly.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n shear : tuple of two floats\n Percentage of shear for height and width direction (0, 1).\n is_random : boolean\n If True, randomly shear. Default is False.\n row_index col_index and channel_index : int\n Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).\n fill_mode : str\n Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see `scipy ndimage affine_transform `__\n cval : float\n Value used for points outside the boundaries of the input if mode='constant'. Default is 0.0.\n order : int\n The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform `__\n\n Returns\n -------\n numpy.array\n A processed image.\n\n References\n -----------\n - `Affine transformation `__\n\n \"\"\"\n if len(shear) != 2:\n raise AssertionError(\n \"shear should be tuple of 2 floats, or you want to use tl.prepro.shear rather than tl.prepro.shear2 ?\"\n )\n if isinstance(shear, tuple):\n shear = list(shear)\n if is_random:\n shear[0] = np.random.uniform(-shear[0], shear[0])\n shear[1] = np.random.uniform(-shear[1], shear[1])\n\n shear_matrix = np.array([[1, shear[0], 0], \\\n [shear[1], 1, 0], \\\n [0, 0, 1]])\n\n h, w = x.shape[row_index], x.shape[col_index]\n transform_matrix = transform_matrix_offset_center(shear_matrix, h, w)\n x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)\n return x"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating a swirl image.", "response": "def swirl(\n x, center=None, strength=1, radius=100, rotation=0, output_shape=None, order=1, mode='constant', cval=0,\n clip=True, preserve_range=False, is_random=False\n):\n \"\"\"Swirl an image randomly or non-randomly, see `scikit-image swirl API `__\n and `example `__.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n center : tuple or 2 int or None\n Center coordinate of transformation (optional).\n strength : float\n The amount of swirling applied.\n radius : float\n The extent of the swirl in pixels. The effect dies out rapidly beyond radius.\n rotation : float\n Additional rotation applied to the image, usually [0, 360], relates to center.\n output_shape : tuple of 2 int or None\n Shape of the output image generated (height, width). By default the shape of the input image is preserved.\n order : int, optional\n The order of the spline interpolation, default is 1. The order has to be in the range 0-5. See skimage.transform.warp for detail.\n mode : str\n One of `constant` (default), `edge`, `symmetric` `reflect` and `wrap`.\n Points outside the boundaries of the input are filled according to the given mode, with `constant` used as the default. Modes match the behaviour of numpy.pad.\n cval : float\n Used in conjunction with mode `constant`, the value outside the image boundaries.\n clip : boolean\n Whether to clip the output to the range of values of the input image. This is enabled by default, since higher order interpolation may produce values outside the given input range.\n preserve_range : boolean\n Whether to keep the original range of values. Otherwise, the input image is converted according to the conventions of img_as_float.\n is_random : boolean,\n If True, random swirl. Default is False.\n - random center = [(0 ~ x.shape[0]), (0 ~ x.shape[1])]\n - random strength = [0, strength]\n - random radius = [1e-10, radius]\n - random rotation = [-rotation, rotation]\n\n Returns\n -------\n numpy.array\n A processed image.\n\n Examples\n ---------\n >>> x --> [row, col, 1] greyscale\n >>> x = tl.prepro.swirl(x, strength=4, radius=100)\n\n \"\"\"\n if radius == 0:\n raise AssertionError(\"Invalid radius value\")\n\n rotation = np.pi / 180 * rotation\n if is_random:\n center_h = int(np.random.uniform(0, x.shape[0]))\n center_w = int(np.random.uniform(0, x.shape[1]))\n center = (center_h, center_w)\n strength = np.random.uniform(0, strength)\n radius = np.random.uniform(1e-10, radius)\n rotation = np.random.uniform(-rotation, rotation)\n\n max_v = np.max(x)\n if max_v > 1: # Note: the input of this fn should be [-1, 1], rescale is required.\n x = x / max_v\n swirled = skimage.transform.swirl(\n x, center=center, strength=strength, radius=radius, rotation=rotation, output_shape=output_shape, order=order,\n mode=mode, cval=cval, clip=clip, preserve_range=preserve_range\n )\n if max_v > 1:\n swirled = swirled * max_v\n return swirled"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef elastic_transform(x, alpha, sigma, mode=\"constant\", cval=0, is_random=False):\n if is_random is False:\n random_state = np.random.RandomState(None)\n else:\n random_state = np.random.RandomState(int(time.time()))\n #\n is_3d = False\n if len(x.shape) == 3 and x.shape[-1] == 1:\n x = x[:, :, 0]\n is_3d = True\n elif len(x.shape) == 3 and x.shape[-1] != 1:\n raise Exception(\"Only support greyscale image\")\n\n if len(x.shape) != 2:\n raise AssertionError(\"input should be grey-scale image\")\n\n shape = x.shape\n\n dx = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode=mode, cval=cval) * alpha\n dy = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode=mode, cval=cval) * alpha\n\n x_, y_ = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]), indexing='ij')\n indices = np.reshape(x_ + dx, (-1, 1)), np.reshape(y_ + dy, (-1, 1))\n if is_3d:\n return map_coordinates(x, indices, order=1).reshape((shape[0], shape[1], 1))\n else:\n return map_coordinates(x, indices, order=1).reshape(shape)", "response": "Elastic transformation for image as described in [ Simard2003 ]."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef zoom(x, zoom_range=(0.9, 1.1), flags=None, border_mode='constant'):\n zoom_matrix = affine_zoom_matrix(zoom_range=zoom_range)\n h, w = x.shape[0], x.shape[1]\n transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)\n x = affine_transform_cv2(x, transform_matrix, flags=flags, border_mode=border_mode)\n return x", "response": "Zooms a single image that height and width are changed together."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef respective_zoom(x, h_range=(0.9, 1.1), w_range=(0.9, 1.1), flags=None, border_mode='constant'):\n zoom_matrix = affine_respective_zoom_matrix(h_range=h_range, w_range=w_range)\n h, w = x.shape[0], x.shape[1]\n transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)\n x = affine_transform_cv2(\n x, transform_matrix, flags=flags, border_mode=border_mode\n ) #affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)\n return x", "response": "Respects the height and width of a single image."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef zoom_multi(x, zoom_range=(0.9, 1.1), flags=None, border_mode='constant'):\n\n zoom_matrix = affine_zoom_matrix(zoom_range=zoom_range)\n results = []\n for img in x:\n h, w = x.shape[0], x.shape[1]\n transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)\n results.append(affine_transform_cv2(x, transform_matrix, flags=flags, border_mode=border_mode))\n return result", "response": "Zooms in and out of images with the same arguments randomly or non - randomly."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef brightness(x, gamma=1, gain=1, is_random=False):\n if is_random:\n gamma = np.random.uniform(1 - gamma, 1 + gamma)\n x = exposure.adjust_gamma(x, gamma, gain)\n return x", "response": "Change the brightness of a single image randomly or non - randomly."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef brightness_multi(x, gamma=1, gain=1, is_random=False):\n if is_random:\n gamma = np.random.uniform(1 - gamma, 1 + gamma)\n\n results = []\n for data in x:\n results.append(exposure.adjust_gamma(data, gamma, gain))\n return np.asarray(results)", "response": "Change the brightness of multiply images randomly or non - randomly."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms illumination augmentation for a single image.", "response": "def illumination(x, gamma=1., contrast=1., saturation=1., is_random=False):\n \"\"\"Perform illumination augmentation for a single image, randomly or non-randomly.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n gamma : float\n Change brightness (the same with ``tl.prepro.brightness``)\n - if is_random=False, one float number, small than one means brighter, greater than one means darker.\n - if is_random=True, tuple of two float numbers, (min, max).\n contrast : float\n Change contrast.\n - if is_random=False, one float number, small than one means blur.\n - if is_random=True, tuple of two float numbers, (min, max).\n saturation : float\n Change saturation.\n - if is_random=False, one float number, small than one means unsaturation.\n - if is_random=True, tuple of two float numbers, (min, max).\n is_random : boolean\n If True, randomly change illumination. Default is False.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n Examples\n ---------\n Random\n\n >>> x = tl.prepro.illumination(x, gamma=(0.5, 5.0), contrast=(0.3, 1.0), saturation=(0.7, 1.0), is_random=True)\n\n Non-random\n\n >>> x = tl.prepro.illumination(x, 0.5, 0.6, 0.8, is_random=False)\n\n \"\"\"\n if is_random:\n if not (len(gamma) == len(contrast) == len(saturation) == 2):\n raise AssertionError(\"if is_random = True, the arguments are (min, max)\")\n\n ## random change brightness # small --> brighter\n illum_settings = np.random.randint(0, 3) # 0-brighter, 1-darker, 2 keep normal\n\n if illum_settings == 0: # brighter\n gamma = np.random.uniform(gamma[0], 1.0) # (.5, 1.0)\n elif illum_settings == 1: # darker\n gamma = np.random.uniform(1.0, gamma[1]) # (1.0, 5.0)\n else:\n gamma = 1\n im_ = brightness(x, gamma=gamma, gain=1, is_random=False)\n\n # tl.logging.info(\"using contrast and saturation\")\n image = PIL.Image.fromarray(im_) # array -> PIL\n contrast_adjust = PIL.ImageEnhance.Contrast(image)\n image = contrast_adjust.enhance(np.random.uniform(contrast[0], contrast[1])) #0.3,0.9))\n\n saturation_adjust = PIL.ImageEnhance.Color(image)\n image = saturation_adjust.enhance(np.random.uniform(saturation[0], saturation[1])) # (0.7,1.0))\n im_ = np.array(image) # PIL -> array\n else:\n im_ = brightness(x, gamma=gamma, gain=1, is_random=False)\n image = PIL.Image.fromarray(im_) # array -> PIL\n contrast_adjust = PIL.ImageEnhance.Contrast(image)\n image = contrast_adjust.enhance(contrast)\n\n saturation_adjust = PIL.ImageEnhance.Color(image)\n image = saturation_adjust.enhance(saturation)\n im_ = np.array(image) # PIL -> array\n return np.asarray(im_)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting RGB image to HSV image.", "response": "def rgb_to_hsv(rgb):\n \"\"\"Input RGB image [0~255] return HSV image [0~1].\n\n Parameters\n ------------\n rgb : numpy.array\n An image with values between 0 and 255.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n \"\"\"\n # Translated from source of colorsys.rgb_to_hsv\n # r,g,b should be a numpy arrays with values between 0 and 255\n # rgb_to_hsv returns an array of floats between 0.0 and 1.0.\n rgb = rgb.astype('float')\n hsv = np.zeros_like(rgb)\n # in case an RGBA array was passed, just copy the A channel\n hsv[..., 3:] = rgb[..., 3:]\n r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2]\n maxc = np.max(rgb[..., :3], axis=-1)\n minc = np.min(rgb[..., :3], axis=-1)\n hsv[..., 2] = maxc\n mask = maxc != minc\n hsv[mask, 1] = (maxc - minc)[mask] / maxc[mask]\n rc = np.zeros_like(r)\n gc = np.zeros_like(g)\n bc = np.zeros_like(b)\n rc[mask] = (maxc - r)[mask] / (maxc - minc)[mask]\n gc[mask] = (maxc - g)[mask] / (maxc - minc)[mask]\n bc[mask] = (maxc - b)[mask] / (maxc - minc)[mask]\n hsv[..., 0] = np.select([r == maxc, g == maxc], [bc - gc, 2.0 + rc - bc], default=4.0 + gc - rc)\n hsv[..., 0] = (hsv[..., 0] / 6.0) % 1.0\n return hsv"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef hsv_to_rgb(hsv):\n # Translated from source of colorsys.hsv_to_rgb\n # h,s should be a numpy arrays with values between 0.0 and 1.0\n # v should be a numpy array with values between 0.0 and 255.0\n # hsv_to_rgb returns an array of uints between 0 and 255.\n rgb = np.empty_like(hsv)\n rgb[..., 3:] = hsv[..., 3:]\n h, s, v = hsv[..., 0], hsv[..., 1], hsv[..., 2]\n i = (h * 6.0).astype('uint8')\n f = (h * 6.0) - i\n p = v * (1.0 - s)\n q = v * (1.0 - s * f)\n t = v * (1.0 - s * (1.0 - f))\n i = i % 6\n conditions = [s == 0.0, i == 1, i == 2, i == 3, i == 4, i == 5]\n rgb[..., 0] = np.select(conditions, [v, q, p, p, t, v], default=v)\n rgb[..., 1] = np.select(conditions, [v, v, v, q, p, p], default=t)\n rgb[..., 2] = np.select(conditions, [v, p, t, v, v, q], default=p)\n return rgb.astype('uint8')", "response": "This function converts an HSV image to RGB image."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadjusting the hue of an RGB image.", "response": "def adjust_hue(im, hout=0.66, is_offset=True, is_clip=True, is_random=False):\n \"\"\"Adjust hue of an RGB image.\n\n This is a convenience method that converts an RGB image to float representation, converts it to HSV, add an offset to the hue channel, converts back to RGB and then back to the original data type.\n For TF, see `tf.image.adjust_hue `__.and `tf.image.random_hue `__.\n\n Parameters\n -----------\n im : numpy.array\n An image with values between 0 and 255.\n hout : float\n The scale value for adjusting hue.\n - If is_offset is False, set all hue values to this value. 0 is red; 0.33 is green; 0.66 is blue.\n - If is_offset is True, add this value as the offset to the hue channel.\n is_offset : boolean\n Whether `hout` is added on HSV as offset or not. Default is True.\n is_clip : boolean\n If HSV value smaller than 0, set to 0. Default is True.\n is_random : boolean\n If True, randomly change hue. Default is False.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n Examples\n ---------\n Random, add a random value between -0.2 and 0.2 as the offset to every hue values.\n\n >>> im_hue = tl.prepro.adjust_hue(image, hout=0.2, is_offset=True, is_random=False)\n\n Non-random, make all hue to green.\n\n >>> im_green = tl.prepro.adjust_hue(image, hout=0.66, is_offset=False, is_random=False)\n\n References\n -----------\n - `tf.image.random_hue `__.\n - `tf.image.adjust_hue `__.\n - `StackOverflow: Changing image hue with python PIL `__.\n\n \"\"\"\n hsv = rgb_to_hsv(im)\n if is_random:\n hout = np.random.uniform(-hout, hout)\n\n if is_offset:\n hsv[..., 0] += hout\n else:\n hsv[..., 0] = hout\n\n if is_clip:\n hsv[..., 0] = np.clip(hsv[..., 0], 0, np.inf) # Hao : can remove green dots\n\n rgb = hsv_to_rgb(hsv)\n return rgb"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef imresize(x, size=None, interp='bicubic', mode=None):\n if size is None:\n size = [100, 100]\n\n if x.shape[-1] == 1:\n # greyscale\n x = scipy.misc.imresize(x[:, :, 0], size, interp=interp, mode=mode)\n return x[:, :, np.newaxis]\n else:\n # rgb, bgr, rgba\n return scipy.misc.imresize(x, size, interp=interp, mode=mode)", "response": "Resizes an image by given output size and interpolation method."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pixel_value_scale(im, val=0.9, clip=None, is_random=False):\n\n clip = clip if clip is not None else (-np.inf, np.inf)\n\n if is_random:\n scale = 1 + np.random.uniform(-val, val)\n im = im * scale\n else:\n im = im * val\n\n if len(clip) == 2:\n im = np.clip(im, clip[0], clip[1])\n else:\n raise Exception(\"clip : tuple of 2 numbers\")\n\n return im", "response": "Scales each value in the pixels of the image."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef samplewise_norm(\n x, rescale=None, samplewise_center=False, samplewise_std_normalization=False, channel_index=2, epsilon=1e-7\n):\n \"\"\"Normalize an image by rescale, samplewise centering and samplewise centering in order.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n rescale : float\n Rescaling factor. If None or 0, no rescaling is applied, otherwise we multiply the data by the value provided (before applying any other transformation)\n samplewise_center : boolean\n If True, set each sample mean to 0.\n samplewise_std_normalization : boolean\n If True, divide each input by its std.\n epsilon : float\n A small position value for dividing standard deviation.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n Examples\n --------\n >>> x = samplewise_norm(x, samplewise_center=True, samplewise_std_normalization=True)\n >>> print(x.shape, np.mean(x), np.std(x))\n (160, 176, 1), 0.0, 1.0\n\n Notes\n ------\n When samplewise_center and samplewise_std_normalization are True.\n - For greyscale image, every pixels are subtracted and divided by the mean and std of whole image.\n - For RGB image, every pixels are subtracted and divided by the mean and std of this pixel i.e. the mean and std of a pixel is 0 and 1.\n\n \"\"\"\n if rescale:\n x *= rescale\n\n if x.shape[channel_index] == 1:\n # greyscale\n if samplewise_center:\n x = x - np.mean(x)\n if samplewise_std_normalization:\n x = x / np.std(x)\n return x\n elif x.shape[channel_index] == 3:\n # rgb\n if samplewise_center:\n x = x - np.mean(x, axis=channel_index, keepdims=True)\n if samplewise_std_normalization:\n x = x / (np.std(x, axis=channel_index, keepdims=True) + epsilon)\n return x\n else:\n raise Exception(\"Unsupported channels %d\" % x.shape[channel_index])", "response": "This function normalizes an image by sampling the image mean and standard deviation of each input image."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef featurewise_norm(x, mean=None, std=None, epsilon=1e-7):\n if mean:\n x = x - mean\n if std:\n x = x / (std + epsilon)\n return x", "response": "Normalize every pixel by the same given mean and std which are usually\n compute from all examples."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the ZCA whitening principal components matrix.", "response": "def get_zca_whitening_principal_components_img(X):\n \"\"\"Return the ZCA whitening principal components matrix.\n\n Parameters\n -----------\n x : numpy.array\n Batch of images with dimension of [n_example, row, col, channel] (default).\n\n Returns\n -------\n numpy.array\n A processed image.\n\n \"\"\"\n flatX = np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2] * X.shape[3]))\n tl.logging.info(\"zca : computing sigma ..\")\n sigma = np.dot(flatX.T, flatX) / flatX.shape[0]\n tl.logging.info(\"zca : computing U, S and V ..\")\n U, S, _ = linalg.svd(sigma) # USV\n tl.logging.info(\"zca : computing principal components ..\")\n principal_components = np.dot(np.dot(U, np.diag(1. / np.sqrt(S + 10e-7))), U.T)\n return principal_components"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\napplying ZCA whitening on an image by given principal components matrix.", "response": "def zca_whitening(x, principal_components):\n \"\"\"Apply ZCA whitening on an image by given principal components matrix.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n principal_components : matrix\n Matrix from ``get_zca_whitening_principal_components_img``.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n \"\"\"\n flatx = np.reshape(x, (x.size))\n # tl.logging.info(principal_components.shape, x.shape) # ((28160, 28160), (160, 176, 1))\n # flatx = np.reshape(x, (x.shape))\n # flatx = np.reshape(x, (x.shape[0], ))\n # tl.logging.info(flatx.shape) # (160, 176, 1)\n whitex = np.dot(flatx, principal_components)\n x = np.reshape(whitex, (x.shape[0], x.shape[1], x.shape[2]))\n return x"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nshifts the channels of an image randomly or non - randomly.", "response": "def channel_shift(x, intensity, is_random=False, channel_index=2):\n \"\"\"Shift the channels of an image, randomly or non-randomly, see `numpy.rollaxis `__.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] (default).\n intensity : float\n Intensity of shifting.\n is_random : boolean\n If True, randomly shift. Default is False.\n channel_index : int\n Index of channel. Default is 2.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n \"\"\"\n if is_random:\n factor = np.random.uniform(-intensity, intensity)\n else:\n factor = intensity\n x = np.rollaxis(x, channel_index, 0)\n min_x, max_x = np.min(x), np.max(x)\n channel_images = [np.clip(x_channel + factor, min_x, max_x) for x_channel in x]\n x = np.stack(channel_images, axis=0)\n x = np.rollaxis(x, 0, channel_index + 1)\n return x"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef channel_shift_multi(x, intensity, is_random=False, channel_index=2):\n if is_random:\n factor = np.random.uniform(-intensity, intensity)\n else:\n factor = intensity\n\n results = []\n for data in x:\n data = np.rollaxis(data, channel_index, 0)\n min_x, max_x = np.min(data), np.max(data)\n channel_images = [np.clip(x_channel + factor, min_x, max_x) for x_channel in x]\n data = np.stack(channel_images, axis=0)\n data = np.rollaxis(x, 0, channel_index + 1)\n results.append(data)\n return np.asarray(results)", "response": "Shift the channels of images with the same arguments randomly or non - randomly."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndrop some pixels from the current image.", "response": "def drop(x, keep=0.5):\n \"\"\"Randomly set some pixels to zero by a given keeping probability.\n\n Parameters\n -----------\n x : numpy.array\n An image with dimension of [row, col, channel] or [row, col].\n keep : float\n The keeping probability (0, 1), the lower more values will be set to zero.\n\n Returns\n -------\n numpy.array\n A processed image.\n\n \"\"\"\n if len(x.shape) == 3:\n if x.shape[-1] == 3: # color\n img_size = x.shape\n mask = np.random.binomial(n=1, p=keep, size=x.shape[:-1])\n for i in range(3):\n x[:, :, i] = np.multiply(x[:, :, i], mask)\n elif x.shape[-1] == 1: # greyscale image\n img_size = x.shape\n x = np.multiply(x, np.random.binomial(n=1, p=keep, size=img_size))\n else:\n raise Exception(\"Unsupported shape {}\".format(x.shape))\n elif len(x.shape) == 2 or 1: # greyscale matrix (image) or vector\n img_size = x.shape\n x = np.multiply(x, np.random.binomial(n=1, p=keep, size=img_size))\n else:\n raise Exception(\"Unsupported shape {}\".format(x.shape))\n return x"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts a numpy array to PIL image object.", "response": "def array_to_img(x, dim_ordering=(0, 1, 2), scale=True):\n \"\"\"Converts a numpy array to PIL image object (uint8 format).\n\n Parameters\n ----------\n x : numpy.array\n An image with dimension of 3 and channels of 1 or 3.\n dim_ordering : tuple of 3 int\n Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).\n scale : boolean\n If True, converts image to [0, 255] from any range of value like [-1, 2]. Default is True.\n\n Returns\n -------\n PIL.image\n An image.\n\n References\n -----------\n `PIL Image.fromarray `__\n\n \"\"\"\n # if dim_ordering == 'default':\n # dim_ordering = K.image_dim_ordering()\n # if dim_ordering == 'th': # theano\n # x = x.transpose(1, 2, 0)\n\n x = x.transpose(dim_ordering)\n\n if scale:\n x += max(-np.min(x), 0)\n x_max = np.max(x)\n if x_max != 0:\n # tl.logging.info(x_max)\n # x /= x_max\n x = x / x_max\n x *= 255\n\n if x.shape[2] == 3:\n # RGB\n return PIL.Image.fromarray(x.astype('uint8'), 'RGB')\n\n elif x.shape[2] == 1:\n # grayscale\n return PIL.Image.fromarray(x[:, :, 0].astype('uint8'), 'L')\n\n else:\n raise Exception('Unsupported channel number: ', x.shape[2])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef find_contours(x, level=0.8, fully_connected='low', positive_orientation='low'):\n return skimage.measure.find_contours(\n x, level, fully_connected=fully_connected, positive_orientation=positive_orientation\n )", "response": "Find iso - valued contours in a 2D array for a given level value."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a 2D image containing the contour values for a list of points.", "response": "def pt2map(list_points=None, size=(100, 100), val=1):\n \"\"\"Inputs a list of points, return a 2D image.\n\n Parameters\n --------------\n list_points : list of 2 int\n [[x, y], [x, y]..] for point coordinates.\n size : tuple of 2 int\n (w, h) for output size.\n val : float or int\n For the contour value.\n\n Returns\n -------\n numpy.array\n An image.\n\n \"\"\"\n if list_points is None:\n raise Exception(\"list_points : list of 2 int\")\n i_m = np.zeros(size)\n if len(list_points) == 0:\n return i_m\n for xx in list_points:\n for x in xx:\n # tl.logging.info(x)\n i_m[int(np.round(x[0]))][int(np.round(x[1]))] = val\n return i_m"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef binary_dilation(x, radius=3):\n mask = disk(radius)\n x = _binary_dilation(x, selem=mask)\n\n return x", "response": "Return a fast binary morphological dilation of an image."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dilation(x, radius=3):\n mask = disk(radius)\n x = dilation(x, selem=mask)\n\n return x", "response": "Return a greyscale morphological dilation of an image."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef binary_erosion(x, radius=3):\n mask = disk(radius)\n x = _binary_erosion(x, selem=mask)\n return x", "response": "Binary EoSion of an image."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a greyscale morphological erosion of an image.", "response": "def erosion(x, radius=3):\n \"\"\"Return greyscale morphological erosion of an image,\n see `skimage.morphology.erosion `__.\n\n Parameters\n -----------\n x : 2D array\n A greyscale image.\n radius : int\n For the radius of mask.\n\n Returns\n -------\n numpy.array\n A processed greyscale image.\n\n \"\"\"\n mask = disk(radius)\n x = _erosion(x, selem=mask)\n return x"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef obj_box_coords_rescale(coords=None, shape=None):\n if coords is None:\n coords = []\n if shape is None:\n shape = [100, 200]\n\n imh, imw = shape[0], shape[1]\n imh = imh * 1.0 # * 1.0 for python2 : force division to be float point\n imw = imw * 1.0\n coords_new = list()\n for coord in coords:\n\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n x = coord[0] / imw\n y = coord[1] / imh\n w = coord[2] / imw\n h = coord[3] / imh\n coords_new.append([x, y, w, h])\n return coords_new", "response": "Scale down a list of coordinates from pixel unit to ratio of image size."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nscale down one coordinates from pixel unit to the ratio of image size i.e. in the range of [0, 1]. It is the reverse process of ``obj_box_coord_scale_to_pixelunit``. Parameters ------------ coords : list of 4 int or None One coordinates of one image e.g. [x, y, w, h]. shape : list of 2 int or None For [height, width]. Returns ------- list of 4 numbers New bounding box. Examples --------- >>> coord = tl.prepro.obj_box_coord_rescale(coord=[30, 40, 50, 50], shape=[100, 100]) [0.3, 0.4, 0.5, 0.5]", "response": "def obj_box_coord_rescale(coord=None, shape=None):\n \"\"\"Scale down one coordinates from pixel unit to the ratio of image size i.e. in the range of [0, 1].\n It is the reverse process of ``obj_box_coord_scale_to_pixelunit``.\n\n Parameters\n ------------\n coords : list of 4 int or None\n One coordinates of one image e.g. [x, y, w, h].\n shape : list of 2 int or None\n For [height, width].\n\n Returns\n -------\n list of 4 numbers\n New bounding box.\n\n Examples\n ---------\n >>> coord = tl.prepro.obj_box_coord_rescale(coord=[30, 40, 50, 50], shape=[100, 100])\n [0.3, 0.4, 0.5, 0.5]\n\n \"\"\"\n if coord is None:\n coord = []\n if shape is None:\n shape = [100, 200]\n\n return obj_box_coords_rescale(coords=[coord], shape=shape)[0]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert one coordinate in ratio format to image coordinate format.", "response": "def obj_box_coord_scale_to_pixelunit(coord, shape=None):\n \"\"\"Convert one coordinate [x, y, w (or x2), h (or y2)] in ratio format to image coordinate format.\n It is the reverse process of ``obj_box_coord_rescale``.\n\n Parameters\n -----------\n coord : list of 4 float\n One coordinate of one image [x, y, w (or x2), h (or y2)] in ratio format, i.e value range [0~1].\n shape : tuple of 2 or None\n For [height, width].\n\n Returns\n -------\n list of 4 numbers\n New bounding box.\n\n Examples\n ---------\n >>> x, y, x2, y2 = tl.prepro.obj_box_coord_scale_to_pixelunit([0.2, 0.3, 0.5, 0.7], shape=(100, 200, 3))\n [40, 30, 100, 70]\n\n \"\"\"\n if shape is None:\n shape = [100, 100]\n\n imh, imw = shape[0:2]\n x = int(coord[0] * imw)\n x2 = int(coord[2] * imw)\n y = int(coord[1] * imh)\n y2 = int(coord[3] * imh)\n return [x, y, x2, y2]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef obj_box_coord_centroid_to_upleft_butright(coord, to_int=False):\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n x_center, y_center, w, h = coord\n x = x_center - w / 2.\n y = y_center - h / 2.\n x2 = x + w\n y2 = y + h\n if to_int:\n return [int(x), int(y), int(x2), int(y2)]\n else:\n return [x, y, x2, y2]", "response": "Convert one coordinate [ x y w h ] to [ x1 y1 x2 y2 w h ] in up - left and botton - right format."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef obj_box_coord_upleft_butright_to_centroid(coord):\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x1, y1, x2, y2]\")\n x1, y1, x2, y2 = coord\n w = x2 - x1\n h = y2 - y1\n x_c = x1 + w / 2.\n y_c = y1 + h / 2.\n return [x_c, y_c, w, h]", "response": "Convert one coordinate [ x1 y1 x2 y2 w h to centroid"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert one coordinate [ x y w h to a new bounding box.", "response": "def obj_box_coord_centroid_to_upleft(coord):\n \"\"\"Convert one coordinate [x_center, y_center, w, h] to [x, y, w, h].\n It is the reverse process of ``obj_box_coord_upleft_to_centroid``.\n\n Parameters\n ------------\n coord : list of 4 int/float\n One coordinate.\n\n Returns\n -------\n list of 4 numbers\n New bounding box.\n\n \"\"\"\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n x_center, y_center, w, h = coord\n x = x_center - w / 2.\n y = y_center - h / 2.\n return [x, y, w, h]"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_darknet_ann_str_to_list(annotations):\n annotations = annotations.split(\"\\n\")\n ann = []\n for a in annotations:\n a = a.split()\n if len(a) == 5:\n for i, _v in enumerate(a):\n if i == 0:\n a[i] = int(a[i])\n else:\n a[i] = float(a[i])\n ann.append(a)\n return ann", "response": "Parse darknet annotation string into list of 4 numbers."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse darknet annotation format into two lists for class and bounding box.", "response": "def parse_darknet_ann_list_to_cls_box(annotations):\n \"\"\"Parse darknet annotation format into two lists for class and bounding box.\n\n Input list of [[class, x, y, w, h], ...], return two list of [class ...] and [[x, y, w, h], ...].\n\n Parameters\n ------------\n annotations : list of list\n A list of class and bounding boxes of images e.g. [[class, x, y, w, h], ...]\n\n Returns\n -------\n list of int\n List of class labels.\n\n list of list of 4 numbers\n List of bounding box.\n\n \"\"\"\n class_list = []\n bbox_list = []\n for ann in annotations:\n class_list.append(ann[0])\n bbox_list.append(ann[1:])\n return class_list, bbox_list"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nleaving - right flip the image and coordinates for object detection.", "response": "def obj_box_horizontal_flip(im, coords=None, is_rescale=False, is_center=False, is_random=False):\n \"\"\"Left-right flip the image and coordinates for object detection.\n\n Parameters\n ----------\n im : numpy.array\n An image with dimension of [row, col, channel] (default).\n coords : list of list of 4 int/float or None\n Coordinates [[x, y, w, h], [x, y, w, h], ...].\n is_rescale : boolean\n Set to True, if the input coordinates are rescaled to [0, 1]. Default is False.\n is_center : boolean\n Set to True, if the x and y of coordinates are the centroid (i.e. darknet format). Default is False.\n is_random : boolean\n If True, randomly flip. Default is False.\n\n Returns\n -------\n numpy.array\n A processed image\n list of list of 4 numbers\n A list of new bounding boxes.\n\n Examples\n --------\n >>> im = np.zeros([80, 100]) # as an image with shape width=100, height=80\n >>> im, coords = obj_box_left_right_flip(im, coords=[[0.2, 0.4, 0.3, 0.3], [0.1, 0.5, 0.2, 0.3]], is_rescale=True, is_center=True, is_random=False)\n >>> print(coords)\n [[0.8, 0.4, 0.3, 0.3], [0.9, 0.5, 0.2, 0.3]]\n >>> im, coords = obj_box_left_right_flip(im, coords=[[0.2, 0.4, 0.3, 0.3]], is_rescale=True, is_center=False, is_random=False)\n >>> print(coords)\n [[0.5, 0.4, 0.3, 0.3]]\n >>> im, coords = obj_box_left_right_flip(im, coords=[[20, 40, 30, 30]], is_rescale=False, is_center=True, is_random=False)\n >>> print(coords)\n [[80, 40, 30, 30]]\n >>> im, coords = obj_box_left_right_flip(im, coords=[[20, 40, 30, 30]], is_rescale=False, is_center=False, is_random=False)\n >>> print(coords)\n [[50, 40, 30, 30]]\n\n \"\"\"\n if coords is None:\n coords = []\n\n def _flip(im, coords):\n im = flip_axis(im, axis=1, is_random=False)\n coords_new = list()\n\n for coord in coords:\n\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n if is_rescale:\n if is_center:\n # x_center' = 1 - x\n x = 1. - coord[0]\n else:\n # x_center' = 1 - x - w\n x = 1. - coord[0] - coord[2]\n else:\n if is_center:\n # x' = im.width - x\n x = im.shape[1] - coord[0]\n else:\n # x' = im.width - x - w\n x = im.shape[1] - coord[0] - coord[2]\n coords_new.append([x, coord[1], coord[2], coord[3]])\n return im, coords_new\n\n if is_random:\n factor = np.random.uniform(-1, 1)\n if factor > 0:\n return _flip(im, coords)\n else:\n return im, coords\n else:\n return _flip(im, coords)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef obj_box_imresize(im, coords=None, size=None, interp='bicubic', mode=None, is_rescale=False):\n if coords is None:\n coords = []\n if size is None:\n size = [100, 100]\n\n imh, imw = im.shape[0:2]\n imh = imh * 1.0 # * 1.0 for python2 : force division to be float point\n imw = imw * 1.0\n im = imresize(im, size=size, interp=interp, mode=mode)\n\n if is_rescale is False:\n coords_new = list()\n\n for coord in coords:\n\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n # x' = x * (imw'/imw)\n x = int(coord[0] * (size[1] / imw))\n # y' = y * (imh'/imh)\n # tl.logging.info('>>', coord[1], size[0], imh)\n y = int(coord[1] * (size[0] / imh))\n # w' = w * (imw'/imw)\n w = int(coord[2] * (size[1] / imw))\n # h' = h * (imh'/imh)\n h = int(coord[3] * (size[0] / imh))\n coords_new.append([x, y, w, h])\n return im, coords_new\n else:\n return im, coords", "response": "Resize an image and compute the new bounding box coordinates."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef obj_box_crop(\n im, classes=None, coords=None, wrg=100, hrg=100, is_rescale=False, is_center=False, is_random=False,\n thresh_wh=0.02, thresh_wh2=12.\n):\n \"\"\"Randomly or centrally crop an image, and compute the new bounding box coordinates.\n Objects outside the cropped image will be removed.\n\n Parameters\n -----------\n im : numpy.array\n An image with dimension of [row, col, channel] (default).\n classes : list of int or None\n Class IDs.\n coords : list of list of 4 int/float or None\n Coordinates [[x, y, w, h], [x, y, w, h], ...]\n wrg hrg and is_random : args\n See ``tl.prepro.crop``.\n is_rescale : boolean\n Set to True, if the input coordinates are rescaled to [0, 1]. Default is False.\n is_center : boolean, default False\n Set to True, if the x and y of coordinates are the centroid (i.e. darknet format). Default is False.\n thresh_wh : float\n Threshold, remove the box if its ratio of width(height) to image size less than the threshold.\n thresh_wh2 : float\n Threshold, remove the box if its ratio of width to height or vice verse higher than the threshold.\n\n Returns\n -------\n numpy.array\n A processed image\n list of int\n A list of classes\n list of list of 4 numbers\n A list of new bounding boxes.\n\n \"\"\"\n if classes is None:\n classes = []\n if coords is None:\n coords = []\n\n h, w = im.shape[0], im.shape[1]\n\n if (h <= hrg) or (w <= wrg):\n raise AssertionError(\"The size of cropping should smaller than the original image\")\n\n if is_random:\n h_offset = int(np.random.uniform(0, h - hrg) - 1)\n w_offset = int(np.random.uniform(0, w - wrg) - 1)\n h_end = hrg + h_offset\n w_end = wrg + w_offset\n im_new = im[h_offset:h_end, w_offset:w_end]\n else: # central crop\n h_offset = int(np.floor((h - hrg) / 2.))\n w_offset = int(np.floor((w - wrg) / 2.))\n h_end = h_offset + hrg\n w_end = w_offset + wrg\n im_new = im[h_offset:h_end, w_offset:w_end]\n\n # w\n # _____________________________\n # | h/w offset |\n # | ------- |\n # h | | | |\n # | | | |\n # | ------- |\n # | h/w end |\n # |___________________________|\n\n def _get_coord(coord):\n \"\"\"Input pixel-unit [x, y, w, h] format, then make sure [x, y] it is the up-left coordinates,\n before getting the new coordinates.\n Boxes outsides the cropped image will be removed.\n\n \"\"\"\n if is_center:\n coord = obj_box_coord_centroid_to_upleft(coord)\n\n ##======= pixel unit format and upleft, w, h ==========##\n\n # x = np.clip( coord[0] - w_offset, 0, w_end - w_offset)\n # y = np.clip( coord[1] - h_offset, 0, h_end - h_offset)\n # w = np.clip( coord[2] , 0, w_end - w_offset)\n # h = np.clip( coord[3] , 0, h_end - h_offset)\n\n x = coord[0] - w_offset\n y = coord[1] - h_offset\n w = coord[2]\n h = coord[3]\n\n if x < 0:\n if x + w <= 0:\n return None\n w = w + x\n x = 0\n elif x > im_new.shape[1]: # object outside the cropped image\n return None\n\n if y < 0:\n if y + h <= 0:\n return None\n h = h + y\n y = 0\n elif y > im_new.shape[0]: # object outside the cropped image\n return None\n\n if (x is not None) and (x + w > im_new.shape[1]): # box outside the cropped image\n w = im_new.shape[1] - x\n\n if (y is not None) and (y + h > im_new.shape[0]): # box outside the cropped image\n h = im_new.shape[0] - y\n\n if (w / (h + 1.) > thresh_wh2) or (h / (w + 1.) > thresh_wh2): # object shape strange: too narrow\n # tl.logging.info('xx', w, h)\n return None\n\n if (w / (im_new.shape[1] * 1.) < thresh_wh) or (h / (im_new.shape[0] * 1.) <\n thresh_wh): # object shape strange: too narrow\n # tl.logging.info('yy', w, im_new.shape[1], h, im_new.shape[0])\n return None\n\n coord = [x, y, w, h]\n\n ## convert back if input format is center.\n if is_center:\n coord = obj_box_coord_upleft_to_centroid(coord)\n\n return coord\n\n coords_new = list()\n classes_new = list()\n for i, _ in enumerate(coords):\n coord = coords[i]\n\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n if is_rescale:\n # for scaled coord, upscaled before process and scale back in the end.\n coord = obj_box_coord_scale_to_pixelunit(coord, im.shape)\n coord = _get_coord(coord)\n if coord is not None:\n coord = obj_box_coord_rescale(coord, im_new.shape)\n coords_new.append(coord)\n classes_new.append(classes[i])\n else:\n coord = _get_coord(coord)\n if coord is not None:\n coords_new.append(coord)\n classes_new.append(classes[i])\n return im_new, classes_new, coords_new", "response": "Randomly crop an image and compute the new bounding box coordinates."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nshifting an image randomly or non - randomly and compute the new bounding box coordinates.", "response": "def obj_box_shift(\n im, classes=None, coords=None, wrg=0.1, hrg=0.1, row_index=0, col_index=1, channel_index=2, fill_mode='nearest',\n cval=0., order=1, is_rescale=False, is_center=False, is_random=False, thresh_wh=0.02, thresh_wh2=12.\n):\n \"\"\"Shift an image randomly or non-randomly, and compute the new bounding box coordinates.\n Objects outside the cropped image will be removed.\n\n Parameters\n -----------\n im : numpy.array\n An image with dimension of [row, col, channel] (default).\n classes : list of int or None\n Class IDs.\n coords : list of list of 4 int/float or None\n Coordinates [[x, y, w, h], [x, y, w, h], ...]\n wrg, hrg row_index col_index channel_index is_random fill_mode cval and order : see ``tl.prepro.shift``.\n is_rescale : boolean\n Set to True, if the input coordinates are rescaled to [0, 1]. Default is False.\n is_center : boolean\n Set to True, if the x and y of coordinates are the centroid (i.e. darknet format). Default is False.\n thresh_wh : float\n Threshold, remove the box if its ratio of width(height) to image size less than the threshold.\n thresh_wh2 : float\n Threshold, remove the box if its ratio of width to height or vice verse higher than the threshold.\n\n\n Returns\n -------\n numpy.array\n A processed image\n list of int\n A list of classes\n list of list of 4 numbers\n A list of new bounding boxes.\n\n \"\"\"\n if classes is None:\n classes = []\n if coords is None:\n coords = []\n\n imh, imw = im.shape[row_index], im.shape[col_index]\n\n if (hrg >= 1.0) and (hrg <= 0.) and (wrg >= 1.0) and (wrg <= 0.):\n raise AssertionError(\"shift range should be (0, 1)\")\n\n if is_random:\n tx = np.random.uniform(-hrg, hrg) * imh\n ty = np.random.uniform(-wrg, wrg) * imw\n else:\n tx, ty = hrg * imh, wrg * imw\n translation_matrix = np.array([[1, 0, tx], [0, 1, ty], [0, 0, 1]])\n\n transform_matrix = translation_matrix # no need to do offset\n im_new = affine_transform(im, transform_matrix, channel_index, fill_mode, cval, order)\n\n # modified from obj_box_crop\n def _get_coord(coord):\n \"\"\"Input pixel-unit [x, y, w, h] format, then make sure [x, y] it is the up-left coordinates,\n before getting the new coordinates.\n Boxes outsides the cropped image will be removed.\n\n \"\"\"\n if is_center:\n coord = obj_box_coord_centroid_to_upleft(coord)\n\n ##======= pixel unit format and upleft, w, h ==========##\n x = coord[0] - ty # only change this\n y = coord[1] - tx # only change this\n w = coord[2]\n h = coord[3]\n\n if x < 0:\n if x + w <= 0:\n return None\n w = w + x\n x = 0\n elif x > im_new.shape[1]: # object outside the cropped image\n return None\n\n if y < 0:\n if y + h <= 0:\n return None\n h = h + y\n y = 0\n elif y > im_new.shape[0]: # object outside the cropped image\n return None\n\n if (x is not None) and (x + w > im_new.shape[1]): # box outside the cropped image\n w = im_new.shape[1] - x\n\n if (y is not None) and (y + h > im_new.shape[0]): # box outside the cropped image\n h = im_new.shape[0] - y\n\n if (w / (h + 1.) > thresh_wh2) or (h / (w + 1.) > thresh_wh2): # object shape strange: too narrow\n # tl.logging.info('xx', w, h)\n return None\n\n if (w / (im_new.shape[1] * 1.) < thresh_wh) or (h / (im_new.shape[0] * 1.) <\n thresh_wh): # object shape strange: too narrow\n # tl.logging.info('yy', w, im_new.shape[1], h, im_new.shape[0])\n return None\n\n coord = [x, y, w, h]\n\n ## convert back if input format is center.\n if is_center:\n coord = obj_box_coord_upleft_to_centroid(coord)\n\n return coord\n\n coords_new = list()\n classes_new = list()\n for i, _ in enumerate(coords):\n coord = coords[i]\n\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n if is_rescale:\n # for scaled coord, upscaled before process and scale back in the end.\n coord = obj_box_coord_scale_to_pixelunit(coord, im.shape)\n coord = _get_coord(coord)\n if coord is not None:\n coord = obj_box_coord_rescale(coord, im_new.shape)\n coords_new.append(coord)\n classes_new.append(classes[i])\n else:\n coord = _get_coord(coord)\n if coord is not None:\n coords_new.append(coord)\n classes_new.append(classes[i])\n return im_new, classes_new, coords_new"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nzooms in and out of a single image randomly or non - randomly.", "response": "def obj_box_zoom(\n im, classes=None, coords=None, zoom_range=(0.9,\n 1.1), row_index=0, col_index=1, channel_index=2, fill_mode='nearest',\n cval=0., order=1, is_rescale=False, is_center=False, is_random=False, thresh_wh=0.02, thresh_wh2=12.\n):\n \"\"\"Zoom in and out of a single image, randomly or non-randomly, and compute the new bounding box coordinates.\n Objects outside the cropped image will be removed.\n\n Parameters\n -----------\n im : numpy.array\n An image with dimension of [row, col, channel] (default).\n classes : list of int or None\n Class IDs.\n coords : list of list of 4 int/float or None\n Coordinates [[x, y, w, h], [x, y, w, h], ...].\n zoom_range row_index col_index channel_index is_random fill_mode cval and order : see ``tl.prepro.zoom``.\n is_rescale : boolean\n Set to True, if the input coordinates are rescaled to [0, 1]. Default is False.\n is_center : boolean\n Set to True, if the x and y of coordinates are the centroid. (i.e. darknet format). Default is False.\n thresh_wh : float\n Threshold, remove the box if its ratio of width(height) to image size less than the threshold.\n thresh_wh2 : float\n Threshold, remove the box if its ratio of width to height or vice verse higher than the threshold.\n\n Returns\n -------\n numpy.array\n A processed image\n list of int\n A list of classes\n list of list of 4 numbers\n A list of new bounding boxes.\n\n \"\"\"\n if classes is None:\n classes = []\n if coords is None:\n coords = []\n\n if len(zoom_range) != 2:\n raise Exception('zoom_range should be a tuple or list of two floats. ' 'Received arg: ', zoom_range)\n if is_random:\n if zoom_range[0] == 1 and zoom_range[1] == 1:\n zx, zy = 1, 1\n tl.logging.info(\" random_zoom : not zoom in/out\")\n else:\n zx, zy = np.random.uniform(zoom_range[0], zoom_range[1], 2)\n else:\n zx, zy = zoom_range\n # tl.logging.info(zx, zy)\n zoom_matrix = np.array([[zx, 0, 0], [0, zy, 0], [0, 0, 1]])\n\n h, w = im.shape[row_index], im.shape[col_index]\n transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)\n im_new = affine_transform(im, transform_matrix, channel_index, fill_mode, cval, order)\n\n # modified from obj_box_crop\n def _get_coord(coord):\n \"\"\"Input pixel-unit [x, y, w, h] format, then make sure [x, y] it is the up-left coordinates,\n before getting the new coordinates.\n Boxes outsides the cropped image will be removed.\n\n \"\"\"\n if is_center:\n coord = obj_box_coord_centroid_to_upleft(coord)\n\n # ======= pixel unit format and upleft, w, h ==========\n x = (coord[0] - im.shape[1] / 2) / zy + im.shape[1] / 2 # only change this\n y = (coord[1] - im.shape[0] / 2) / zx + im.shape[0] / 2 # only change this\n w = coord[2] / zy # only change this\n h = coord[3] / zx # only change thisS\n\n if x < 0:\n if x + w <= 0:\n return None\n w = w + x\n x = 0\n elif x > im_new.shape[1]: # object outside the cropped image\n return None\n\n if y < 0:\n if y + h <= 0:\n return None\n h = h + y\n y = 0\n elif y > im_new.shape[0]: # object outside the cropped image\n return None\n\n if (x is not None) and (x + w > im_new.shape[1]): # box outside the cropped image\n w = im_new.shape[1] - x\n\n if (y is not None) and (y + h > im_new.shape[0]): # box outside the cropped image\n h = im_new.shape[0] - y\n\n if (w / (h + 1.) > thresh_wh2) or (h / (w + 1.) > thresh_wh2): # object shape strange: too narrow\n # tl.logging.info('xx', w, h)\n return None\n\n if (w / (im_new.shape[1] * 1.) < thresh_wh) or (h / (im_new.shape[0] * 1.) <\n thresh_wh): # object shape strange: too narrow\n # tl.logging.info('yy', w, im_new.shape[1], h, im_new.shape[0])\n return None\n\n coord = [x, y, w, h]\n\n # convert back if input format is center.\n if is_center:\n coord = obj_box_coord_upleft_to_centroid(coord)\n\n return coord\n\n coords_new = list()\n classes_new = list()\n for i, _ in enumerate(coords):\n coord = coords[i]\n\n if len(coord) != 4:\n raise AssertionError(\"coordinate should be 4 values : [x, y, w, h]\")\n\n if is_rescale:\n # for scaled coord, upscaled before process and scale back in the end.\n coord = obj_box_coord_scale_to_pixelunit(coord, im.shape)\n coord = _get_coord(coord)\n if coord is not None:\n coord = obj_box_coord_rescale(coord, im_new.shape)\n coords_new.append(coord)\n classes_new.append(classes[i])\n else:\n coord = _get_coord(coord)\n if coord is not None:\n coords_new.append(coord)\n classes_new.append(classes[i])\n return im_new, classes_new, coords_new"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pad_sequences(sequences, maxlen=None, dtype='int32', padding='post', truncating='pre', value=0.):\n lengths = [len(s) for s in sequences]\n\n nb_samples = len(sequences)\n if maxlen is None:\n maxlen = np.max(lengths)\n\n # take the sample shape from the first non empty sequence\n # checking for consistency in the main loop below.\n sample_shape = tuple()\n for s in sequences:\n if len(s) > 0:\n sample_shape = np.asarray(s).shape[1:]\n break\n\n x = (np.ones((nb_samples, maxlen) + sample_shape) * value).astype(dtype)\n for idx, s in enumerate(sequences):\n if len(s) == 0:\n continue # empty list was found\n if truncating == 'pre':\n trunc = s[-maxlen:]\n elif truncating == 'post':\n trunc = s[:maxlen]\n else:\n raise ValueError('Truncating type \"%s\" not understood' % truncating)\n\n # check `trunc` has expected shape\n trunc = np.asarray(trunc, dtype=dtype)\n if trunc.shape[1:] != sample_shape:\n raise ValueError(\n 'Shape of sample %s of sequence at position %s is different from expected shape %s' %\n (trunc.shape[1:], idx, sample_shape)\n )\n\n if padding == 'post':\n x[idx, :len(trunc)] = trunc\n elif padding == 'pre':\n x[idx, -len(trunc):] = trunc\n else:\n raise ValueError('Padding type \"%s\" not understood' % padding)\n return x.tolist()", "response": "Returns a new array with the same shape as the given list of integers."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving padding from a list of intA sequence.", "response": "def remove_pad_sequences(sequences, pad_id=0):\n \"\"\"Remove padding.\n\n Parameters\n -----------\n sequences : list of list of int\n All sequences where each row is a sequence.\n pad_id : int\n The pad ID.\n\n Returns\n ----------\n list of list of int\n The processed sequences.\n\n Examples\n ----------\n >>> sequences = [[2,3,4,0,0], [5,1,2,3,4,0,0,0], [4,5,0,2,4,0,0,0]]\n >>> print(remove_pad_sequences(sequences, pad_id=0))\n [[2, 3, 4], [5, 1, 2, 3, 4], [4, 5, 0, 2, 4]]\n\n \"\"\"\n sequences_out = copy.deepcopy(sequences)\n\n for i, _ in enumerate(sequences):\n # for j in range(len(sequences[i])):\n # if sequences[i][j] == pad_id:\n # sequences_out[i] = sequences_out[i][:j]\n # break\n for j in range(1, len(sequences[i])):\n if sequences[i][-j] != pad_id:\n sequences_out[i] = sequences_out[i][0:-j + 1]\n break\n\n return sequences_out"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds special start token IDs in the beginning of each sequence.", "response": "def sequences_add_start_id(sequences, start_id=0, remove_last=False):\n \"\"\"Add special start token(id) in the beginning of each sequence.\n\n Parameters\n ------------\n sequences : list of list of int\n All sequences where each row is a sequence.\n start_id : int\n The start ID.\n remove_last : boolean\n Remove the last value of each sequences. Usually be used for removing the end ID.\n\n Returns\n ----------\n list of list of int\n The processed sequences.\n\n Examples\n ---------\n >>> sentences_ids = [[4,3,5,3,2,2,2,2], [5,3,9,4,9,2,2,3]]\n >>> sentences_ids = sequences_add_start_id(sentences_ids, start_id=2)\n [[2, 4, 3, 5, 3, 2, 2, 2, 2], [2, 5, 3, 9, 4, 9, 2, 2, 3]]\n >>> sentences_ids = sequences_add_start_id(sentences_ids, start_id=2, remove_last=True)\n [[2, 4, 3, 5, 3, 2, 2, 2], [2, 5, 3, 9, 4, 9, 2, 2]]\n\n For Seq2seq\n\n >>> input = [a, b, c]\n >>> target = [x, y, z]\n >>> decode_seq = [start_id, a, b] <-- sequences_add_start_id(input, start_id, True)\n\n \"\"\"\n sequences_out = [[] for _ in range(len(sequences))] #[[]] * len(sequences)\n for i, _ in enumerate(sequences):\n if remove_last:\n sequences_out[i] = [start_id] + sequences[i][:-1]\n else:\n sequences_out[i] = [start_id] + sequences[i]\n return sequences_out"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd special end token id in the end of each sequence.", "response": "def sequences_add_end_id(sequences, end_id=888):\n \"\"\"Add special end token(id) in the end of each sequence.\n\n Parameters\n -----------\n sequences : list of list of int\n All sequences where each row is a sequence.\n end_id : int\n The end ID.\n\n Returns\n ----------\n list of list of int\n The processed sequences.\n\n Examples\n ---------\n >>> sequences = [[1,2,3],[4,5,6,7]]\n >>> print(sequences_add_end_id(sequences, end_id=999))\n [[1, 2, 3, 999], [4, 5, 6, 999]]\n\n \"\"\"\n sequences_out = [[] for _ in range(len(sequences))] #[[]] * len(sequences)\n for i, _ in enumerate(sequences):\n sequences_out[i] = sequences[i] + [end_id]\n return sequences_out"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds special end token(id) in the end of each sequence. Parameters ----------- sequences : list of list of int All sequences where each row is a sequence. end_id : int The end ID. pad_id : int The pad ID. Returns ---------- list of list of int The processed sequences. Examples --------- >>> sequences = [[1,2,0,0], [1,2,3,0], [1,2,3,4]] >>> print(sequences_add_end_id_after_pad(sequences, end_id=99, pad_id=0)) [[1, 2, 99, 0], [1, 2, 3, 99], [1, 2, 3, 4]]", "response": "def sequences_add_end_id_after_pad(sequences, end_id=888, pad_id=0):\n \"\"\"Add special end token(id) in the end of each sequence.\n\n Parameters\n -----------\n sequences : list of list of int\n All sequences where each row is a sequence.\n end_id : int\n The end ID.\n pad_id : int\n The pad ID.\n\n Returns\n ----------\n list of list of int\n The processed sequences.\n\n Examples\n ---------\n >>> sequences = [[1,2,0,0], [1,2,3,0], [1,2,3,4]]\n >>> print(sequences_add_end_id_after_pad(sequences, end_id=99, pad_id=0))\n [[1, 2, 99, 0], [1, 2, 3, 99], [1, 2, 3, 4]]\n\n \"\"\"\n # sequences_out = [[] for _ in range(len(sequences))]#[[]] * len(sequences)\n\n sequences_out = copy.deepcopy(sequences)\n # # add a pad to all\n # for i in range(len(sequences)):\n # for j in range(len(sequences[i])):\n # sequences_out[i].append(pad_id)\n # # pad -- > end\n # max_len = 0\n\n for i, v in enumerate(sequences):\n for j, _v2 in enumerate(v):\n if sequences[i][j] == pad_id:\n sequences_out[i][j] = end_id\n # if j > max_len:\n # max_len = j\n break\n\n # # remove pad if too long\n # for i in range(len(sequences)):\n # for j in range(len(sequences[i])):\n # sequences_out[i] = sequences_out[i][:max_len+1]\n return sequences_out"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sequences_get_mask(sequences, pad_val=0):\n mask = np.ones_like(sequences)\n for i, seq in enumerate(sequences):\n for i_w in reversed(range(len(seq))):\n if seq[i_w] == pad_val:\n mask[i, i_w] = 0\n else:\n break # <-- exit the for loop, prepcess next sequence\n return mask", "response": "Return a mask for the given list of intunused sequences."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef keypoint_resize_random_crop(image, annos, mask=None, size=(368, 368)):\n\n if len(np.shape(image)) == 2:\n image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n\n def resize_image(image, annos, mask, target_width, target_height):\n \"\"\"Reszie image\n\n Parameters\n -----------\n image : 3 channel image\n The given image.\n annos : list of list of floats\n Keypoints of people\n mask : single channel image or None\n The mask if available.\n target_width : int\n Expected width of returned image.\n target_height : int\n Expected height of returned image.\n\n Returns\n ----------\n preprocessed input image, annos, mask\n\n \"\"\"\n y, x, _ = np.shape(image)\n\n ratio_y = target_height / y\n ratio_x = target_width / x\n\n new_joints = []\n # update meta\n for people in annos:\n new_keypoints = []\n for keypoints in people:\n if keypoints[0] < 0 or keypoints[1] < 0:\n new_keypoints.append((-1000, -1000))\n continue\n pts = (int(keypoints[0] * ratio_x + 0.5), int(keypoints[1] * ratio_y + 0.5))\n if pts[0] > target_width - 1 or pts[1] > target_height - 1:\n new_keypoints.append((-1000, -1000))\n continue\n\n new_keypoints.append(pts)\n new_joints.append(new_keypoints)\n annos = new_joints\n\n new_image = cv2.resize(image, (target_width, target_height), interpolation=cv2.INTER_AREA)\n if mask is not None:\n new_mask = cv2.resize(mask, (target_width, target_height), interpolation=cv2.INTER_AREA)\n return new_image, annos, new_mask\n else:\n return new_image, annos, None\n\n _target_height = size[0]\n _target_width = size[1]\n if len(np.shape(image)) == 2:\n image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n height, width, _ = np.shape(image)\n # print(\"the size of original img is:\", height, width)\n if height <= width:\n ratio = _target_height / height\n new_width = int(ratio * width)\n if height == width:\n new_width = _target_height\n\n image, annos, mask = resize_image(image, annos, mask, new_width, _target_height)\n\n # for i in annos:\n # if len(i) is not 19:\n # print('Joints of person is not 19 ERROR FROM RESIZE')\n\n if new_width > _target_width:\n crop_range_x = np.random.randint(0, new_width - _target_width)\n else:\n crop_range_x = 0\n image = image[:, crop_range_x:crop_range_x + _target_width, :]\n if mask is not None:\n mask = mask[:, crop_range_x:crop_range_x + _target_width]\n # joint_list= []\n new_joints = []\n #annos-pepople-joints (must be 19 or [])\n for people in annos:\n # print(\"number of keypoints is\", np.shape(people))\n new_keypoints = []\n for keypoints in people:\n if keypoints[0] < -10 or keypoints[1] < -10:\n new_keypoints.append((-1000, -1000))\n continue\n top = crop_range_x + _target_width - 1\n if keypoints[0] >= crop_range_x and keypoints[0] <= top:\n # pts = (keypoints[0]-crop_range_x, keypoints[1])\n pts = (int(keypoints[0] - crop_range_x), int(keypoints[1]))\n else:\n pts = (-1000, -1000)\n new_keypoints.append(pts)\n\n new_joints.append(new_keypoints)\n # if len(new_keypoints) != 19:\n # print('1:The Length of joints list should be 0 or 19 but actually:', len(new_keypoints))\n annos = new_joints\n\n if height > width:\n ratio = _target_width / width\n new_height = int(ratio * height)\n image, annos, mask = resize_image(image, annos, mask, _target_width, new_height)\n\n # for i in annos:\n # if len(i) is not 19:\n # print('Joints of person is not 19 ERROR')\n\n if new_height > _target_height:\n crop_range_y = np.random.randint(0, new_height - _target_height)\n\n else:\n crop_range_y = 0\n image = image[crop_range_y:crop_range_y + _target_width, :, :]\n if mask is not None:\n mask = mask[crop_range_y:crop_range_y + _target_width, :]\n new_joints = []\n\n for people in annos: # TODO : speed up with affine transform\n new_keypoints = []\n for keypoints in people:\n\n # case orginal points are not usable\n if keypoints[0] < 0 or keypoints[1] < 0:\n new_keypoints.append((-1000, -1000))\n continue\n # y axis coordinate change\n bot = crop_range_y + _target_height - 1\n if keypoints[1] >= crop_range_y and keypoints[1] <= bot:\n # pts = (keypoints[0], keypoints[1]-crop_range_y)\n pts = (int(keypoints[0]), int(keypoints[1] - crop_range_y))\n # if pts[0]>367 or pts[1]>367:\n # print('Error2')\n else:\n pts = (-1000, -1000)\n\n new_keypoints.append(pts)\n\n new_joints.append(new_keypoints)\n # if len(new_keypoints) != 19:\n # print('2:The Length of joints list should be 0 or 19 but actually:', len(new_keypoints))\n\n annos = new_joints\n\n # mask = cv2.resize(mask, (46, 46), interpolation=cv2.INTER_AREA)\n if mask is not None:\n return image, annos, mask\n else:\n return image, annos, None", "response": "Resizes the image with randomly cropping the image with influence scales."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrotating an image and corresponding keypoints.", "response": "def keypoint_random_rotate(image, annos, mask=None, rg=15.):\n \"\"\"Rotate an image and corresponding keypoints.\n\n Parameters\n -----------\n image : 3 channel image\n The given image for augmentation.\n annos : list of list of floats\n The keypoints annotation of people.\n mask : single channel image or None\n The mask if available.\n rg : int or float\n Degree to rotate, usually 0 ~ 180.\n\n Returns\n ----------\n preprocessed image, annos, mask\n\n \"\"\"\n\n def _rotate_coord(shape, newxy, point, angle):\n angle = -1 * angle / 180.0 * math.pi\n ox, oy = shape\n px, py = point\n ox /= 2\n oy /= 2\n qx = math.cos(angle) * (px - ox) - math.sin(angle) * (py - oy)\n qy = math.sin(angle) * (px - ox) + math.cos(angle) * (py - oy)\n new_x, new_y = newxy\n qx += ox - new_x\n qy += oy - new_y\n return int(qx + 0.5), int(qy + 0.5)\n\n def _largest_rotated_rect(w, h, angle):\n \"\"\"\n Get largest rectangle after rotation.\n http://stackoverflow.com/questions/16702966/rotate-image-and-crop-out-black-borders\n \"\"\"\n angle = angle / 180.0 * math.pi\n if w <= 0 or h <= 0:\n return 0, 0\n\n width_is_longer = w >= h\n side_long, side_short = (w, h) if width_is_longer else (h, w)\n\n # since the solutions for angle, -angle and 180-angle are all the same,\n # if suffices to look at the first quadrant and the absolute values of sin,cos:\n sin_a, cos_a = abs(math.sin(angle)), abs(math.cos(angle))\n if side_short <= 2. * sin_a * cos_a * side_long:\n # half constrained case: two crop corners touch the longer side,\n # the other two corners are on the mid-line parallel to the longer line\n x = 0.5 * side_short\n wr, hr = (x / sin_a, x / cos_a) if width_is_longer else (x / cos_a, x / sin_a)\n else:\n # fully constrained case: crop touches all 4 sides\n cos_2a = cos_a * cos_a - sin_a * sin_a\n wr, hr = (w * cos_a - h * sin_a) / cos_2a, (h * cos_a - w * sin_a) / cos_2a\n return int(np.round(wr)), int(np.round(hr))\n\n img_shape = np.shape(image)\n height = img_shape[0]\n width = img_shape[1]\n deg = np.random.uniform(-rg, rg)\n\n img = image\n center = (img.shape[1] * 0.5, img.shape[0] * 0.5) # x, y\n rot_m = cv2.getRotationMatrix2D((int(center[0]), int(center[1])), deg, 1)\n ret = cv2.warpAffine(img, rot_m, img.shape[1::-1], flags=cv2.INTER_AREA, borderMode=cv2.BORDER_CONSTANT)\n if img.ndim == 3 and ret.ndim == 2:\n ret = ret[:, :, np.newaxis]\n neww, newh = _largest_rotated_rect(ret.shape[1], ret.shape[0], deg)\n neww = min(neww, ret.shape[1])\n newh = min(newh, ret.shape[0])\n newx = int(center[0] - neww * 0.5)\n newy = int(center[1] - newh * 0.5)\n # print(ret.shape, deg, newx, newy, neww, newh)\n img = ret[newy:newy + newh, newx:newx + neww]\n # adjust meta data\n adjust_joint_list = []\n for joint in annos: # TODO : speed up with affine transform\n adjust_joint = []\n for point in joint:\n if point[0] < -100 or point[1] < -100:\n adjust_joint.append((-1000, -1000))\n continue\n\n x, y = _rotate_coord((width, height), (newx, newy), point, deg)\n\n if x > neww - 1 or y > newh - 1:\n adjust_joint.append((-1000, -1000))\n continue\n if x < 0 or y < 0:\n adjust_joint.append((-1000, -1000))\n continue\n\n adjust_joint.append((x, y))\n adjust_joint_list.append(adjust_joint)\n joint_list = adjust_joint_list\n\n if mask is not None:\n msk = mask\n center = (msk.shape[1] * 0.5, msk.shape[0] * 0.5) # x, y\n rot_m = cv2.getRotationMatrix2D((int(center[0]), int(center[1])), deg, 1)\n ret = cv2.warpAffine(msk, rot_m, msk.shape[1::-1], flags=cv2.INTER_AREA, borderMode=cv2.BORDER_CONSTANT)\n if msk.ndim == 3 and msk.ndim == 2:\n ret = ret[:, :, np.newaxis]\n neww, newh = _largest_rotated_rect(ret.shape[1], ret.shape[0], deg)\n neww = min(neww, ret.shape[1])\n newh = min(newh, ret.shape[0])\n newx = int(center[0] - neww * 0.5)\n newy = int(center[1] - newh * 0.5)\n # print(ret.shape, deg, newx, newy, neww, newh)\n msk = ret[newy:newy + newh, newx:newx + neww]\n return img, joint_list, msk\n else:\n return img, joint_list, None"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef keypoint_random_flip(\n image, annos, mask=None, prob=0.5, flip_list=(0, 1, 5, 6, 7, 2, 3, 4, 11, 12, 13, 8, 9, 10, 15, 14, 17, 16, 18)\n):\n \"\"\"Flip an image and corresponding keypoints.\n\n Parameters\n -----------\n image : 3 channel image\n The given image for augmentation.\n annos : list of list of floats\n The keypoints annotation of people.\n mask : single channel image or None\n The mask if available.\n prob : float, 0 to 1\n The probability to flip the image, if 1, always flip the image.\n flip_list : tuple of int\n Denotes how the keypoints number be changed after flipping which is required for pose estimation task.\n The left and right body should be maintained rather than switch.\n (Default COCO format).\n Set to an empty tuple if you don't need to maintain left and right information.\n\n Returns\n ----------\n preprocessed image, annos, mask\n\n \"\"\"\n\n _prob = np.random.uniform(0, 1.0)\n if _prob < prob:\n return image, annos, mask\n\n _, width, _ = np.shape(image)\n image = cv2.flip(image, 1)\n mask = cv2.flip(mask, 1)\n new_joints = []\n for people in annos: # TODO : speed up with affine transform\n new_keypoints = []\n for k in flip_list:\n point = people[k]\n if point[0] < 0 or point[1] < 0:\n new_keypoints.append((-1000, -1000))\n continue\n if point[0] > image.shape[1] - 1 or point[1] > image.shape[0] - 1:\n new_keypoints.append((-1000, -1000))\n continue\n if (width - point[0]) > image.shape[1] - 1:\n new_keypoints.append((-1000, -1000))\n continue\n new_keypoints.append((width - point[0], point[1]))\n new_joints.append(new_keypoints)\n annos = new_joints\n\n return image, annos, mask", "response": "Flips an image and corresponding keypoints."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nbuilding the VGG 19 Model", "response": "def Vgg19(rgb):\n \"\"\"\n Build the VGG 19 Model\n\n Parameters\n -----------\n rgb : rgb image placeholder [batch, height, width, 3] values scaled [0, 1]\n \"\"\"\n start_time = time.time()\n print(\"build model started\")\n rgb_scaled = rgb * 255.0\n # Convert RGB to BGR\n red, green, blue = tf.split(rgb_scaled, 3, 3)\n\n if red.get_shape().as_list()[1:] != [224, 224, 1]:\n raise Exception(\"image size unmatch\")\n\n if green.get_shape().as_list()[1:] != [224, 224, 1]:\n raise Exception(\"image size unmatch\")\n\n if blue.get_shape().as_list()[1:] != [224, 224, 1]:\n raise Exception(\"image size unmatch\")\n\n bgr = tf.concat([\n blue - VGG_MEAN[0],\n green - VGG_MEAN[1],\n red - VGG_MEAN[2],\n ], axis=3)\n\n if bgr.get_shape().as_list()[1:] != [224, 224, 3]:\n raise Exception(\"image size unmatch\")\n # input layer\n net_in = InputLayer(bgr, name='input')\n # conv1\n net = Conv2dLayer(net_in, act=tf.nn.relu, shape=[3, 3, 3, 64], strides=[1, 1, 1, 1], padding='SAME', name='conv1_1')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 64, 64], strides=[1, 1, 1, 1], padding='SAME', name='conv1_2')\n net = PoolLayer(net, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', pool=tf.nn.max_pool, name='pool1')\n # conv2\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 64, 128], strides=[1, 1, 1, 1], padding='SAME', name='conv2_1')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 128, 128], strides=[1, 1, 1, 1], padding='SAME', name='conv2_2')\n net = PoolLayer(net, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', pool=tf.nn.max_pool, name='pool2')\n # conv3\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 128, 256], strides=[1, 1, 1, 1], padding='SAME', name='conv3_1')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 256, 256], strides=[1, 1, 1, 1], padding='SAME', name='conv3_2')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 256, 256], strides=[1, 1, 1, 1], padding='SAME', name='conv3_3')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 256, 256], strides=[1, 1, 1, 1], padding='SAME', name='conv3_4')\n net = PoolLayer(net, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', pool=tf.nn.max_pool, name='pool3')\n # conv4\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 256, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv4_1')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 512, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv4_2')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 512, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv4_3')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 512, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv4_4')\n net = PoolLayer(net, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', pool=tf.nn.max_pool, name='pool4')\n # conv5\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 512, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv5_1')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 512, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv5_2')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 512, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv5_3')\n net = Conv2dLayer(net, act=tf.nn.relu, shape=[3, 3, 512, 512], strides=[1, 1, 1, 1], padding='SAME', name='conv5_4')\n net = PoolLayer(net, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', pool=tf.nn.max_pool, name='pool5')\n # fc 6~8\n net = FlattenLayer(net, name='flatten')\n net = DenseLayer(net, n_units=4096, act=tf.nn.relu, name='fc6')\n net = DenseLayer(net, n_units=4096, act=tf.nn.relu, name='fc7')\n net = DenseLayer(net, n_units=1000, act=None, name='fc8')\n print(\"build model finished: %fs\" % (time.time() - start_time))\n return net"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Vgg19_simple_api(rgb):\n start_time = time.time()\n print(\"build model started\")\n rgb_scaled = rgb * 255.0\n # Convert RGB to BGR\n red, green, blue = tf.split(rgb_scaled, 3, 3)\n\n if red.get_shape().as_list()[1:] != [224, 224, 1]:\n raise Exception(\"image size unmatch\")\n\n if green.get_shape().as_list()[1:] != [224, 224, 1]:\n raise Exception(\"image size unmatch\")\n\n if blue.get_shape().as_list()[1:] != [224, 224, 1]:\n raise Exception(\"image size unmatch\")\n\n bgr = tf.concat([\n blue - VGG_MEAN[0],\n green - VGG_MEAN[1],\n red - VGG_MEAN[2],\n ], axis=3)\n\n if bgr.get_shape().as_list()[1:] != [224, 224, 3]:\n raise Exception(\"image size unmatch\")\n\n # input layer\n net_in = InputLayer(bgr, name='input')\n # conv1\n net = Conv2d(net_in, 64, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv1_1')\n net = Conv2d(net, n_filter=64, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv1_2')\n net = MaxPool2d(net, filter_size=(2, 2), strides=(2, 2), padding='SAME', name='pool1')\n # conv2\n net = Conv2d(net, n_filter=128, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv2_1')\n net = Conv2d(net, n_filter=128, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv2_2')\n net = MaxPool2d(net, filter_size=(2, 2), strides=(2, 2), padding='SAME', name='pool2')\n # conv3\n net = Conv2d(net, n_filter=256, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv3_1')\n net = Conv2d(net, n_filter=256, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv3_2')\n net = Conv2d(net, n_filter=256, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv3_3')\n net = Conv2d(net, n_filter=256, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv3_4')\n net = MaxPool2d(net, filter_size=(2, 2), strides=(2, 2), padding='SAME', name='pool3')\n # conv4\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv4_1')\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv4_2')\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv4_3')\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv4_4')\n net = MaxPool2d(net, filter_size=(2, 2), strides=(2, 2), padding='SAME', name='pool4')\n # conv5\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv5_1')\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv5_2')\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv5_3')\n net = Conv2d(net, n_filter=512, filter_size=(3, 3), strides=(1, 1), act=tf.nn.relu, padding='SAME', name='conv5_4')\n net = MaxPool2d(net, filter_size=(2, 2), strides=(2, 2), padding='SAME', name='pool5')\n # fc 6~8\n net = FlattenLayer(net, name='flatten')\n net = DenseLayer(net, n_units=4096, act=tf.nn.relu, name='fc6')\n net = DenseLayer(net, n_units=4096, act=tf.nn.relu, name='fc7')\n net = DenseLayer(net, n_units=1000, act=None, name='fc8')\n print(\"build model finished: %fs\" % (time.time() - start_time))\n return net", "response": "Simple API for VGG 19 Model"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef prepro(I):\n I = I[35:195]\n I = I[::2, ::2, 0]\n I[I == 144] = 0\n I[I == 109] = 0\n I[I != 0] = 1\n return I.astype(np.float).ravel()", "response": "Prepro 210x160x3 uint8 frame into 6400 1D float vector."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ntakes 1D float array of rewards and compute discounted rewards for an anonymized episode.", "response": "def discount_episode_rewards(rewards=None, gamma=0.99, mode=0):\n \"\"\"Take 1D float array of rewards and compute discounted rewards for an\n episode. When encount a non-zero value, consider as the end a of an episode.\n\n Parameters\n ----------\n rewards : list\n List of rewards\n gamma : float\n Discounted factor\n mode : int\n Mode for computing the discount rewards.\n - If mode == 0, reset the discount process when encount a non-zero reward (Ping-pong game).\n - If mode == 1, would not reset the discount process.\n\n Returns\n --------\n list of float\n The discounted rewards.\n\n Examples\n ----------\n >>> rewards = np.asarray([0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1])\n >>> gamma = 0.9\n >>> discount_rewards = tl.rein.discount_episode_rewards(rewards, gamma)\n >>> print(discount_rewards)\n [ 0.72899997 0.81 0.89999998 1. 0.72899997 0.81\n 0.89999998 1. 0.72899997 0.81 0.89999998 1. ]\n >>> discount_rewards = tl.rein.discount_episode_rewards(rewards, gamma, mode=1)\n >>> print(discount_rewards)\n [ 1.52110755 1.69011939 1.87791049 2.08656716 1.20729685 1.34144104\n 1.49048996 1.65610003 0.72899997 0.81 0.89999998 1. ]\n\n \"\"\"\n if rewards is None:\n raise Exception(\"rewards should be a list\")\n discounted_r = np.zeros_like(rewards, dtype=np.float32)\n running_add = 0\n for t in reversed(xrange(0, rewards.size)):\n if mode == 0:\n if rewards[t] != 0: running_add = 0\n\n running_add = running_add * gamma + rewards[t]\n discounted_r[t] = running_add\n return discounted_r"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cross_entropy_reward_loss(logits, actions, rewards, name=None):\n cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=actions, logits=logits, name=name)\n\n return tf.reduce_sum(tf.multiply(cross_entropy, rewards))", "response": "Calculate the loss for Policy Gradient Network."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nlogging weight. Parameters ----------- probs : tensor If it is a network output, usually we should scale it to [0, 1] via softmax. weights : tensor The weights. Returns -------- Tensor The Tensor after appling the log weighted expression.", "response": "def log_weight(probs, weights, name='log_weight'):\n \"\"\"Log weight.\n\n Parameters\n -----------\n probs : tensor\n If it is a network output, usually we should scale it to [0, 1] via softmax.\n weights : tensor\n The weights.\n\n Returns\n --------\n Tensor\n The Tensor after appling the log weighted expression.\n\n \"\"\"\n with tf.variable_scope(name):\n exp_v = tf.reduce_mean(tf.log(probs) * weights)\n return exp_v"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cross_entropy(output, target, name=None):\n if name is None:\n raise Exception(\"Please give a unique name to tl.cost.cross_entropy for TF1.0+\")\n return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output), name=name)", "response": "Softmax cross - entropy operation returns the TensorFlow expression of cross - entropy for two distributions."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sigmoid_cross_entropy(output, target, name=None):\n return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=target, logits=output), name=name)", "response": "Sigmoid cross - entropy operation."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef binary_cross_entropy(output, target, epsilon=1e-8, name='bce_loss'):\n # with ops.op_scope([output, target], name, \"bce_loss\") as name:\n # output = ops.convert_to_tensor(output, name=\"preds\")\n # target = ops.convert_to_tensor(targets, name=\"target\")\n\n # with tf.name_scope(name):\n return tf.reduce_mean(\n tf.reduce_sum(-(target * tf.log(output + epsilon) + (1. - target) * tf.log(1. - output + epsilon)), axis=1),\n name=name\n )", "response": "Binary cross entropy operation."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the TensorFlow expression of mean - squared - error of two target distributions.", "response": "def mean_squared_error(output, target, is_mean=False, name=\"mean_squared_error\"):\n \"\"\"Return the TensorFlow expression of mean-square-error (L2) of two batch of data.\n\n Parameters\n ----------\n output : Tensor\n 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].\n target : Tensor\n The target distribution, format the same with `output`.\n is_mean : boolean\n Whether compute the mean or sum for each example.\n - If True, use ``tf.reduce_mean`` to compute the loss between one target and predict data.\n - If False, use ``tf.reduce_sum`` (default).\n name : str\n An optional name to attach to this function.\n\n References\n ------------\n - `Wiki Mean Squared Error `__\n\n \"\"\"\n # with tf.name_scope(name):\n if output.get_shape().ndims == 2: # [batch_size, n_feature]\n if is_mean:\n mse = tf.reduce_mean(tf.reduce_mean(tf.squared_difference(output, target), 1), name=name)\n else:\n mse = tf.reduce_mean(tf.reduce_sum(tf.squared_difference(output, target), 1), name=name)\n elif output.get_shape().ndims == 3: # [batch_size, w, h]\n if is_mean:\n mse = tf.reduce_mean(tf.reduce_mean(tf.squared_difference(output, target), [1, 2]), name=name)\n else:\n mse = tf.reduce_mean(tf.reduce_sum(tf.squared_difference(output, target), [1, 2]), name=name)\n elif output.get_shape().ndims == 4: # [batch_size, w, h, c]\n if is_mean:\n mse = tf.reduce_mean(tf.reduce_mean(tf.squared_difference(output, target), [1, 2, 3]), name=name)\n else:\n mse = tf.reduce_mean(tf.reduce_sum(tf.squared_difference(output, target), [1, 2, 3]), name=name)\n else:\n raise Exception(\"Unknow dimension\")\n return mse"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the TensorFlow expression of normalized mean - square - error of two distributions.", "response": "def normalized_mean_square_error(output, target, name=\"normalized_mean_squared_error_loss\"):\n \"\"\"Return the TensorFlow expression of normalized mean-square-error of two distributions.\n\n Parameters\n ----------\n output : Tensor\n 2D, 3D or 4D tensor i.e. [batch_size, n_feature], [batch_size, height, width] or [batch_size, height, width, channel].\n target : Tensor\n The target distribution, format the same with `output`.\n name : str\n An optional name to attach to this function.\n\n \"\"\"\n # with tf.name_scope(\"normalized_mean_squared_error_loss\"):\n if output.get_shape().ndims == 2: # [batch_size, n_feature]\n nmse_a = tf.sqrt(tf.reduce_sum(tf.squared_difference(output, target), axis=1))\n nmse_b = tf.sqrt(tf.reduce_sum(tf.square(target), axis=1))\n elif output.get_shape().ndims == 3: # [batch_size, w, h]\n nmse_a = tf.sqrt(tf.reduce_sum(tf.squared_difference(output, target), axis=[1, 2]))\n nmse_b = tf.sqrt(tf.reduce_sum(tf.square(target), axis=[1, 2]))\n elif output.get_shape().ndims == 4: # [batch_size, w, h, c]\n nmse_a = tf.sqrt(tf.reduce_sum(tf.squared_difference(output, target), axis=[1, 2, 3]))\n nmse_b = tf.sqrt(tf.reduce_sum(tf.square(target), axis=[1, 2, 3]))\n nmse = tf.reduce_mean(nmse_a / nmse_b, name=name)\n return nmse"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef absolute_difference_error(output, target, is_mean=False, name=\"absolute_difference_error_loss\"):\n # with tf.name_scope(\"absolute_difference_error_loss\"):\n if output.get_shape().ndims == 2: # [batch_size, n_feature]\n if is_mean:\n loss = tf.reduce_mean(tf.reduce_mean(tf.abs(output - target), 1), name=name)\n else:\n loss = tf.reduce_mean(tf.reduce_sum(tf.abs(output - target), 1), name=name)\n elif output.get_shape().ndims == 3: # [batch_size, w, h]\n if is_mean:\n loss = tf.reduce_mean(tf.reduce_mean(tf.abs(output - target), [1, 2]), name=name)\n else:\n loss = tf.reduce_mean(tf.reduce_sum(tf.abs(output - target), [1, 2]), name=name)\n elif output.get_shape().ndims == 4: # [batch_size, w, h, c]\n if is_mean:\n loss = tf.reduce_mean(tf.reduce_mean(tf.abs(output - target), [1, 2, 3]), name=name)\n else:\n loss = tf.reduce_mean(tf.reduce_sum(tf.abs(output - target), [1, 2, 3]), name=name)\n else:\n raise Exception(\"Unknow dimension\")\n return loss", "response": "Returns the TensorFlow expression of the absolute difference error of two target distribution."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef dice_coe(output, target, loss_type='jaccard', axis=(1, 2, 3), smooth=1e-5):\n inse = tf.reduce_sum(output * target, axis=axis)\n if loss_type == 'jaccard':\n l = tf.reduce_sum(output * output, axis=axis)\n r = tf.reduce_sum(target * target, axis=axis)\n elif loss_type == 'sorensen':\n l = tf.reduce_sum(output, axis=axis)\n r = tf.reduce_sum(target, axis=axis)\n else:\n raise Exception(\"Unknow loss_type\")\n # old axis=[0,1,2,3]\n # dice = 2 * (inse) / (l + r)\n # epsilon = 1e-5\n # dice = tf.clip_by_value(dice, 0, 1.0-epsilon) # if all empty, dice = 1\n # new haodong\n dice = (2. * inse + smooth) / (l + r + smooth)\n ##\n dice = tf.reduce_mean(dice, name='dice_coe')\n return dice", "response": "Calculates the similarity between two images and outputs."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef dice_hard_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):\n output = tf.cast(output > threshold, dtype=tf.float32)\n target = tf.cast(target > threshold, dtype=tf.float32)\n inse = tf.reduce_sum(tf.multiply(output, target), axis=axis)\n l = tf.reduce_sum(output, axis=axis)\n r = tf.reduce_sum(target, axis=axis)\n # old axis=[0,1,2,3]\n # hard_dice = 2 * (inse) / (l + r)\n # epsilon = 1e-5\n # hard_dice = tf.clip_by_value(hard_dice, 0, 1.0-epsilon)\n # new haodong\n hard_dice = (2. * inse + smooth) / (l + r + smooth)\n ##\n hard_dice = tf.reduce_mean(hard_dice, name='hard_dice')\n return hard_dice", "response": "Non - differentiable S\u00f8rensen\u2013Dice coefficient for comparing the similarity between two images."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef iou_coe(output, target, threshold=0.5, axis=(1, 2, 3), smooth=1e-5):\n pre = tf.cast(output > threshold, dtype=tf.float32)\n truth = tf.cast(target > threshold, dtype=tf.float32)\n inse = tf.reduce_sum(tf.multiply(pre, truth), axis=axis) # AND\n union = tf.reduce_sum(tf.cast(tf.add(pre, truth) >= 1, dtype=tf.float32), axis=axis) # OR\n # old axis=[0,1,2,3]\n # epsilon = 1e-5\n # batch_iou = inse / (union + epsilon)\n # new haodong\n batch_iou = (inse + smooth) / (union + smooth)\n iou = tf.reduce_mean(batch_iou, name='iou_coe')\n return iou", "response": "Non - differentiable Intersection over Intersection over Union IoU for comparing the output and target."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the expression of cross - entropy of two sequences implement softmax internally. Normally be used for fixed length RNN outputs.", "response": "def cross_entropy_seq(logits, target_seqs, batch_size=None): # , batch_size=1, num_steps=None):\n \"\"\"Returns the expression of cross-entropy of two sequences, implement\n softmax internally. Normally be used for fixed length RNN outputs, see `PTB example `__.\n\n Parameters\n ----------\n logits : Tensor\n 2D tensor with shape of `[batch_size * n_steps, n_classes]`.\n target_seqs : Tensor\n The target sequence, 2D tensor `[batch_size, n_steps]`, if the number of step is dynamic, please use ``tl.cost.cross_entropy_seq_with_mask`` instead.\n batch_size : None or int.\n Whether to divide the cost by batch size.\n - If integer, the return cost will be divided by `batch_size`.\n - If None (default), the return cost will not be divided by anything.\n\n Examples\n --------\n >>> see `PTB example `__.for more details\n >>> input_data = tf.placeholder(tf.int32, [batch_size, n_steps])\n >>> targets = tf.placeholder(tf.int32, [batch_size, n_steps])\n >>> # build the network\n >>> print(net.outputs)\n (batch_size * n_steps, n_classes)\n >>> cost = tl.cost.cross_entropy_seq(network.outputs, targets)\n\n \"\"\"\n sequence_loss_by_example_fn = tf.contrib.legacy_seq2seq.sequence_loss_by_example\n\n loss = sequence_loss_by_example_fn(\n [logits], [tf.reshape(target_seqs, [-1])], [tf.ones_like(tf.reshape(target_seqs, [-1]), dtype=tf.float32)]\n )\n # [tf.ones([batch_size * num_steps])])\n cost = tf.reduce_sum(loss) # / batch_size\n if batch_size is not None:\n cost = cost / batch_size\n return cost"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cross_entropy_seq_with_mask(logits, target_seqs, input_mask, return_details=False, name=None):\n targets = tf.reshape(target_seqs, [-1]) # to one vector\n weights = tf.to_float(tf.reshape(input_mask, [-1])) # to one vector like targets\n losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets, name=name) * weights\n # losses = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets, name=name)) # for TF1.0 and others\n\n loss = tf.divide(\n tf.reduce_sum(losses), # loss from mask. reduce_sum before element-wise mul with mask !!\n tf.reduce_sum(weights),\n name=\"seq_loss_with_mask\"\n )\n\n if return_details:\n return loss, losses, weights, targets\n else:\n return loss", "response": "Returns the expression of cross - entropy of two sequences implement\n softmax internally. Normally be used for Dynamic RNN with Synced sequence input and output."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cosine_similarity(v1, v2):\n\n return tf.reduce_sum(tf.multiply(v1, v2), 1) / \\\n (tf.sqrt(tf.reduce_sum(tf.multiply(v1, v1), 1)) *\n tf.sqrt(tf.reduce_sum(tf.multiply(v2, v2), 1)))", "response": "Cosine similarity between two sets of images."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a function that applies Li regularization to the given scale.", "response": "def li_regularizer(scale, scope=None):\n \"\"\"Li regularization removes the neurons of previous layer. The `i` represents `inputs`.\n Returns a function that can be used to apply group li regularization to weights.\n The implementation follows `TensorFlow contrib `__.\n\n Parameters\n ----------\n scale : float\n A scalar multiplier `Tensor`. 0.0 disables the regularizer.\n scope: str\n An optional scope name for this function.\n\n Returns\n --------\n A function with signature `li(weights, name=None)` that apply Li regularization.\n\n Raises\n ------\n ValueError : if scale is outside of the range [0.0, 1.0] or if scale is not a float.\n\n \"\"\"\n if isinstance(scale, numbers.Integral):\n raise ValueError('scale cannot be an integer: %s' % scale)\n if isinstance(scale, numbers.Real):\n if scale < 0.:\n raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)\n if scale >= 1.:\n raise ValueError('Setting a scale greater than 1 on a regularizer: %g' % scale)\n if scale == 0.:\n tl.logging.info('Scale of 0 disables regularizer.')\n return lambda _, name=None: None\n\n def li(weights):\n \"\"\"Applies li regularization to weights.\"\"\"\n with tf.name_scope('li_regularizer') as scope:\n my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')\n # if tf.__version__ <= '0.12':\n # standard_ops_fn = standard_ops.mul\n # else:\n standard_ops_fn = standard_ops.multiply\n return standard_ops_fn(\n my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),\n name=scope\n )\n\n return li"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef maxnorm_regularizer(scale=1.0):\n if isinstance(scale, numbers.Integral):\n raise ValueError('scale cannot be an integer: %s' % scale)\n\n if isinstance(scale, numbers.Real):\n if scale < 0.:\n raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)\n # if scale >= 1.:\n # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %\n # scale)\n if scale == 0.:\n tl.logging.info('Scale of 0 disables regularizer.')\n return lambda _, name=None: None\n\n def mn(weights, name='max_regularizer'):\n \"\"\"Applies max-norm regularization to weights.\"\"\"\n with tf.name_scope(name) as scope:\n my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')\n # if tf.__version__ <= '0.12':\n # standard_ops_fn = standard_ops.mul\n # else:\n standard_ops_fn = standard_ops.multiply\n return standard_ops_fn(my_scale, standard_ops.reduce_max(standard_ops.abs(weights)), name=scope)\n\n return mn", "response": "Max - norm regularization returns a function that can be used to apply max - norm regularization to weights."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef maxnorm_o_regularizer(scale):\n if isinstance(scale, numbers.Integral):\n raise ValueError('scale cannot be an integer: %s' % scale)\n\n if isinstance(scale, numbers.Real):\n if scale < 0.:\n raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale)\n # if scale >= 1.:\n # raise ValueError('Setting a scale greater than 1 on a regularizer: %g' %\n # scale)\n if scale == 0.:\n tl.logging.info('Scale of 0 disables regularizer.')\n return lambda _, name=None: None\n\n def mn_o(weights, name='maxnorm_o_regularizer'):\n \"\"\"Applies max-norm regularization to weights.\"\"\"\n with tf.name_scope(name) as scope:\n my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')\n if tf.__version__ <= '0.12':\n standard_ops_fn = standard_ops.mul\n else:\n standard_ops_fn = standard_ops.multiply\n return standard_ops_fn(\n my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 0)), name=scope\n )\n\n return mn_o", "response": "Max - norm output regularization removes the neurons of current layer."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nramp activation function. Parameters ---------- x : Tensor input. v_min : float cap input to v_min as a lower bound. v_max : float cap input to v_max as a upper bound. name : str The function name (optional). Returns ------- Tensor A ``Tensor`` in the same type as ``x``.", "response": "def ramp(x, v_min=0, v_max=1, name=None):\n \"\"\"Ramp activation function.\n\n Parameters\n ----------\n x : Tensor\n input.\n v_min : float\n cap input to v_min as a lower bound.\n v_max : float\n cap input to v_max as a upper bound.\n name : str\n The function name (optional).\n\n Returns\n -------\n Tensor\n A ``Tensor`` in the same type as ``x``.\n\n \"\"\"\n return tf.clip_by_value(x, clip_value_min=v_min, clip_value_max=v_max, name=name)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef leaky_relu(x, alpha=0.2, name=\"leaky_relu\"):\n if not (0 < alpha <= 1):\n raise ValueError(\"`alpha` value must be in [0, 1]`\")\n\n with tf.name_scope(name, \"leaky_relu\") as name_scope:\n x = tf.convert_to_tensor(x, name=\"features\")\n return tf.maximum(x, alpha * x, name=name_scope)", "response": "leaky_relu can be used through its shortcut: :func:`tl.act.lrelu`.\n\n This function is a modified version of ReLU, introducing a nonzero gradient for negative input. Introduced by the paper:\n `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] `__\n\n The function return the following results:\n - When x < 0: ``f(x) = alpha_low * x``.\n - When x >= 0: ``f(x) = x``.\n\n Parameters\n ----------\n x : Tensor\n Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``.\n alpha : float\n Slope.\n name : str\n The function name (optional).\n\n Examples\n --------\n >>> import tensorlayer as tl\n >>> net = tl.layers.DenseLayer(net, 100, act=lambda x : tl.act.lrelu(x, 0.2), name='dense')\n\n Returns\n -------\n Tensor\n A ``Tensor`` in the same type as ``x``.\n\n References\n ----------\n - `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] `__"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nswish function. See `Swish: a Self-Gated Activation Function `__. Parameters ---------- x : Tensor input. name: str function name (optional). Returns ------- Tensor A ``Tensor`` in the same type as ``x``.", "response": "def swish(x, name='swish'):\n \"\"\"Swish function.\n\n See `Swish: a Self-Gated Activation Function `__.\n\n Parameters\n ----------\n x : Tensor\n input.\n name: str\n function name (optional).\n\n Returns\n -------\n Tensor\n A ``Tensor`` in the same type as ``x``.\n\n \"\"\"\n with tf.name_scope(name):\n x = tf.nn.sigmoid(x) * x\n return x"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pixel_wise_softmax(x, name='pixel_wise_softmax'):\n with tf.name_scope(name):\n return tf.nn.softmax(x)", "response": "A function that computes the softmax outputs of images every pixel have multiple label."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _conv_linear(args, filter_size, num_features, bias, bias_start=0.0, scope=None):\n # Calculate the total size of arguments on dimension 1.\n total_arg_size_depth = 0\n shapes = [a.get_shape().as_list() for a in args]\n for shape in shapes:\n if len(shape) != 4:\n raise ValueError(\"Linear is expecting 4D arguments: %s\" % str(shapes))\n if not shape[3]:\n raise ValueError(\"Linear expects shape[4] of arguments: %s\" % str(shapes))\n else:\n total_arg_size_depth += shape[3]\n\n dtype = [a.dtype for a in args][0]\n\n # Now the computation.\n with tf.variable_scope(scope or \"Conv\"):\n matrix = tf.get_variable(\n \"Matrix\", [filter_size[0], filter_size[1], total_arg_size_depth, num_features], dtype=dtype\n )\n if len(args) == 1:\n res = tf.nn.conv2d(args[0], matrix, strides=[1, 1, 1, 1], padding='SAME')\n else:\n res = tf.nn.conv2d(tf.concat(args, 3), matrix, strides=[1, 1, 1, 1], padding='SAME')\n if not bias:\n return res\n bias_term = tf.get_variable(\n \"Bias\", [num_features], dtype=dtype, initializer=tf.constant_initializer(bias_start, dtype=dtype)\n )\n return res + bias_term", "response": "Linear convolution of the current set of classes."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadvances Indexing for Sequences returns the outputs by given sequence lengths.", "response": "def advanced_indexing_op(inputs, index):\n \"\"\"Advanced Indexing for Sequences, returns the outputs by given sequence lengths.\n When return the last output :class:`DynamicRNNLayer` uses it to get the last outputs with the sequence lengths.\n\n Parameters\n -----------\n inputs : tensor for data\n With shape of [batch_size, n_step(max), n_features]\n index : tensor for indexing\n Sequence length in Dynamic RNN. [batch_size]\n\n Examples\n ---------\n >>> import numpy as np\n >>> import tensorflow as tf\n >>> import tensorlayer as tl\n >>> batch_size, max_length, n_features = 3, 5, 2\n >>> z = np.random.uniform(low=-1, high=1, size=[batch_size, max_length, n_features]).astype(np.float32)\n >>> b_z = tf.constant(z)\n >>> sl = tf.placeholder(dtype=tf.int32, shape=[batch_size])\n >>> o = advanced_indexing_op(b_z, sl)\n >>>\n >>> sess = tf.InteractiveSession()\n >>> tl.layers.initialize_global_variables(sess)\n >>>\n >>> order = np.asarray([1,1,2])\n >>> print(\"real\",z[0][order[0]-1], z[1][order[1]-1], z[2][order[2]-1])\n >>> y = sess.run([o], feed_dict={sl:order})\n >>> print(\"given\",order)\n >>> print(\"out\", y)\n real [-0.93021595 0.53820813] [-0.92548317 -0.77135968] [ 0.89952248 0.19149846]\n given [1 1 2]\n out [array([[-0.93021595, 0.53820813],\n [-0.92548317, -0.77135968],\n [ 0.89952248, 0.19149846]], dtype=float32)]\n\n References\n -----------\n - Modified from TFlearn (the original code is used for fixed length rnn), `references `__.\n\n \"\"\"\n batch_size = tf.shape(inputs)[0]\n # max_length = int(inputs.get_shape()[1]) # for fixed length rnn, length is given\n max_length = tf.shape(inputs)[1] # for dynamic_rnn, length is unknown\n dim_size = int(inputs.get_shape()[2])\n index = tf.range(0, batch_size) * max_length + (index - 1)\n flat = tf.reshape(inputs, [-1, dim_size])\n relevant = tf.gather(flat, index)\n return relevant"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef retrieve_seq_length_op(data):\n with tf.name_scope('GetLength'):\n used = tf.sign(tf.reduce_max(tf.abs(data), 2))\n length = tf.reduce_sum(used, 1)\n\n return tf.cast(length, tf.int32)", "response": "An op to compute the length of a sequence from input shape of batch_size n_step max n_features."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef retrieve_seq_length_op3(data, pad_val=0): # HangSheng: return tensor for sequence length, if input is tf.string\n data_shape_size = data.get_shape().ndims\n if data_shape_size == 3:\n return tf.reduce_sum(tf.cast(tf.reduce_any(tf.not_equal(data, pad_val), axis=2), dtype=tf.int32), 1)\n elif data_shape_size == 2:\n return tf.reduce_sum(tf.cast(tf.not_equal(data, pad_val), dtype=tf.int32), 1)\n elif data_shape_size == 1:\n raise ValueError(\"retrieve_seq_length_op3: data has wrong shape!\")\n else:\n raise ValueError(\n \"retrieve_seq_length_op3: handling data_shape_size %s hasn't been implemented!\" % (data_shape_size)\n )", "response": "Return tensor for sequence length, if input is ``tf.string``."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef zero_state(self, batch_size, dtype=LayersConfig.tf_dtype):\n shape = self.shape\n num_features = self.num_features\n # TODO : TypeError: 'NoneType' object is not subscriptable\n zeros = tf.zeros([batch_size, shape[0], shape[1], num_features * 2], dtype=dtype)\n return zeros", "response": "Return zero - filled state tensor with batch_size x shape [ 0 1 ) x num_features x 2."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef state_size(self):\n return (LSTMStateTuple(self._num_units, self._num_units) if self._state_is_tuple else 2 * self._num_units)", "response": "Return the size of the state tuple."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert from bchw to bcww", "response": "def _to_bc_h_w(self, x, x_shape):\n \"\"\"(b, h, w, c) -> (b*c, h, w)\"\"\"\n x = tf.transpose(x, [0, 3, 1, 2])\n x = tf.reshape(x, (-1, x_shape[1], x_shape[2]))\n return x"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _to_b_h_w_n_c(self, x, x_shape):\n x = tf.reshape(x, (-1, x_shape[4], x_shape[1], x_shape[2], x_shape[3]))\n x = tf.transpose(x, [0, 2, 3, 4, 1])\n return x", "response": "Convert from b h w n c to b h w n c."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _tf_repeat(self, a, repeats):\n # https://github.com/tensorflow/tensorflow/issues/8521\n\n if len(a.get_shape()) != 1:\n raise AssertionError(\"This is not a 1D Tensor\")\n\n a = tf.expand_dims(a, -1)\n a = tf.tile(a, [1, repeats])\n a = self.tf_flatten(a)\n return a", "response": "Tensorflow version of np. repeat for 1D tensors"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbatching version of tf_map_coordinates Only supports 2D feature maps", "response": "def _tf_batch_map_coordinates(self, inputs, coords):\n \"\"\"Batch version of tf_map_coordinates\n\n Only supports 2D feature maps\n\n Parameters\n ----------\n inputs : ``tf.Tensor``\n shape = (b*c, h, w)\n coords : ``tf.Tensor``\n shape = (b*c, h, w, n, 2)\n\n Returns\n -------\n ``tf.Tensor``\n A Tensor with the shape as (b*c, h, w, n)\n\n \"\"\"\n input_shape = inputs.get_shape()\n coords_shape = coords.get_shape()\n batch_channel = tf.shape(inputs)[0]\n input_h = int(input_shape[1])\n input_w = int(input_shape[2])\n kernel_n = int(coords_shape[3])\n n_coords = input_h * input_w * kernel_n\n\n coords_lt = tf.cast(tf.floor(coords), 'int32')\n coords_rb = tf.cast(tf.ceil(coords), 'int32')\n coords_lb = tf.stack([coords_lt[:, :, :, :, 0], coords_rb[:, :, :, :, 1]], axis=-1)\n coords_rt = tf.stack([coords_rb[:, :, :, :, 0], coords_lt[:, :, :, :, 1]], axis=-1)\n\n idx = self._tf_repeat(tf.range(batch_channel), n_coords)\n\n vals_lt = self._get_vals_by_coords(inputs, coords_lt, idx, (batch_channel, input_h, input_w, kernel_n))\n vals_rb = self._get_vals_by_coords(inputs, coords_rb, idx, (batch_channel, input_h, input_w, kernel_n))\n vals_lb = self._get_vals_by_coords(inputs, coords_lb, idx, (batch_channel, input_h, input_w, kernel_n))\n vals_rt = self._get_vals_by_coords(inputs, coords_rt, idx, (batch_channel, input_h, input_w, kernel_n))\n\n coords_offset_lt = coords - tf.cast(coords_lt, 'float32')\n\n vals_t = vals_lt + (vals_rt - vals_lt) * coords_offset_lt[:, :, :, :, 0]\n vals_b = vals_lb + (vals_rb - vals_lb) * coords_offset_lt[:, :, :, :, 0]\n mapped_vals = vals_t + (vals_b - vals_t) * coords_offset_lt[:, :, :, :, 1]\n\n return mapped_vals"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nbatching map offsets into input.", "response": "def _tf_batch_map_offsets(self, inputs, offsets, grid_offset):\n \"\"\"Batch map offsets into input\n\n Parameters\n ------------\n inputs : ``tf.Tensor``\n shape = (b, h, w, c)\n offsets: ``tf.Tensor``\n shape = (b, h, w, 2*n)\n grid_offset: `tf.Tensor``\n Offset grids shape = (h, w, n, 2)\n\n Returns\n -------\n ``tf.Tensor``\n A Tensor with the shape as (b, h, w, c)\n\n \"\"\"\n input_shape = inputs.get_shape()\n batch_size = tf.shape(inputs)[0]\n kernel_n = int(int(offsets.get_shape()[3]) / 2)\n input_h = input_shape[1]\n input_w = input_shape[2]\n channel = input_shape[3]\n\n # inputs (b, h, w, c) --> (b*c, h, w)\n inputs = self._to_bc_h_w(inputs, input_shape)\n\n # offsets (b, h, w, 2*n) --> (b, h, w, n, 2)\n offsets = tf.reshape(offsets, (batch_size, input_h, input_w, kernel_n, 2))\n # offsets (b, h, w, n, 2) --> (b*c, h, w, n, 2)\n # offsets = tf.tile(offsets, [channel, 1, 1, 1, 1])\n\n coords = tf.expand_dims(grid_offset, 0) # grid_offset --> (1, h, w, n, 2)\n coords = tf.tile(coords, [batch_size, 1, 1, 1, 1]) + offsets # grid_offset --> (b, h, w, n, 2)\n\n # clip out of bound\n coords = tf.stack(\n [\n tf.clip_by_value(coords[:, :, :, :, 0], 0.0, tf.cast(input_h - 1, 'float32')),\n tf.clip_by_value(coords[:, :, :, :, 1], 0.0, tf.cast(input_w - 1, 'float32'))\n ], axis=-1\n )\n coords = tf.tile(coords, [channel, 1, 1, 1, 1])\n\n mapped_vals = self._tf_batch_map_coordinates(inputs, coords)\n # (b*c, h, w, n) --> (b, h, w, n, c)\n mapped_vals = self._to_b_h_w_n_c(mapped_vals, [batch_size, input_h, input_w, kernel_n, channel])\n\n return mapped_vals"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating a generator that input a group of example in numpy. array and their labels return the examples and labels by the given batch size.", "response": "def minibatches(inputs=None, targets=None, batch_size=None, allow_dynamic_batch_size=False, shuffle=False):\n \"\"\"Generate a generator that input a group of example in numpy.array and\n their labels, return the examples and labels by the given batch size.\n\n Parameters\n ----------\n inputs : numpy.array\n The input features, every row is a example.\n targets : numpy.array\n The labels of inputs, every row is a example.\n batch_size : int\n The batch size.\n allow_dynamic_batch_size: boolean\n Allow the use of the last data batch in case the number of examples is not a multiple of batch_size, this may result in unexpected behaviour if other functions expect a fixed-sized batch-size.\n shuffle : boolean\n Indicating whether to use a shuffling queue, shuffle the dataset before return.\n\n Examples\n --------\n >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])\n >>> y = np.asarray([0,1,2,3,4,5])\n >>> for batch in tl.iterate.minibatches(inputs=X, targets=y, batch_size=2, shuffle=False):\n >>> print(batch)\n (array([['a', 'a'], ['b', 'b']], dtype=' len(inputs):\n if allow_dynamic_batch_size:\n end_idx = len(inputs)\n else:\n break\n if shuffle:\n excerpt = indices[start_idx:end_idx]\n else:\n excerpt = slice(start_idx, end_idx)\n if (isinstance(inputs, list) or isinstance(targets, list)) and (shuffle ==True):\n # zsdonghao: for list indexing when shuffle==True\n yield [inputs[i] for i in excerpt], [targets[i] for i in excerpt]\n else:\n yield inputs[excerpt], targets[excerpt]"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates a generator that returns a batch of sequence inputs and targets.", "response": "def seq_minibatches(inputs, targets, batch_size, seq_length, stride=1):\n \"\"\"Generate a generator that return a batch of sequence inputs and targets.\n If `batch_size=100` and `seq_length=5`, one return will have 500 rows (examples).\n\n Parameters\n ----------\n inputs : numpy.array\n The input features, every row is a example.\n targets : numpy.array\n The labels of inputs, every element is a example.\n batch_size : int\n The batch size.\n seq_length : int\n The sequence length.\n stride : int\n The stride step, default is 1.\n\n Examples\n --------\n Synced sequence input and output.\n\n >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])\n >>> y = np.asarray([0, 1, 2, 3, 4, 5])\n >>> for batch in tl.iterate.seq_minibatches(inputs=X, targets=y, batch_size=2, seq_length=2, stride=1):\n >>> print(batch)\n (array([['a', 'a'], ['b', 'b'], ['b', 'b'], ['c', 'c']], dtype='>> return_last = True\n >>> num_steps = 2\n >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])\n >>> Y = np.asarray([0,1,2,3,4,5])\n >>> for batch in tl.iterate.seq_minibatches(inputs=X, targets=Y, batch_size=2, seq_length=num_steps, stride=1):\n >>> x, y = batch\n >>> if return_last:\n >>> tmp_y = y.reshape((-1, num_steps) + y.shape[1:])\n >>> y = tmp_y[:, -1]\n >>> print(x, y)\n [['a' 'a']\n ['b' 'b']\n ['b' 'b']\n ['c' 'c']] [1 2]\n [['c' 'c']\n ['d' 'd']\n ['d' 'd']\n ['e' 'e']] [3 4]\n\n \"\"\"\n if len(inputs) != len(targets):\n raise AssertionError(\"The length of inputs and targets should be equal\")\n\n n_loads = (batch_size * stride) + (seq_length - stride)\n\n for start_idx in range(0, len(inputs) - n_loads + 1, (batch_size * stride)):\n seq_inputs = np.zeros((batch_size, seq_length) + inputs.shape[1:], dtype=inputs.dtype)\n seq_targets = np.zeros((batch_size, seq_length) + targets.shape[1:], dtype=targets.dtype)\n for b_idx in xrange(batch_size):\n start_seq_idx = start_idx + (b_idx * stride)\n end_seq_idx = start_seq_idx + seq_length\n seq_inputs[b_idx] = inputs[start_seq_idx:end_seq_idx]\n seq_targets[b_idx] = targets[start_seq_idx:end_seq_idx]\n flatten_inputs = seq_inputs.reshape((-1, ) + inputs.shape[1:])\n flatten_targets = seq_targets.reshape((-1, ) + targets.shape[1:])\n yield flatten_inputs, flatten_targets"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef seq_minibatches2(inputs, targets, batch_size, num_steps):\n if len(inputs) != len(targets):\n raise AssertionError(\"The length of inputs and targets should be equal\")\n\n data_len = len(inputs)\n batch_len = data_len // batch_size\n # data = np.zeros([batch_size, batch_len])\n data = np.zeros((batch_size, batch_len) + inputs.shape[1:], dtype=inputs.dtype)\n data2 = np.zeros([batch_size, batch_len])\n\n for i in range(batch_size):\n data[i] = inputs[batch_len * i:batch_len * (i + 1)]\n data2[i] = targets[batch_len * i:batch_len * (i + 1)]\n\n epoch_size = (batch_len - 1) // num_steps\n\n if epoch_size == 0:\n raise ValueError(\"epoch_size == 0, decrease batch_size or num_steps\")\n\n for i in range(epoch_size):\n x = data[:, i * num_steps:(i + 1) * num_steps]\n x2 = data2[:, i * num_steps:(i + 1) * num_steps]\n yield (x, x2)", "response": "Generates a generator that iterates on two list of words and returns the source contexts and target contexts."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating a generator that iterates on a list of words using PTB s algorithm.", "response": "def ptb_iterator(raw_data, batch_size, num_steps):\n \"\"\"Generate a generator that iterates on a list of words, see `PTB example `__.\n Yields the source contexts and the target context by the given batch_size and num_steps (sequence_length).\n\n In TensorFlow's tutorial, this generates `batch_size` pointers into the raw\n PTB data, and allows minibatch iteration along these pointers.\n\n Parameters\n ----------\n raw_data : a list\n the context in list format; note that context usually be\n represented by splitting by space, and then convert to unique\n word IDs.\n batch_size : int\n the batch size.\n num_steps : int\n the number of unrolls. i.e. sequence_length\n\n Yields\n ------\n Pairs of the batched data, each a matrix of shape [batch_size, num_steps].\n The second element of the tuple is the same data time-shifted to the\n right by one.\n\n Raises\n ------\n ValueError : if batch_size or num_steps are too high.\n\n Examples\n --------\n >>> train_data = [i for i in range(20)]\n >>> for batch in tl.iterate.ptb_iterator(train_data, batch_size=2, num_steps=3):\n >>> x, y = batch\n >>> print(x, y)\n [[ 0 1 2] <---x 1st subset/ iteration\n [10 11 12]]\n [[ 1 2 3] <---y\n [11 12 13]]\n\n [[ 3 4 5] <--- 1st batch input 2nd subset/ iteration\n [13 14 15]] <--- 2nd batch input\n [[ 4 5 6] <--- 1st batch target\n [14 15 16]] <--- 2nd batch target\n\n [[ 6 7 8] 3rd subset/ iteration\n [16 17 18]]\n [[ 7 8 9]\n [17 18 19]]\n \"\"\"\n raw_data = np.array(raw_data, dtype=np.int32)\n\n data_len = len(raw_data)\n batch_len = data_len // batch_size\n data = np.zeros([batch_size, batch_len], dtype=np.int32)\n for i in range(batch_size):\n data[i] = raw_data[batch_len * i:batch_len * (i + 1)]\n\n epoch_size = (batch_len - 1) // num_steps\n\n if epoch_size == 0:\n raise ValueError(\"epoch_size == 0, decrease batch_size or num_steps\")\n\n for i in range(epoch_size):\n x = data[:, i * num_steps:(i + 1) * num_steps]\n y = data[:, i * num_steps + 1:(i + 1) * num_steps + 1]\n yield (x, y)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef deconv2d_bilinear_upsampling_initializer(shape):\n if shape[0] != shape[1]:\n raise Exception('deconv2d_bilinear_upsampling_initializer only supports symmetrical filter sizes')\n\n if shape[3] < shape[2]:\n raise Exception(\n 'deconv2d_bilinear_upsampling_initializer behaviour is not defined for num_in_channels < num_out_channels '\n )\n\n filter_size = shape[0]\n num_out_channels = shape[2]\n num_in_channels = shape[3]\n\n # Create bilinear filter kernel as numpy array\n bilinear_kernel = np.zeros([filter_size, filter_size], dtype=np.float32)\n scale_factor = (filter_size + 1) // 2\n if filter_size % 2 == 1:\n center = scale_factor - 1\n else:\n center = scale_factor - 0.5\n for x in range(filter_size):\n for y in range(filter_size):\n bilinear_kernel[x, y] = (1 - abs(x - center) / scale_factor) * (1 - abs(y - center) / scale_factor)\n weights = np.zeros((filter_size, filter_size, num_out_channels, num_in_channels))\n for i in range(num_out_channels):\n weights[:, :, i, i] = bilinear_kernel\n\n # assign numpy array to constant_initalizer and pass to get_variable\n return tf.constant_initializer(value=weights, dtype=LayersConfig.tf_dtype)", "response": "Returns the initializer that can be passed to DeConv2dLayer for initializing the\n weights in correspondence to channel - wise bilinear up - sampling."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_model(self, network=None, model_name='model', **kwargs):\n kwargs.update({'model_name': model_name})\n self._fill_project_info(kwargs) # put project_name into kwargs\n\n params = network.get_all_params()\n\n s = time.time()\n\n kwargs.update({'architecture': network.all_graphs, 'time': datetime.utcnow()})\n\n try:\n params_id = self.model_fs.put(self._serialization(params))\n kwargs.update({'params_id': params_id, 'time': datetime.utcnow()})\n self.db.Model.insert_one(kwargs)\n print(\"[Database] Save model: SUCCESS, took: {}s\".format(round(time.time() - s, 2)))\n return True\n except Exception as e:\n exc_type, exc_obj, exc_tb = sys.exc_info()\n fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n logging.info(\"{} {} {} {} {}\".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))\n print(\"[Database] Save model: FAIL\")\n return False", "response": "Save model architecture and parameters into database."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind and returns a model architecture and its parameters from the database which matches the requirement.", "response": "def find_top_model(self, sess, sort=None, model_name='model', **kwargs):\n \"\"\"Finds and returns a model architecture and its parameters from the database which matches the requirement.\n\n Parameters\n ----------\n sess : Session\n TensorFlow session.\n sort : List of tuple\n PyMongo sort comment, search \"PyMongo find one sorting\" and `collection level operations `__ for more details.\n model_name : str or None\n The name/key of model.\n kwargs : other events\n Other events, such as name, accuracy, loss, step number and etc (optinal).\n\n Examples\n ---------\n - see ``save_model``.\n\n Returns\n ---------\n network : TensorLayer layer\n Note that, the returned network contains all information of the document (record), e.g. if you saved accuracy in the document, you can get the accuracy by using ``net._accuracy``.\n \"\"\"\n # print(kwargs) # {}\n kwargs.update({'model_name': model_name})\n self._fill_project_info(kwargs)\n\n s = time.time()\n\n d = self.db.Model.find_one(filter=kwargs, sort=sort)\n\n _temp_file_name = '_find_one_model_ztemp_file'\n if d is not None:\n params_id = d['params_id']\n graphs = d['architecture']\n _datetime = d['time']\n exists_or_mkdir(_temp_file_name, False)\n with open(os.path.join(_temp_file_name, 'graph.pkl'), 'wb') as file:\n pickle.dump(graphs, file, protocol=pickle.HIGHEST_PROTOCOL)\n else:\n print(\"[Database] FAIL! Cannot find model: {}\".format(kwargs))\n return False\n try:\n params = self._deserialization(self.model_fs.get(params_id).read())\n np.savez(os.path.join(_temp_file_name, 'params.npz'), params=params)\n\n network = load_graph_and_params(name=_temp_file_name, sess=sess)\n del_folder(_temp_file_name)\n\n pc = self.db.Model.find(kwargs)\n print(\n \"[Database] Find one model SUCCESS. kwargs:{} sort:{} save time:{} took: {}s\".\n format(kwargs, sort, _datetime, round(time.time() - s, 2))\n )\n\n # put all informations of model into the TL layer\n for key in d:\n network.__dict__.update({\"_%s\" % key: d[key]})\n\n # check whether more parameters match the requirement\n params_id_list = pc.distinct('params_id')\n n_params = len(params_id_list)\n if n_params != 1:\n print(\" Note that there are {} models match the kwargs\".format(n_params))\n return network\n except Exception as e:\n exc_type, exc_obj, exc_tb = sys.exc_info()\n fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n logging.info(\"{} {} {} {} {}\".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))\n return False"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndelete all items in the database.", "response": "def delete_model(self, **kwargs):\n \"\"\"Delete model.\n\n Parameters\n -----------\n kwargs : logging information\n Find items to delete, leave it empty to delete all log.\n \"\"\"\n self._fill_project_info(kwargs)\n self.db.Model.delete_many(kwargs)\n logging.info(\"[Database] Delete Model SUCCESS\")"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsave one dataset into database.", "response": "def save_dataset(self, dataset=None, dataset_name=None, **kwargs):\n \"\"\"Saves one dataset into database, timestamp will be added automatically.\n\n Parameters\n ----------\n dataset : any type\n The dataset you want to store.\n dataset_name : str\n The name of dataset.\n kwargs : other events\n Other events, such as description, author and etc (optinal).\n\n Examples\n ----------\n Save dataset\n >>> db.save_dataset([X_train, y_train, X_test, y_test], 'mnist', description='this is a tutorial')\n\n Get dataset\n >>> dataset = db.find_top_dataset('mnist')\n\n Returns\n ---------\n boolean : Return True if save success, otherwise, return False.\n \"\"\"\n self._fill_project_info(kwargs)\n if dataset_name is None:\n raise Exception(\"dataset_name is None, please give a dataset name\")\n kwargs.update({'dataset_name': dataset_name})\n\n s = time.time()\n try:\n dataset_id = self.dataset_fs.put(self._serialization(dataset))\n kwargs.update({'dataset_id': dataset_id, 'time': datetime.utcnow()})\n self.db.Dataset.insert_one(kwargs)\n # print(\"[Database] Save params: {} SUCCESS, took: {}s\".format(file_name, round(time.time()-s, 2)))\n print(\"[Database] Save dataset: SUCCESS, took: {}s\".format(round(time.time() - s, 2)))\n return True\n except Exception as e:\n exc_type, exc_obj, exc_tb = sys.exc_info()\n fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n logging.info(\"{} {} {} {} {}\".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))\n print(\"[Database] Save dataset: FAIL\")\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfinds and returns a dataset from the database which matches the requirement.", "response": "def find_top_dataset(self, dataset_name=None, sort=None, **kwargs):\n \"\"\"Finds and returns a dataset from the database which matches the requirement.\n\n Parameters\n ----------\n dataset_name : str\n The name of dataset.\n sort : List of tuple\n PyMongo sort comment, search \"PyMongo find one sorting\" and `collection level operations `__ for more details.\n kwargs : other events\n Other events, such as description, author and etc (optinal).\n\n Examples\n ---------\n Save dataset\n >>> db.save_dataset([X_train, y_train, X_test, y_test], 'mnist', description='this is a tutorial')\n\n Get dataset\n >>> dataset = db.find_top_dataset('mnist')\n >>> datasets = db.find_datasets('mnist')\n\n Returns\n --------\n dataset : the dataset or False\n Return False if nothing found.\n\n \"\"\"\n\n self._fill_project_info(kwargs)\n if dataset_name is None:\n raise Exception(\"dataset_name is None, please give a dataset name\")\n kwargs.update({'dataset_name': dataset_name})\n\n s = time.time()\n\n d = self.db.Dataset.find_one(filter=kwargs, sort=sort)\n\n if d is not None:\n dataset_id = d['dataset_id']\n else:\n print(\"[Database] FAIL! Cannot find dataset: {}\".format(kwargs))\n return False\n try:\n dataset = self._deserialization(self.dataset_fs.get(dataset_id).read())\n pc = self.db.Dataset.find(kwargs)\n print(\"[Database] Find one dataset SUCCESS, {} took: {}s\".format(kwargs, round(time.time() - s, 2)))\n\n # check whether more datasets match the requirement\n dataset_id_list = pc.distinct('dataset_id')\n n_dataset = len(dataset_id_list)\n if n_dataset != 1:\n print(\" Note that there are {} datasets match the requirement\".format(n_dataset))\n return dataset\n except Exception as e:\n exc_type, exc_obj, exc_tb = sys.exc_info()\n fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n logging.info(\"{} {} {} {} {}\".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))\n return False"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nfinding and returns all datasets in the database which match the requirement.", "response": "def find_datasets(self, dataset_name=None, **kwargs):\n \"\"\"Finds and returns all datasets from the database which matches the requirement.\n In some case, the data in a dataset can be stored separately for better management.\n\n Parameters\n ----------\n dataset_name : str\n The name/key of dataset.\n kwargs : other events\n Other events, such as description, author and etc (optional).\n\n Returns\n --------\n params : the parameters, return False if nothing found.\n\n \"\"\"\n\n self._fill_project_info(kwargs)\n if dataset_name is None:\n raise Exception(\"dataset_name is None, please give a dataset name\")\n kwargs.update({'dataset_name': dataset_name})\n\n s = time.time()\n pc = self.db.Dataset.find(kwargs)\n\n if pc is not None:\n dataset_id_list = pc.distinct('dataset_id')\n dataset_list = []\n for dataset_id in dataset_id_list: # you may have multiple Buckets files\n tmp = self.dataset_fs.get(dataset_id).read()\n dataset_list.append(self._deserialization(tmp))\n else:\n print(\"[Database] FAIL! Cannot find any dataset: {}\".format(kwargs))\n return False\n\n print(\"[Database] Find {} datasets SUCCESS, took: {}s\".format(len(dataset_list), round(time.time() - s, 2)))\n return dataset_list"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndelete datasets. Parameters ----------- kwargs : logging information Find items to delete, leave it empty to delete all log.", "response": "def delete_datasets(self, **kwargs):\n \"\"\"Delete datasets.\n\n Parameters\n -----------\n kwargs : logging information\n Find items to delete, leave it empty to delete all log.\n\n \"\"\"\n\n self._fill_project_info(kwargs)\n self.db.Dataset.delete_many(kwargs)\n logging.info(\"[Database] Delete Dataset SUCCESS\")"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_training_log(self, **kwargs):\n\n self._fill_project_info(kwargs)\n kwargs.update({'time': datetime.utcnow()})\n _result = self.db.TrainLog.insert_one(kwargs)\n _log = self._print_dict(kwargs)\n logging.info(\"[Database] train log: \" + _log)", "response": "Saves the training log of the current project."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef save_validation_log(self, **kwargs):\n\n self._fill_project_info(kwargs)\n kwargs.update({'time': datetime.utcnow()})\n _result = self.db.ValidLog.insert_one(kwargs)\n _log = self._print_dict(kwargs)\n logging.info(\"[Database] valid log: \" + _log)", "response": "Saves the validation log of the current project."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndeletes training log. Parameters ----------- kwargs : logging information Find items to delete, leave it empty to delete all log. Examples --------- Save training log >>> db.save_training_log(accuracy=0.33) >>> db.save_training_log(accuracy=0.44) Delete logs that match the requirement >>> db.delete_training_log(accuracy=0.33) Delete all logs >>> db.delete_training_log()", "response": "def delete_training_log(self, **kwargs):\n \"\"\"Deletes training log.\n\n Parameters\n -----------\n kwargs : logging information\n Find items to delete, leave it empty to delete all log.\n\n Examples\n ---------\n Save training log\n >>> db.save_training_log(accuracy=0.33)\n >>> db.save_training_log(accuracy=0.44)\n\n Delete logs that match the requirement\n >>> db.delete_training_log(accuracy=0.33)\n\n Delete all logs\n >>> db.delete_training_log()\n \"\"\"\n self._fill_project_info(kwargs)\n self.db.TrainLog.delete_many(kwargs)\n logging.info(\"[Database] Delete TrainLog SUCCESS\")"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef delete_validation_log(self, **kwargs):\n self._fill_project_info(kwargs)\n self.db.ValidLog.delete_many(kwargs)\n logging.info(\"[Database] Delete ValidLog SUCCESS\")", "response": "Deletes all items in the validation log."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nuploading a task to the database.", "response": "def create_task(self, task_name=None, script=None, hyper_parameters=None, saved_result_keys=None, **kwargs):\n \"\"\"Uploads a task to the database, timestamp will be added automatically.\n\n Parameters\n -----------\n task_name : str\n The task name.\n script : str\n File name of the python script.\n hyper_parameters : dictionary\n The hyper parameters pass into the script.\n saved_result_keys : list of str\n The keys of the task results to keep in the database when the task finishes.\n kwargs : other parameters\n Users customized parameters such as description, version number.\n\n Examples\n -----------\n Uploads a task\n >>> db.create_task(task_name='mnist', script='example/tutorial_mnist_simple.py', description='simple tutorial')\n\n Finds and runs the latest task\n >>> db.run_top_task(sess=sess, sort=[(\"time\", pymongo.DESCENDING)])\n >>> db.run_top_task(sess=sess, sort=[(\"time\", -1)])\n\n Finds and runs the oldest task\n >>> db.run_top_task(sess=sess, sort=[(\"time\", pymongo.ASCENDING)])\n >>> db.run_top_task(sess=sess, sort=[(\"time\", 1)])\n\n \"\"\"\n if not isinstance(task_name, str): # is None:\n raise Exception(\"task_name should be string\")\n if not isinstance(script, str): # is None:\n raise Exception(\"script should be string\")\n if hyper_parameters is None:\n hyper_parameters = {}\n if saved_result_keys is None:\n saved_result_keys = []\n\n self._fill_project_info(kwargs)\n kwargs.update({'time': datetime.utcnow()})\n kwargs.update({'hyper_parameters': hyper_parameters})\n kwargs.update({'saved_result_keys': saved_result_keys})\n\n _script = open(script, 'rb').read()\n\n kwargs.update({'status': 'pending', 'script': _script, 'result': {}})\n self.db.Task.insert_one(kwargs)\n logging.info(\"[Database] Saved Task - task_name: {} script: {}\".format(task_name, script))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef run_top_task(self, task_name=None, sort=None, **kwargs):\n if not isinstance(task_name, str): # is None:\n raise Exception(\"task_name should be string\")\n self._fill_project_info(kwargs)\n kwargs.update({'status': 'pending'})\n\n # find task and set status to running\n task = self.db.Task.find_one_and_update(kwargs, {'$set': {'status': 'running'}}, sort=sort)\n\n try:\n # get task info e.g. hyper parameters, python script\n if task is None:\n logging.info(\"[Database] Find Task FAIL: key: {} sort: {}\".format(task_name, sort))\n return False\n else:\n logging.info(\"[Database] Find Task SUCCESS: key: {} sort: {}\".format(task_name, sort))\n _datetime = task['time']\n _script = task['script']\n _id = task['_id']\n _hyper_parameters = task['hyper_parameters']\n _saved_result_keys = task['saved_result_keys']\n logging.info(\" hyper parameters:\")\n for key in _hyper_parameters:\n globals()[key] = _hyper_parameters[key]\n logging.info(\" {}: {}\".format(key, _hyper_parameters[key]))\n # run task\n s = time.time()\n logging.info(\"[Database] Start Task: key: {} sort: {} push time: {}\".format(task_name, sort, _datetime))\n _script = _script.decode('utf-8')\n with tf.Graph().as_default(): # as graph: # clear all TF graphs\n exec(_script, globals())\n\n # set status to finished\n _ = self.db.Task.find_one_and_update({'_id': _id}, {'$set': {'status': 'finished'}})\n\n # return results\n __result = {}\n for _key in _saved_result_keys:\n logging.info(\" result: {}={} {}\".format(_key, globals()[_key], type(globals()[_key])))\n __result.update({\"%s\" % _key: globals()[_key]})\n _ = self.db.Task.find_one_and_update(\n {\n '_id': _id\n }, {'$set': {\n 'result': __result\n }}, return_document=pymongo.ReturnDocument.AFTER\n )\n logging.info(\n \"[Database] Finished Task: task_name - {} sort: {} push time: {} took: {}s\".\n format(task_name, sort, _datetime,\n time.time() - s)\n )\n return True\n except Exception as e:\n exc_type, exc_obj, exc_tb = sys.exc_info()\n fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]\n logging.info(\"{} {} {} {} {}\".format(exc_type, exc_obj, fname, exc_tb.tb_lineno, e))\n logging.info(\"[Database] Fail to run task\")\n # if fail, set status back to pending\n _ = self.db.Task.find_one_and_update({'_id': _id}, {'$set': {'status': 'pending'}})\n return False", "response": "This function runs a pending task that is in the first of the sorting list."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndelete all items in the database", "response": "def delete_tasks(self, **kwargs):\n \"\"\"Delete tasks.\n\n Parameters\n -----------\n kwargs : logging information\n Find items to delete, leave it empty to delete all log.\n\n Examples\n ---------\n >>> db.delete_tasks()\n\n \"\"\"\n\n self._fill_project_info(kwargs)\n self.db.Task.delete_many(kwargs)\n logging.info(\"[Database] Delete Task SUCCESS\")"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck if a task is not finished.", "response": "def check_unfinished_task(self, task_name=None, **kwargs):\n \"\"\"Finds and runs a pending task.\n\n Parameters\n -----------\n task_name : str\n The task name.\n kwargs : other parameters\n Users customized parameters such as description, version number.\n\n Examples\n ---------\n Wait until all tasks finish in user's local console\n\n >>> while not db.check_unfinished_task():\n >>> time.sleep(1)\n >>> print(\"all tasks finished\")\n >>> sess = tf.InteractiveSession()\n >>> net = db.find_top_model(sess=sess, sort=[(\"test_accuracy\", -1)])\n >>> print(\"the best accuracy {} is from model {}\".format(net._test_accuracy, net._name))\n\n Returns\n --------\n boolean : True for success, False for fail.\n\n \"\"\"\n\n if not isinstance(task_name, str): # is None:\n raise Exception(\"task_name should be string\")\n self._fill_project_info(kwargs)\n\n kwargs.update({'$or': [{'status': 'pending'}, {'status': 'running'}]})\n\n # ## find task\n # task = self.db.Task.find_one(kwargs)\n task = self.db.Task.find(kwargs)\n\n task_id_list = task.distinct('_id')\n n_task = len(task_id_list)\n\n if n_task == 0:\n logging.info(\"[Database] No unfinished task - task_name: {}\".format(task_name))\n return False\n else:\n\n logging.info(\"[Database] Find {} unfinished task - task_name: {}\".format(n_task, task_name))\n return True"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef augment_with_ngrams(unigrams, unigram_vocab_size, n_buckets, n=2):\n\n def get_ngrams(n):\n return list(zip(*[unigrams[i:] for i in range(n)]))\n\n def hash_ngram(ngram):\n bytes_ = array.array('L', ngram).tobytes()\n hash_ = int(hashlib.sha256(bytes_).hexdigest(), 16)\n return unigram_vocab_size + hash_ % n_buckets\n\n return unigrams + [hash_ngram(ngram) for i in range(2, n + 1) for ngram in get_ngrams(i)]", "response": "Augment unigram features with hashed n - gram features."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads IMDb data and augment with hashed n - gram features.", "response": "def load_and_preprocess_imdb_data(n_gram=None):\n \"\"\"Load IMDb data and augment with hashed n-gram features.\"\"\"\n X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(nb_words=VOCAB_SIZE)\n\n if n_gram is not None:\n X_train = np.array([augment_with_ngrams(x, VOCAB_SIZE, N_BUCKETS, n=n_gram) for x in X_train])\n X_test = np.array([augment_with_ngrams(x, VOCAB_SIZE, N_BUCKETS, n=n_gram) for x in X_test])\n\n return X_train, y_train, X_test, y_test"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading one image. Parameters ----------- image : str The image file name. path : str The image folder path. Returns ------- numpy.array The image.", "response": "def read_image(image, path=''):\n \"\"\"Read one image.\n\n Parameters\n -----------\n image : str\n The image file name.\n path : str\n The image folder path.\n\n Returns\n -------\n numpy.array\n The image.\n\n \"\"\"\n return imageio.imread(os.path.join(path, image))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef read_images(img_list, path='', n_threads=10, printable=True):\n imgs = []\n for idx in range(0, len(img_list), n_threads):\n b_imgs_list = img_list[idx:idx + n_threads]\n b_imgs = tl.prepro.threading_data(b_imgs_list, fn=read_image, path=path)\n # tl.logging.info(b_imgs.shape)\n imgs.extend(b_imgs)\n if printable:\n tl.logging.info('read %d from %s' % (len(imgs), path))\n return imgs", "response": "Reads all images in list by given path and name of each image file."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsaving a image. Parameters ----------- image : numpy array [w, h, c] image_path : str path", "response": "def save_image(image, image_path='_temp.png'):\n \"\"\"Save a image.\n\n Parameters\n -----------\n image : numpy array\n [w, h, c]\n image_path : str\n path\n\n \"\"\"\n try: # RGB\n imageio.imwrite(image_path, image)\n except Exception: # Greyscale\n imageio.imwrite(image_path, image[:, :, 0])"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves multiple images into one image.", "response": "def save_images(images, size, image_path='_temp.png'):\n \"\"\"Save multiple images into one single image.\n\n Parameters\n -----------\n images : numpy array\n (batch, w, h, c)\n size : list of 2 ints\n row and column number.\n number of images should be equal or less than size[0] * size[1]\n image_path : str\n save path\n\n Examples\n ---------\n >>> import numpy as np\n >>> import tensorlayer as tl\n >>> images = np.random.rand(64, 100, 100, 3)\n >>> tl.visualize.save_images(images, [8, 8], 'temp.png')\n\n \"\"\"\n if len(images.shape) == 3: # Greyscale [batch, h, w] --> [batch, h, w, 1]\n images = images[:, :, :, np.newaxis]\n\n def merge(images, size):\n h, w = images.shape[1], images.shape[2]\n img = np.zeros((h * size[0], w * size[1], 3), dtype=images.dtype)\n for idx, image in enumerate(images):\n i = idx % size[1]\n j = idx // size[1]\n img[j * h:j * h + h, i * w:i * w + w, :] = image\n return img\n\n def imsave(images, size, path):\n if np.max(images) <= 1 and (-1 <= np.min(images) < 0):\n images = ((images + 1) * 127.5).astype(np.uint8)\n elif np.max(images) <= 1 and np.min(images) >= 0:\n images = (images * 255).astype(np.uint8)\n\n return imageio.imwrite(path, merge(images, size))\n\n if len(images) > size[0] * size[1]:\n raise AssertionError(\"number of images should be equal or less than size[0] * size[1] {}\".format(len(images)))\n\n return imsave(images, size, image_path)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndraw bboxes and class labels on image. Return or save the image with bboxes and class labels.", "response": "def draw_boxes_and_labels_to_image(\n image, classes, coords, scores, classes_list, is_center=True, is_rescale=True, save_name=None\n):\n \"\"\"Draw bboxes and class labels on image. Return or save the image with bboxes, example in the docs of ``tl.prepro``.\n\n Parameters\n -----------\n image : numpy.array\n The RGB image [height, width, channel].\n classes : list of int\n A list of class ID (int).\n coords : list of int\n A list of list for coordinates.\n - Should be [x, y, x2, y2] (up-left and botton-right format)\n - If [x_center, y_center, w, h] (set is_center to True).\n scores : list of float\n A list of score (float). (Optional)\n classes_list : list of str\n for converting ID to string on image.\n is_center : boolean\n Whether the coordinates is [x_center, y_center, w, h]\n - If coordinates are [x_center, y_center, w, h], set it to True for converting it to [x, y, x2, y2] (up-left and botton-right) internally.\n - If coordinates are [x1, x2, y1, y2], set it to False.\n is_rescale : boolean\n Whether to rescale the coordinates from pixel-unit format to ratio format.\n - If True, the input coordinates are the portion of width and high, this API will scale the coordinates to pixel unit internally.\n - If False, feed the coordinates with pixel unit format.\n save_name : None or str\n The name of image file (i.e. image.png), if None, not to save image.\n\n Returns\n -------\n numpy.array\n The saved image.\n\n References\n -----------\n - OpenCV rectangle and putText.\n - `scikit-image `__.\n\n \"\"\"\n if len(coords) != len(classes):\n raise AssertionError(\"number of coordinates and classes are equal\")\n\n if len(scores) > 0 and len(scores) != len(classes):\n raise AssertionError(\"number of scores and classes are equal\")\n\n # don't change the original image, and avoid error https://stackoverflow.com/questions/30249053/python-opencv-drawing-errors-after-manipulating-array-with-numpy\n image = image.copy()\n\n imh, imw = image.shape[0:2]\n thick = int((imh + imw) // 430)\n\n for i, _v in enumerate(coords):\n if is_center:\n x, y, x2, y2 = tl.prepro.obj_box_coord_centroid_to_upleft_butright(coords[i])\n else:\n x, y, x2, y2 = coords[i]\n\n if is_rescale: # scale back to pixel unit if the coords are the portion of width and high\n x, y, x2, y2 = tl.prepro.obj_box_coord_scale_to_pixelunit([x, y, x2, y2], (imh, imw))\n\n cv2.rectangle(\n image,\n (int(x), int(y)),\n (int(x2), int(y2)), # up-left and botton-right\n [0, 255, 0],\n thick\n )\n\n cv2.putText(\n image,\n classes_list[classes[i]] + ((\" %.2f\" % (scores[i])) if (len(scores) != 0) else \" \"),\n (int(x), int(y)), # button left\n 0,\n 1.5e-3 * imh, # bigger = larger font\n [0, 0, 256], # self.meta['colors'][max_indx],\n int(thick / 2) + 1\n ) # bold\n\n if save_name is not None:\n # cv2.imwrite('_my.png', image)\n save_image(image, save_name)\n # if len(coords) == 0:\n # tl.logging.info(\"draw_boxes_and_labels_to_image: no bboxes exist, cannot draw !\")\n return image"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndraw people into image using MPII dataset format as input return or save the result image.", "response": "def draw_mpii_pose_to_image(image, poses, save_name='image.png'):\n \"\"\"Draw people(s) into image using MPII dataset format as input, return or save the result image.\n\n This is an experimental API, can be changed in the future.\n\n Parameters\n -----------\n image : numpy.array\n The RGB image [height, width, channel].\n poses : list of dict\n The people(s) annotation in MPII format, see ``tl.files.load_mpii_pose_dataset``.\n save_name : None or str\n The name of image file (i.e. image.png), if None, not to save image.\n\n Returns\n --------\n numpy.array\n The saved image.\n\n Examples\n --------\n >>> import pprint\n >>> import tensorlayer as tl\n >>> img_train_list, ann_train_list, img_test_list, ann_test_list = tl.files.load_mpii_pose_dataset()\n >>> image = tl.vis.read_image(img_train_list[0])\n >>> tl.vis.draw_mpii_pose_to_image(image, ann_train_list[0], 'image.png')\n >>> pprint.pprint(ann_train_list[0])\n\n References\n -----------\n - `MPII Keyponts and ID `__\n \"\"\"\n # import skimage\n # don't change the original image, and avoid error https://stackoverflow.com/questions/30249053/python-opencv-drawing-errors-after-manipulating-array-with-numpy\n image = image.copy()\n\n imh, imw = image.shape[0:2]\n thick = int((imh + imw) // 430)\n # radius = int(image.shape[1] / 500) + 1\n radius = int(thick * 1.5)\n\n if image.max() < 1:\n image = image * 255\n\n for people in poses:\n # Pose Keyponts\n joint_pos = people['joint_pos']\n # draw sketch\n # joint id (0 - r ankle, 1 - r knee, 2 - r hip, 3 - l hip, 4 - l knee,\n # 5 - l ankle, 6 - pelvis, 7 - thorax, 8 - upper neck,\n # 9 - head top, 10 - r wrist, 11 - r elbow, 12 - r shoulder,\n # 13 - l shoulder, 14 - l elbow, 15 - l wrist)\n #\n # 9\n # 8\n # 12 ** 7 ** 13\n # * * *\n # 11 * 14\n # * * *\n # 10 2 * 6 * 3 15\n # * *\n # 1 4\n # * *\n # 0 5\n\n lines = [\n [(0, 1), [100, 255, 100]],\n [(1, 2), [50, 255, 50]],\n [(2, 6), [0, 255, 0]], # right leg\n [(3, 4), [100, 100, 255]],\n [(4, 5), [50, 50, 255]],\n [(6, 3), [0, 0, 255]], # left leg\n [(6, 7), [255, 255, 100]],\n [(7, 8), [255, 150, 50]], # body\n [(8, 9), [255, 200, 100]], # head\n [(10, 11), [255, 100, 255]],\n [(11, 12), [255, 50, 255]],\n [(12, 8), [255, 0, 255]], # right hand\n [(8, 13), [0, 255, 255]],\n [(13, 14), [100, 255, 255]],\n [(14, 15), [200, 255, 255]] # left hand\n ]\n for line in lines:\n start, end = line[0]\n if (start in joint_pos) and (end in joint_pos):\n cv2.line(\n image,\n (int(joint_pos[start][0]), int(joint_pos[start][1])),\n (int(joint_pos[end][0]), int(joint_pos[end][1])), # up-left and botton-right\n line[1],\n thick\n )\n # rr, cc, val = skimage.draw.line_aa(int(joint_pos[start][1]), int(joint_pos[start][0]), int(joint_pos[end][1]), int(joint_pos[end][0]))\n # image[rr, cc] = line[1]\n # draw circles\n for pos in joint_pos.items():\n _, pos_loc = pos # pos_id, pos_loc\n pos_loc = (int(pos_loc[0]), int(pos_loc[1]))\n cv2.circle(image, center=pos_loc, radius=radius, color=(200, 200, 200), thickness=-1)\n # rr, cc = skimage.draw.circle(int(pos_loc[1]), int(pos_loc[0]), radius)\n # image[rr, cc] = [0, 255, 0]\n\n # Head\n head_rect = people['head_rect']\n if head_rect: # if head exists\n cv2.rectangle(\n image,\n (int(head_rect[0]), int(head_rect[1])),\n (int(head_rect[2]), int(head_rect[3])), # up-left and botton-right\n [0, 180, 0],\n thick\n )\n\n if save_name is not None:\n # cv2.imwrite(save_name, image)\n save_image(image, save_name)\n return image"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef frame(I=None, second=5, saveable=True, name='frame', cmap=None, fig_idx=12836):\n import matplotlib.pyplot as plt\n if saveable is False:\n plt.ion()\n plt.figure(fig_idx) # show all feature images\n\n if len(I.shape) and I.shape[-1] == 1: # (10,10,1) --> (10,10)\n I = I[:, :, 0]\n\n plt.imshow(I, cmap)\n plt.title(name)\n # plt.gca().xaxis.set_major_locator(plt.NullLocator()) # distable tick\n # plt.gca().yaxis.set_major_locator(plt.NullLocator())\n\n if saveable:\n plt.savefig(name + '.pdf', format='pdf')\n else:\n plt.draw()\n plt.pause(second)", "response": "Display a frame of the image."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndisplay a group of RGB or Greyscale CNN masks.", "response": "def CNN2d(CNN=None, second=10, saveable=True, name='cnn', fig_idx=3119362):\n \"\"\"Display a group of RGB or Greyscale CNN masks.\n\n Parameters\n ----------\n CNN : numpy.array\n The image. e.g: 64 5x5 RGB images can be (5, 5, 3, 64).\n second : int\n The display second(s) for the image(s), if saveable is False.\n saveable : boolean\n Save or plot the figure.\n name : str\n A name to save the image, if saveable is True.\n fig_idx : int\n The matplotlib figure index.\n\n Examples\n --------\n >>> tl.visualize.CNN2d(network.all_params[0].eval(), second=10, saveable=True, name='cnn1_mnist', fig_idx=2012)\n\n \"\"\"\n import matplotlib.pyplot as plt\n # tl.logging.info(CNN.shape) # (5, 5, 3, 64)\n # exit()\n n_mask = CNN.shape[3]\n n_row = CNN.shape[0]\n n_col = CNN.shape[1]\n n_color = CNN.shape[2]\n row = int(np.sqrt(n_mask))\n col = int(np.ceil(n_mask / row))\n plt.ion() # active mode\n fig = plt.figure(fig_idx)\n count = 1\n for _ir in range(1, row + 1):\n for _ic in range(1, col + 1):\n if count > n_mask:\n break\n fig.add_subplot(col, row, count)\n # tl.logging.info(CNN[:,:,:,count-1].shape, n_row, n_col) # (5, 1, 32) 5 5\n # exit()\n # plt.imshow(\n # np.reshape(CNN[count-1,:,:,:], (n_row, n_col)),\n # cmap='gray', interpolation=\"nearest\") # theano\n if n_color == 1:\n plt.imshow(np.reshape(CNN[:, :, :, count - 1], (n_row, n_col)), cmap='gray', interpolation=\"nearest\")\n elif n_color == 3:\n plt.imshow(\n np.reshape(CNN[:, :, :, count - 1], (n_row, n_col, n_color)), cmap='gray', interpolation=\"nearest\"\n )\n else:\n raise Exception(\"Unknown n_color\")\n plt.gca().xaxis.set_major_locator(plt.NullLocator()) # distable tick\n plt.gca().yaxis.set_major_locator(plt.NullLocator())\n count = count + 1\n if saveable:\n plt.savefig(name + '.pdf', format='pdf')\n else:\n plt.draw()\n plt.pause(second)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef tsne_embedding(embeddings, reverse_dictionary, plot_only=500, second=5, saveable=False, name='tsne', fig_idx=9862):\n import matplotlib.pyplot as plt\n\n def plot_with_labels(low_dim_embs, labels, figsize=(18, 18), second=5, saveable=True, name='tsne', fig_idx=9862):\n\n if low_dim_embs.shape[0] < len(labels):\n raise AssertionError(\"More labels than embeddings\")\n\n if saveable is False:\n plt.ion()\n plt.figure(fig_idx)\n\n plt.figure(figsize=figsize) # in inches\n\n for i, label in enumerate(labels):\n x, y = low_dim_embs[i, :]\n plt.scatter(x, y)\n plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n\n if saveable:\n plt.savefig(name + '.pdf', format='pdf')\n else:\n plt.draw()\n plt.pause(second)\n\n try:\n from sklearn.manifold import TSNE\n from six.moves import xrange\n\n tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n # plot_only = 500\n low_dim_embs = tsne.fit_transform(embeddings[:plot_only, :])\n labels = [reverse_dictionary[i] for i in xrange(plot_only)]\n plot_with_labels(low_dim_embs, labels, second=second, saveable=saveable, name=name, fig_idx=fig_idx)\n\n except ImportError:\n _err = \"Please install sklearn and matplotlib to visualize embeddings.\"\n tl.logging.error(_err)\n raise ImportError(_err)", "response": "Visualize the embeddings by using t - SNE."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nvisualizes every column of the weight matrix to a group of Greyscale img.", "response": "def draw_weights(W=None, second=10, saveable=True, shape=None, name='mnist', fig_idx=2396512):\n \"\"\"Visualize every columns of the weight matrix to a group of Greyscale img.\n\n Parameters\n ----------\n W : numpy.array\n The weight matrix\n second : int\n The display second(s) for the image(s), if saveable is False.\n saveable : boolean\n Save or plot the figure.\n shape : a list with 2 int or None\n The shape of feature image, MNIST is [28, 80].\n name : a string\n A name to save the image, if saveable is True.\n fig_idx : int\n matplotlib figure index.\n\n Examples\n --------\n >>> tl.visualize.draw_weights(network.all_params[0].eval(), second=10, saveable=True, name='weight_of_1st_layer', fig_idx=2012)\n\n \"\"\"\n if shape is None:\n shape = [28, 28]\n\n import matplotlib.pyplot as plt\n if saveable is False:\n plt.ion()\n fig = plt.figure(fig_idx) # show all feature images\n n_units = W.shape[1]\n\n num_r = int(np.sqrt(n_units)) # \u6bcf\u884c\u663e\u793a\u7684\u4e2a\u6570 \u82e525\u4e2ahidden unit -> \u6bcf\u884c\u663e\u793a5\u4e2a\n num_c = int(np.ceil(n_units / num_r))\n count = int(1)\n for _row in range(1, num_r + 1):\n for _col in range(1, num_c + 1):\n if count > n_units:\n break\n fig.add_subplot(num_r, num_c, count)\n # ------------------------------------------------------------\n # plt.imshow(np.reshape(W[:,count-1],(28,28)), cmap='gray')\n # ------------------------------------------------------------\n feature = W[:, count - 1] / np.sqrt((W[:, count - 1]**2).sum())\n # feature[feature<0.0001] = 0 # value threshold\n # if count == 1 or count == 2:\n # print(np.mean(feature))\n # if np.std(feature) < 0.03: # condition threshold\n # feature = np.zeros_like(feature)\n # if np.mean(feature) < -0.015: # condition threshold\n # feature = np.zeros_like(feature)\n plt.imshow(\n np.reshape(feature, (shape[0], shape[1])), cmap='gray', interpolation=\"nearest\"\n ) # , vmin=np.min(feature), vmax=np.max(feature))\n # plt.title(name)\n # ------------------------------------------------------------\n # plt.imshow(np.reshape(W[:,count-1] ,(np.sqrt(size),np.sqrt(size))), cmap='gray', interpolation=\"nearest\")\n plt.gca().xaxis.set_major_locator(plt.NullLocator()) # distable tick\n plt.gca().yaxis.set_major_locator(plt.NullLocator())\n count = count + 1\n if saveable:\n plt.savefig(name + '.pdf', format='pdf')\n else:\n plt.draw()\n plt.pause(second)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef data_to_tfrecord(images, labels, filename):\n if os.path.isfile(filename):\n print(\"%s exists\" % filename)\n return\n print(\"Converting data into %s ...\" % filename)\n # cwd = os.getcwd()\n writer = tf.python_io.TFRecordWriter(filename)\n for index, img in enumerate(images):\n img_raw = img.tobytes()\n # Visualize a image\n # tl.visualize.frame(np.asarray(img, dtype=np.uint8), second=1, saveable=False, name='frame', fig_idx=1236)\n label = int(labels[index])\n # print(label)\n # Convert the bytes back to image as follow:\n # image = Image.frombytes('RGB', (32, 32), img_raw)\n # image = np.fromstring(img_raw, np.float32)\n # image = image.reshape([32, 32, 3])\n # tl.visualize.frame(np.asarray(image, dtype=np.uint8), second=1, saveable=False, name='frame', fig_idx=1236)\n example = tf.train.Example(\n features=tf.train.Features(\n feature={\n \"label\": tf.train.Feature(int64_list=tf.train.Int64List(value=[label])),\n 'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw])),\n }\n )\n )\n writer.write(example.SerializeToString()) # Serialize To String\n writer.close()", "response": "Save data into TFRecord."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreads and decode a single file into a single object.", "response": "def read_and_decode(filename, is_train=None):\n \"\"\"Return tensor to read from TFRecord.\"\"\"\n filename_queue = tf.train.string_input_producer([filename])\n reader = tf.TFRecordReader()\n _, serialized_example = reader.read(filename_queue)\n features = tf.parse_single_example(\n serialized_example, features={\n 'label': tf.FixedLenFeature([], tf.int64),\n 'img_raw': tf.FixedLenFeature([], tf.string),\n }\n )\n # You can do more image distortion here for training data\n img = tf.decode_raw(features['img_raw'], tf.float32)\n img = tf.reshape(img, [32, 32, 3])\n # img = tf.cast(img, tf.float32) #* (1. / 255) - 0.5\n if is_train ==True:\n # 1. Randomly crop a [height, width] section of the image.\n img = tf.random_crop(img, [24, 24, 3])\n\n # 2. Randomly flip the image horizontally.\n img = tf.image.random_flip_left_right(img)\n\n # 3. Randomly change brightness.\n img = tf.image.random_brightness(img, max_delta=63)\n\n # 4. Randomly change contrast.\n img = tf.image.random_contrast(img, lower=0.2, upper=1.8)\n\n # 5. Subtract off the mean and divide by the variance of the pixels.\n img = tf.image.per_image_standardization(img)\n\n elif is_train == False:\n # 1. Crop the central [height, width] of the image.\n img = tf.image.resize_image_with_crop_or_pad(img, 24, 24)\n\n # 2. Subtract off the mean and divide by the variance of the pixels.\n img = tf.image.per_image_standardization(img)\n\n elif is_train == None:\n img = img\n\n label = tf.cast(features['label'], tf.int32)\n return img, label"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprint all info of parameters in the network", "response": "def print_params(self, details=True, session=None):\n \"\"\"Print all info of parameters in the network\"\"\"\n for i, p in enumerate(self.all_params):\n if details:\n try:\n val = p.eval(session=session)\n logging.info(\n \" param {:3}: {:20} {:15} {} (mean: {:<18}, median: {:<18}, std: {:<18}) \".\n format(i, p.name, str(val.shape), p.dtype.name, val.mean(), np.median(val), val.std())\n )\n except Exception as e:\n logging.info(str(e))\n raise Exception(\n \"Hint: print params details after tl.layers.initialize_global_variables(sess) \"\n \"or use network.print_params(False).\"\n )\n else:\n logging.info(\" param {:3}: {:20} {:15} {}\".format(i, p.name, str(p.get_shape()), p.dtype.name))\n logging.info(\" num of params: %d\" % self.count_params())"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef print_layers(self):\n for i, layer in enumerate(self.all_layers):\n # logging.info(\" layer %d: %s\" % (i, str(layer)))\n logging.info(\n \" layer {:3}: {:20} {:15} {}\".format(i, layer.name, str(layer.get_shape()), layer.dtype.name)\n )", "response": "Print all info of layers in the network."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef count_params(self):\n n_params = 0\n for _i, p in enumerate(self.all_params):\n n = 1\n # for s in p.eval().shape:\n for s in p.get_shape():\n try:\n s = int(s)\n except Exception:\n s = 1\n if s:\n n = n * s\n n_params = n_params + n\n return n_params", "response": "Returns the number of parameters in the network."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_all_params(self, session=None):\n _params = []\n for p in self.all_params:\n if session is None:\n _params.append(p.eval())\n else:\n _params.append(session.run(p))\n return _params", "response": "Return the parameters in a list of array."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting all arguments of current layer for saving the graph.", "response": "def _get_init_args(self, skip=4):\n \"\"\"Get all arguments of current layer for saving the graph.\"\"\"\n stack = inspect.stack()\n\n if len(stack) < skip + 1:\n raise ValueError(\"The length of the inspection stack is shorter than the requested start position.\")\n\n args, _, _, values = inspect.getargvalues(stack[skip][0])\n\n params = {}\n\n for arg in args:\n\n # some args dont need to be saved into the graph. e.g. the input placeholder\n if values[arg] is not None and arg not in ['self', 'prev_layer', 'inputs']:\n\n val = values[arg]\n\n # change function (e.g. act) into dictionary of module path and function name\n if inspect.isfunction(val):\n params[arg] = {\"module_path\": val.__module__, \"func_name\": val.__name__}\n # ignore more args e.g. TF class\n elif arg.endswith('init'):\n continue\n # for other data type, save them directly\n else:\n params[arg] = val\n\n return params"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a tensorflow operation for computing the Region of Interest Pooling @arg input: feature maps on which to perform the pooling operation @arg rois: list of regions of interest in the format (feature map index, upper left, bottom right) @arg pool_width: size of the pooling sections", "response": "def roi_pooling(input, rois, pool_height, pool_width):\n \"\"\"\n returns a tensorflow operation for computing the Region of Interest Pooling\n \n @arg input: feature maps on which to perform the pooling operation\n @arg rois: list of regions of interest in the format (feature map index, upper left, bottom right)\n @arg pool_width: size of the pooling sections\n \"\"\"\n # TODO(maciek): ops scope\n out = roi_pooling_module.roi_pooling(input, rois, pool_height=pool_height, pool_width=pool_width)\n output, argmax_output = out[0], out[1]\n return output"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrap for inserting an int64 Feature into a SequenceExample proto", "response": "def _int64_feature(value):\n \"\"\"Wrapper for inserting an int64 Feature into a SequenceExample proto,\n e.g, An integer label.\n \"\"\"\n return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _bytes_feature(value):\n # return tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(value)]))\n return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))", "response": "Wrapper for inserting a bytes Feature into a SequenceExample proto"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrap for inserting an int64 FeatureList into a SequenceExample proto", "response": "def _int64_feature_list(values):\n \"\"\"Wrapper for inserting an int64 FeatureList into a SequenceExample proto,\n e.g, sentence in list of ints\n \"\"\"\n return tf.train.FeatureList(feature=[_int64_feature(v) for v in values])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _bytes_feature_list(values):\n return tf.train.FeatureList(feature=[_bytes_feature(v) for v in values])", "response": "Wrapper for inserting a bytes FeatureList into a SequenceExample proto"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms random distortions on an image.", "response": "def distort_image(image, thread_id):\n \"\"\"Perform random distortions on an image.\n Args:\n image: A float32 Tensor of shape [height, width, 3] with values in [0, 1).\n thread_id: Preprocessing thread id used to select the ordering of color\n distortions. There should be a multiple of 2 preprocessing threads.\n Returns:````\n distorted_image: A float32 Tensor of shape [height, width, 3] with values in\n [0, 1].\n \"\"\"\n # Randomly flip horizontally.\n with tf.name_scope(\"flip_horizontal\"): # , values=[image]): # DH MOdify\n # with tf.name_scope(\"flip_horizontal\", values=[image]):\n image = tf.image.random_flip_left_right(image)\n # Randomly distort the colors based on thread id.\n color_ordering = thread_id % 2\n with tf.name_scope(\"distort_color\"): # , values=[image]): # DH MOdify\n # with tf.name_scope(\"distort_color\", values=[image]): # DH MOdify\n if color_ordering == 0:\n image = tf.image.random_brightness(image, max_delta=32. / 255.)\n image = tf.image.random_saturation(image, lower=0.5, upper=1.5)\n image = tf.image.random_hue(image, max_delta=0.032)\n image = tf.image.random_contrast(image, lower=0.5, upper=1.5)\n elif color_ordering == 1:\n image = tf.image.random_brightness(image, max_delta=32. / 255.)\n image = tf.image.random_contrast(image, lower=0.5, upper=1.5)\n image = tf.image.random_saturation(image, lower=0.5, upper=1.5)\n image = tf.image.random_hue(image, max_delta=0.032)\n # The random_* ops do not necessarily clamp.\n image = tf.clip_by_value(image, 0.0, 1.0)\n\n return image"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef prefetch_input_data(\n reader, file_pattern, is_training, batch_size, values_per_shard, input_queue_capacity_factor=16,\n num_reader_threads=1, shard_queue_name=\"filename_queue\", value_queue_name=\"input_queue\"\n):\n \"\"\"Prefetches string values from disk into an input queue.\n\n In training the capacity of the queue is important because a larger queue\n means better mixing of training examples between shards. The minimum number of\n values kept in the queue is values_per_shard * input_queue_capacity_factor,\n where input_queue_memory factor should be chosen to trade-off better mixing\n with memory usage.\n\n Args:\n reader: Instance of tf.ReaderBase.\n file_pattern: Comma-separated list of file patterns (e.g.\n /tmp/train_data-?????-of-00100).\n is_training: Boolean; whether prefetching for training or eval.\n batch_size: Model batch size used to determine queue capacity.\n values_per_shard: Approximate number of values per shard.\n input_queue_capacity_factor: Minimum number of values to keep in the queue\n in multiples of values_per_shard. See comments above.\n num_reader_threads: Number of reader threads to fill the queue.\n shard_queue_name: Name for the shards filename queue.\n value_queue_name: Name for the values input queue.\n\n Returns:\n A Queue containing prefetched string values.\n \"\"\"\n data_files = []\n for pattern in file_pattern.split(\",\"):\n data_files.extend(tf.gfile.Glob(pattern))\n if not data_files:\n tl.logging.fatal(\"Found no input files matching %s\", file_pattern)\n else:\n tl.logging.info(\"Prefetching values from %d files matching %s\", len(data_files), file_pattern)\n\n if is_training:\n print(\" is_training == True : RandomShuffleQueue\")\n filename_queue = tf.train.string_input_producer(data_files, shuffle=True, capacity=16, name=shard_queue_name)\n min_queue_examples = values_per_shard * input_queue_capacity_factor\n capacity = min_queue_examples + 100 * batch_size\n values_queue = tf.RandomShuffleQueue(\n capacity=capacity, min_after_dequeue=min_queue_examples, dtypes=[tf.string],\n name=\"random_\" + value_queue_name\n )\n else:\n print(\" is_training == False : FIFOQueue\")\n filename_queue = tf.train.string_input_producer(data_files, shuffle=False, capacity=1, name=shard_queue_name)\n capacity = values_per_shard + 3 * batch_size\n values_queue = tf.FIFOQueue(capacity=capacity, dtypes=[tf.string], name=\"fifo_\" + value_queue_name)\n\n enqueue_ops = []\n for _ in range(num_reader_threads):\n _, value = reader.read(filename_queue)\n enqueue_ops.append(values_queue.enqueue([value]))\n tf.train.queue_runner.add_queue_runner(tf.train.queue_runner.QueueRunner(values_queue, enqueue_ops))\n\n tf.summary.scalar(\n \"queue/%s/fraction_of_%d_full\" % (values_queue.name, capacity),\n tf.cast(values_queue.size(), tf.float32) * (1. / capacity)\n )\n\n return values_queue", "response": "Prefetches string values from disk into an input queue."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbatches input images and captions. This function splits the caption into an input sequence and a target sequence, where the target sequence is the input sequence right-shifted by 1. Input and target sequences are batched and padded up to the maximum length of sequences in the batch. A mask is created to distinguish real words from padding words. Example: Actual captions in the batch ('-' denotes padded character): [ [ 1 2 5 4 5 ], [ 1 2 3 4 - ], [ 1 2 3 - - ], ] input_seqs: [ [ 1 2 3 4 ], [ 1 2 3 - ], [ 1 2 - - ], ] target_seqs: [ [ 2 3 4 5 ], [ 2 3 4 - ], [ 2 3 - - ], ] mask: [ [ 1 1 1 1 ], [ 1 1 1 0 ], [ 1 1 0 0 ], ] Args: images_and_captions: A list of pairs [image, caption], where image is a Tensor of shape [height, width, channels] and caption is a 1-D Tensor of any length. Each pair will be processed and added to the queue in a separate thread. batch_size: Batch size. queue_capacity: Queue capacity. add_summaries: If true, add caption length summaries. Returns: images: A Tensor of shape [batch_size, height, width, channels]. input_seqs: An int32 Tensor of shape [batch_size, padded_length]. target_seqs: An int32 Tensor of shape [batch_size, padded_length]. mask: An int32 0/1 Tensor of shape [batch_size, padded_length].", "response": "def batch_with_dynamic_pad(images_and_captions, batch_size, queue_capacity, add_summaries=True):\n \"\"\"Batches input images and captions.\n\n This function splits the caption into an input sequence and a target sequence,\n where the target sequence is the input sequence right-shifted by 1. Input and\n target sequences are batched and padded up to the maximum length of sequences\n in the batch. A mask is created to distinguish real words from padding words.\n\n Example:\n Actual captions in the batch ('-' denotes padded character):\n [\n [ 1 2 5 4 5 ],\n [ 1 2 3 4 - ],\n [ 1 2 3 - - ],\n ]\n\n input_seqs:\n [\n [ 1 2 3 4 ],\n [ 1 2 3 - ],\n [ 1 2 - - ],\n ]\n\n target_seqs:\n [\n [ 2 3 4 5 ],\n [ 2 3 4 - ],\n [ 2 3 - - ],\n ]\n\n mask:\n [\n [ 1 1 1 1 ],\n [ 1 1 1 0 ],\n [ 1 1 0 0 ],\n ]\n\n Args:\n images_and_captions: A list of pairs [image, caption], where image is a\n Tensor of shape [height, width, channels] and caption is a 1-D Tensor of\n any length. Each pair will be processed and added to the queue in a\n separate thread.\n batch_size: Batch size.\n queue_capacity: Queue capacity.\n add_summaries: If true, add caption length summaries.\n\n Returns:\n images: A Tensor of shape [batch_size, height, width, channels].\n input_seqs: An int32 Tensor of shape [batch_size, padded_length].\n target_seqs: An int32 Tensor of shape [batch_size, padded_length].\n mask: An int32 0/1 Tensor of shape [batch_size, padded_length].\n \"\"\"\n enqueue_list = []\n for image, caption in images_and_captions:\n caption_length = tf.shape(caption)[0]\n input_length = tf.expand_dims(tf.subtract(caption_length, 1), 0)\n\n input_seq = tf.slice(caption, [0], input_length)\n target_seq = tf.slice(caption, [1], input_length)\n indicator = tf.ones(input_length, dtype=tf.int32)\n enqueue_list.append([image, input_seq, target_seq, indicator])\n\n images, input_seqs, target_seqs, mask = tf.train.batch_join(\n enqueue_list, batch_size=batch_size, capacity=queue_capacity, dynamic_pad=True, name=\"batch_and_pad\"\n )\n\n if add_summaries:\n lengths = tf.add(tf.reduce_sum(mask, 1), 1)\n tf.summary.scalar(\"caption_length/batch_min\", tf.reduce_min(lengths))\n tf.summary.scalar(\"caption_length/batch_max\", tf.reduce_max(lengths))\n tf.summary.scalar(\"caption_length/batch_mean\", tf.reduce_mean(lengths))\n\n return images, input_seqs, target_seqs, mask"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _bias_scale(x, b, data_format):\n if data_format == 'NHWC':\n return x * b\n elif data_format == 'NCHW':\n return x * _to_channel_first_bias(b)\n else:\n raise ValueError('invalid data_format: %s' % data_format)", "response": "The multiplication counter part of tf. nn. bias_add."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _bias_add(x, b, data_format):\n if data_format == 'NHWC':\n return tf.add(x, b)\n elif data_format == 'NCHW':\n return tf.add(x, _to_channel_first_bias(b))\n else:\n raise ValueError('invalid data_format: %s' % data_format)", "response": "Alternative implementation of tf. nn. bias_add which is compatiable with tensorRT."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef batch_normalization(x, mean, variance, offset, scale, variance_epsilon, data_format, name=None):\n with ops.name_scope(name, 'batchnorm', [x, mean, variance, scale, offset]):\n inv = math_ops.rsqrt(variance + variance_epsilon)\n if scale is not None:\n inv *= scale\n\n a = math_ops.cast(inv, x.dtype)\n b = math_ops.cast(offset - mean * inv if offset is not None else -mean * inv, x.dtype)\n\n # Return a * x + b with customized data_format.\n # Currently TF doesn't have bias_scale, and tensorRT has bug in converting tf.nn.bias_add\n # So we reimplemted them to allow make the model work with tensorRT.\n # See https://github.com/tensorlayer/openpose-plus/issues/75 for more details.\n df = {'channels_first': 'NCHW', 'channels_last': 'NHWC'}\n return _bias_add(_bias_scale(x, a, df[data_format]), b, df[data_format])", "response": "Data Format aware version of tf. nn. batch_normalization."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompute the scale parameter.", "response": "def compute_alpha(x):\n \"\"\"Computing the scale parameter.\"\"\"\n threshold = _compute_threshold(x)\n alpha1_temp1 = tf.where(tf.greater(x, threshold), x, tf.zeros_like(x, tf.float32))\n alpha1_temp2 = tf.where(tf.less(x, -threshold), x, tf.zeros_like(x, tf.float32))\n alpha_array = tf.add(alpha1_temp1, alpha1_temp2, name=None)\n alpha_array_abs = tf.abs(alpha_array)\n alpha_array_abs1 = tf.where(\n tf.greater(alpha_array_abs, 0), tf.ones_like(alpha_array_abs, tf.float32),\n tf.zeros_like(alpha_array_abs, tf.float32)\n )\n alpha_sum = tf.reduce_sum(alpha_array_abs)\n n = tf.reduce_sum(alpha_array_abs1)\n alpha = tf.div(alpha_sum, n)\n return alpha"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets a list of layers output that contain this name scope.", "response": "def get_layers_with_name(net, name=\"\", verbose=False):\n \"\"\"Get a list of layers' output in a network by a given name scope.\n\n Parameters\n -----------\n net : :class:`Layer`\n The last layer of the network.\n name : str\n Get the layers' output that contain this name.\n verbose : boolean\n If True, print information of all the layers' output\n\n Returns\n --------\n list of Tensor\n A list of layers' output (TensorFlow tensor)\n\n Examples\n ---------\n >>> import tensorlayer as tl\n >>> layers = tl.layers.get_layers_with_name(net, \"CNN\", True)\n\n \"\"\"\n logging.info(\" [*] geting layers with %s\" % name)\n\n layers = []\n i = 0\n\n for layer in net.all_layers:\n # logging.info(type(layer.name))\n if name in layer.name:\n layers.append(layer)\n\n if verbose:\n logging.info(\" got {:3}: {:15} {}\".format(i, layer.name, str(layer.get_shape())))\n i = i + 1\n\n return layers"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets a list of TensorFlow variables that contain a given name scope.", "response": "def get_variables_with_name(name=None, train_only=True, verbose=False):\n \"\"\"Get a list of TensorFlow variables by a given name scope.\n\n Parameters\n ----------\n name : str\n Get the variables that contain this name.\n train_only : boolean\n If Ture, only get the trainable variables.\n verbose : boolean\n If True, print the information of all variables.\n\n Returns\n -------\n list of Tensor\n A list of TensorFlow variables\n\n Examples\n --------\n >>> import tensorlayer as tl\n >>> dense_vars = tl.layers.get_variables_with_name('dense', True, True)\n\n \"\"\"\n if name is None:\n raise Exception(\"please input a name\")\n\n logging.info(\" [*] geting variables with %s\" % name)\n\n # tvar = tf.trainable_variables() if train_only else tf.all_variables()\n if train_only:\n t_vars = tf.trainable_variables()\n\n else:\n t_vars = tf.global_variables()\n\n d_vars = [var for var in t_vars if name in var.name]\n\n if verbose:\n for idx, v in enumerate(d_vars):\n logging.info(\" got {:3}: {:15} {}\".format(idx, v.name, str(v.get_shape())))\n\n return d_vars"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the initialized RNN state.", "response": "def initialize_rnn_state(state, feed_dict=None):\n \"\"\"Returns the initialized RNN state.\n The inputs are `LSTMStateTuple` or `State` of `RNNCells`, and an optional `feed_dict`.\n\n Parameters\n ----------\n state : RNN state.\n The TensorFlow's RNN state.\n feed_dict : dictionary\n Initial RNN state; if None, returns zero state.\n\n Returns\n -------\n RNN state\n The TensorFlow's RNN state.\n\n \"\"\"\n if isinstance(state, LSTMStateTuple):\n c = state.c.eval(feed_dict=feed_dict)\n h = state.h.eval(feed_dict=feed_dict)\n return c, h\n else:\n new_state = state.eval(feed_dict=feed_dict)\n return new_state"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nremoves repeated items in a list and return the processed list.", "response": "def list_remove_repeat(x):\n \"\"\"Remove the repeated items in a list, and return the processed list.\n You may need it to create merged layer like Concat, Elementwise and etc.\n\n Parameters\n ----------\n x : list\n Input\n\n Returns\n -------\n list\n A list that after removing it's repeated items\n\n Examples\n -------\n >>> l = [2, 3, 4, 2, 3]\n >>> l = list_remove_repeat(l)\n [2, 3, 4]\n\n \"\"\"\n y = []\n for i in x:\n if i not in y:\n y.append(i)\n\n return y"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nmerging all parameters layers and dropout probabilities to a single node.", "response": "def merge_networks(layers=None):\n \"\"\"Merge all parameters, layers and dropout probabilities to a :class:`Layer`.\n The output of return network is the first network in the list.\n\n Parameters\n ----------\n layers : list of :class:`Layer`\n Merge all parameters, layers and dropout probabilities to the first layer in the list.\n\n Returns\n --------\n :class:`Layer`\n The network after merging all parameters, layers and dropout probabilities to the first network in the list.\n\n Examples\n ---------\n >>> import tensorlayer as tl\n >>> n1 = ...\n >>> n2 = ...\n >>> n1 = tl.layers.merge_networks([n1, n2])\n\n \"\"\"\n if layers is None:\n raise Exception(\"layers should be a list of TensorLayer's Layers.\")\n layer = layers[0]\n\n all_params = []\n all_layers = []\n all_drop = {}\n\n for l in layers:\n all_params.extend(l.all_params)\n all_layers.extend(l.all_layers)\n all_drop.update(l.all_drop)\n\n layer.all_params = list(all_params)\n layer.all_layers = list(all_layers)\n layer.all_drop = dict(all_drop)\n\n layer.all_layers = list_remove_repeat(layer.all_layers)\n layer.all_params = list_remove_repeat(layer.all_params)\n\n return layer"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef print_all_variables(train_only=False):\n # tvar = tf.trainable_variables() if train_only else tf.all_variables()\n if train_only:\n t_vars = tf.trainable_variables()\n logging.info(\" [*] printing trainable variables\")\n\n else:\n t_vars = tf.global_variables()\n logging.info(\" [*] printing global variables\")\n\n for idx, v in enumerate(t_vars):\n logging.info(\" var {:3}: {:15} {}\".format(idx, str(v.get_shape()), v.name))", "response": "Print information of all variables."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ternary_operation(x):\n g = tf.get_default_graph()\n with g.gradient_override_map({\"Sign\": \"Identity\"}):\n threshold = _compute_threshold(x)\n x = tf.sign(tf.add(tf.sign(tf.add(x, threshold)), tf.sign(tf.add(x, -threshold))))\n return x", "response": "Ternary operation use threshold computed with weights."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _compute_threshold(x):\n x_sum = tf.reduce_sum(tf.abs(x), reduction_indices=None, keepdims=False, name=None)\n threshold = tf.div(x_sum, tf.cast(tf.size(x), tf.float32), name=None)\n threshold = tf.multiply(0.7, threshold, name=None)\n return threshold", "response": "ref: https://github.com/XJTUWYD/TWN\n Computing the threshold."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef freeze_graph(graph_path, checkpoint_path, output_path, end_node_names, is_binary_graph):\n _freeze_graph(\n input_graph=graph_path, input_saver='', input_binary=is_binary_graph, input_checkpoint=checkpoint_path,\n output_graph=output_path, output_node_names=end_node_names, restore_op_name='save/restore_all',\n filename_tensor_name='save/Const:0', clear_devices=True, initializer_nodes=None\n )", "response": "Reimplementation of the TensorFlow official freeze_graph function to freeze the graph and checkpoint together."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef convert_model_to_onnx(frozen_graph_path, end_node_names, onnx_output_path):\n with tf.gfile.GFile(frozen_graph_path, \"rb\") as f:\n graph_def = tf.GraphDef()\n graph_def.ParseFromString(f.read())\n onnx_model = tensorflow_graph_to_onnx_model(graph_def, end_node_names, opset=6)\n file = open(onnx_output_path, \"wb\")\n file.write(onnx_model.SerializeToString())\n file.close()", "response": "Reimplementation of the TensorFlow - onnx official tutorial convert the protobuf file to onnx file"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _add_deprecated_function_notice_to_docstring(doc, date, instructions):\n\n if instructions:\n deprecation_message = \"\"\"\n .. warning::\n **THIS FUNCTION IS DEPRECATED:** It will be removed after %s.\n *Instructions for updating:* %s.\n \"\"\" % (('in a future version' if date is None else ('after %s' % date)), instructions)\n\n else:\n deprecation_message = \"\"\"\n .. warning::\n **THIS FUNCTION IS DEPRECATED:** It will be removed after %s.\n \"\"\" % (('in a future version' if date is None else ('after %s' % date)))\n\n main_text = [deprecation_message]\n\n return _add_notice_to_docstring(doc, 'DEPRECATED FUNCTION', main_text)", "response": "Adds a deprecation notice to a docstring for deprecated functions."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _add_notice_to_docstring(doc, no_doc_str, notice):\n if not doc:\n lines = [no_doc_str]\n\n else:\n lines = _normalize_docstring(doc).splitlines()\n\n notice = [''] + notice\n\n if len(lines) > 1:\n # Make sure that we keep our distance from the main body\n if lines[1].strip():\n notice.append('')\n\n lines[1:1] = notice\n else:\n lines += notice\n\n return '\\n'.join(lines)", "response": "Adds a deprecation notice to a docstring."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a tensor with all elements set to alpha_value.", "response": "def alphas(shape, alpha_value, name=None):\n \"\"\"Creates a tensor with all elements set to `alpha_value`.\n This operation returns a tensor of type `dtype` with shape `shape` and all\n elements set to alpha.\n\n Parameters\n ----------\n shape: A list of integers, a tuple of integers, or a 1-D `Tensor` of type `int32`.\n The shape of the desired tensor\n alpha_value: `float32`, `float64`, `int8`, `uint8`, `int16`, `uint16`, int32`, `int64`\n The value used to fill the resulting `Tensor`.\n name: str\n A name for the operation (optional).\n\n Returns\n -------\n A `Tensor` with all elements set to alpha.\n\n Examples\n --------\n >>> tl.alphas([2, 3], tf.int32) # [[alpha, alpha, alpha], [alpha, alpha, alpha]]\n \"\"\"\n with ops.name_scope(name, \"alphas\", [shape]) as name:\n\n alpha_tensor = convert_to_tensor(alpha_value)\n alpha_dtype = dtypes.as_dtype(alpha_tensor.dtype).base_dtype\n\n if not isinstance(shape, ops.Tensor):\n try:\n shape = constant_op._tensor_shape_tensor_conversion_function(tensor_shape.TensorShape(shape))\n except (TypeError, ValueError):\n shape = ops.convert_to_tensor(shape, dtype=dtypes.int32)\n\n if not shape._shape_tuple():\n shape = reshape(shape, [-1]) # Ensure it's a vector\n\n try:\n output = constant(alpha_value, shape=shape, dtype=alpha_dtype, name=name)\n\n except (TypeError, ValueError):\n output = fill(shape, constant(alpha_value, dtype=alpha_dtype), name=name)\n\n if output.dtype.base_dtype != alpha_dtype:\n raise AssertionError(\"Dtypes do not corresponds: %s and %s\" % (output.dtype.base_dtype, alpha_dtype))\n\n return output"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef alphas_like(tensor, alpha_value, name=None, optimize=True):\n with ops.name_scope(name, \"alphas_like\", [tensor]) as name:\n tensor = ops.convert_to_tensor(tensor, name=\"tensor\")\n\n if context.in_eager_mode(): # and dtype is not None and dtype != tensor.dtype:\n ret = alphas(shape_internal(tensor, optimize=optimize), alpha_value=alpha_value, name=name)\n\n else: # if context.in_graph_mode():\n\n # For now, variant types must be created via zeros_like; as we need to\n # pass the input variant object to the proper zeros callback.\n\n if (optimize and tensor.shape.is_fully_defined()):\n # We can produce a zeros tensor independent of the value of 'tensor',\n # since the shape is known statically.\n ret = alphas(tensor.shape, alpha_value=alpha_value, name=name)\n\n # elif dtype is not None and dtype != tensor.dtype and dtype != dtypes.variant:\n else:\n ret = alphas(shape_internal(tensor, optimize=optimize), alpha_value=alpha_value, name=name)\n\n ret.set_shape(tensor.get_shape())\n\n return ret", "response": "Creates a tensor with all elements set to alpha_value. Given a single tensor this operation returns a tensor with all elements set to alpha_value."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef example2():\n st = time.time()\n for _ in range(100): # Repeat 100 times and compute the averaged speed\n transform_matrix = create_transformation_matrix()\n result = tl.prepro.affine_transform_cv2(image, transform_matrix) # Transform the image using a single operation\n print(\"apply all transforms once took %fs for each image\" % ((time.time() - st) / 100)) # usually 50x faster\n tl.vis.save_image(result, '_result_fast.png')", "response": "This example is very fast!"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef example4():\n transform_matrix = create_transformation_matrix()\n result = tl.prepro.affine_transform_cv2(image, transform_matrix) # 76 times faster\n # Transform keypoint coordinates\n coords = [[(50, 100), (100, 100), (100, 50), (200, 200)], [(250, 50), (200, 50), (200, 100)]]\n coords_result = tl.prepro.affine_transform_keypoints(coords, transform_matrix)\n\n def imwrite(image, coords_list, name):\n coords_list_ = []\n for coords in coords_list:\n coords = np.array(coords, np.int32)\n coords = coords.reshape((-1, 1, 2))\n coords_list_.append(coords)\n image = cv2.polylines(image, coords_list_, True, (0, 255, 255), 3)\n cv2.imwrite(name, image[..., ::-1])\n\n imwrite(image, coords, '_with_keypoints_origin.png')\n imwrite(result, coords_result, '_with_keypoints_result.png')", "response": "Example 4: Transforming coordinates using affine matrix."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndistort the image x", "response": "def distort_fn(x, is_train=False):\n \"\"\"\n The images are processed as follows:\n .. They are cropped to 24 x 24 pixels, centrally for evaluation or randomly for training.\n .. They are approximately whitened to make the model insensitive to dynamic range.\n For training, we additionally apply a series of random distortions to\n artificially increase the data set size:\n .. Randomly flip the image from left to right.\n .. Randomly distort the image brightness.\n \"\"\"\n # print('begin',x.shape, np.min(x), np.max(x))\n x = tl.prepro.crop(x, 24, 24, is_random=is_train)\n # print('after crop',x.shape, np.min(x), np.max(x))\n if is_train:\n # x = tl.prepro.zoom(x, zoom_range=(0.9, 1.0), is_random=True)\n # print('after zoom', x.shape, np.min(x), np.max(x))\n x = tl.prepro.flip_axis(x, axis=1, is_random=True)\n # print('after flip',x.shape, np.min(x), np.max(x))\n x = tl.prepro.brightness(x, gamma=0.1, gain=1, is_random=True)\n # print('after brightness',x.shape, np.min(x), np.max(x))\n # tmp = np.max(x)\n # x += np.random.uniform(-20, 20)\n # x /= tmp\n # normalize the image\n x = (x - np.mean(x)) / max(np.std(x), 1e-5) # avoid values divided by 0\n # print('after norm', x.shape, np.min(x), np.max(x), np.mean(x))\n return x"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntrains a given non - time - series network by the given cost function training data and validation data.", "response": "def fit(\n sess, network, train_op, cost, X_train, y_train, x, y_, acc=None, batch_size=100, n_epoch=100, print_freq=5,\n X_val=None, y_val=None, eval_train=True, tensorboard_dir=None, tensorboard_epoch_freq=5,\n tensorboard_weight_histograms=True, tensorboard_graph_vis=True\n):\n \"\"\"Training a given non time-series network by the given cost function, training data, batch_size, n_epoch etc.\n\n - MNIST example click `here `_.\n - In order to control the training details, the authors HIGHLY recommend ``tl.iterate`` see two MNIST examples `1 `_, `2 `_.\n\n Parameters\n ----------\n sess : Session\n TensorFlow Session.\n network : TensorLayer layer\n the network to be trained.\n train_op : TensorFlow optimizer\n The optimizer for training e.g. tf.train.AdamOptimizer.\n X_train : numpy.array\n The input of training data\n y_train : numpy.array\n The target of training data\n x : placeholder\n For inputs.\n y_ : placeholder\n For targets.\n acc : TensorFlow expression or None\n Metric for accuracy or others. If None, would not print the information.\n batch_size : int\n The batch size for training and evaluating.\n n_epoch : int\n The number of training epochs.\n print_freq : int\n Print the training information every ``print_freq`` epochs.\n X_val : numpy.array or None\n The input of validation data. If None, would not perform validation.\n y_val : numpy.array or None\n The target of validation data. If None, would not perform validation.\n eval_train : boolean\n Whether to evaluate the model during training.\n If X_val and y_val are not None, it reflects whether to evaluate the model on training data.\n tensorboard_dir : string\n path to log dir, if set, summary data will be stored to the tensorboard_dir/ directory for visualization with tensorboard. (default None)\n Also runs `tl.layers.initialize_global_variables(sess)` internally in fit() to setup the summary nodes.\n tensorboard_epoch_freq : int\n How many epochs between storing tensorboard checkpoint for visualization to log/ directory (default 5).\n tensorboard_weight_histograms : boolean\n If True updates tensorboard data in the logs/ directory for visualization\n of the weight histograms every tensorboard_epoch_freq epoch (default True).\n tensorboard_graph_vis : boolean\n If True stores the graph in the tensorboard summaries saved to log/ (default True).\n\n Examples\n --------\n See `tutorial_mnist_simple.py `_\n\n >>> tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_,\n ... acc=acc, batch_size=500, n_epoch=200, print_freq=5,\n ... X_val=X_val, y_val=y_val, eval_train=False)\n >>> tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_,\n ... acc=acc, batch_size=500, n_epoch=200, print_freq=5,\n ... X_val=X_val, y_val=y_val, eval_train=False,\n ... tensorboard=True, tensorboard_weight_histograms=True, tensorboard_graph_vis=True)\n\n Notes\n --------\n If tensorboard_dir not None, the `global_variables_initializer` will be run inside the fit function\n in order to initialize the automatically generated summary nodes used for tensorboard visualization,\n thus `tf.global_variables_initializer().run()` before the `fit()` call will be undefined.\n\n \"\"\"\n if X_train.shape[0] < batch_size:\n raise AssertionError(\"Number of training examples should be bigger than the batch size\")\n\n if tensorboard_dir is not None:\n tl.logging.info(\"Setting up tensorboard ...\")\n #Set up tensorboard summaries and saver\n tl.files.exists_or_mkdir(tensorboard_dir)\n\n #Only write summaries for more recent TensorFlow versions\n if hasattr(tf, 'summary') and hasattr(tf.summary, 'FileWriter'):\n if tensorboard_graph_vis:\n train_writer = tf.summary.FileWriter(tensorboard_dir + '/train', sess.graph)\n val_writer = tf.summary.FileWriter(tensorboard_dir + '/validation', sess.graph)\n else:\n train_writer = tf.summary.FileWriter(tensorboard_dir + '/train')\n val_writer = tf.summary.FileWriter(tensorboard_dir + '/validation')\n\n #Set up summary nodes\n if (tensorboard_weight_histograms):\n for param in network.all_params:\n if hasattr(tf, 'summary') and hasattr(tf.summary, 'histogram'):\n tl.logging.info('Param name %s' % param.name)\n tf.summary.histogram(param.name, param)\n\n if hasattr(tf, 'summary') and hasattr(tf.summary, 'histogram'):\n tf.summary.scalar('cost', cost)\n\n merged = tf.summary.merge_all()\n\n #Initalize all variables and summaries\n tl.layers.initialize_global_variables(sess)\n tl.logging.info(\"Finished! use `tensorboard --logdir=%s/` to start tensorboard\" % tensorboard_dir)\n\n tl.logging.info(\"Start training the network ...\")\n start_time_begin = time.time()\n tensorboard_train_index, tensorboard_val_index = 0, 0\n for epoch in range(n_epoch):\n start_time = time.time()\n loss_ep = 0\n n_step = 0\n for X_train_a, y_train_a in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):\n feed_dict = {x: X_train_a, y_: y_train_a}\n feed_dict.update(network.all_drop) # enable noise layers\n loss, _ = sess.run([cost, train_op], feed_dict=feed_dict)\n loss_ep += loss\n n_step += 1\n loss_ep = loss_ep / n_step\n\n if tensorboard_dir is not None and hasattr(tf, 'summary'):\n if epoch + 1 == 1 or (epoch + 1) % tensorboard_epoch_freq == 0:\n for X_train_a, y_train_a in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):\n dp_dict = dict_to_one(network.all_drop) # disable noise layers\n feed_dict = {x: X_train_a, y_: y_train_a}\n feed_dict.update(dp_dict)\n result = sess.run(merged, feed_dict=feed_dict)\n train_writer.add_summary(result, tensorboard_train_index)\n tensorboard_train_index += 1\n if (X_val is not None) and (y_val is not None):\n for X_val_a, y_val_a in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=True):\n dp_dict = dict_to_one(network.all_drop) # disable noise layers\n feed_dict = {x: X_val_a, y_: y_val_a}\n feed_dict.update(dp_dict)\n result = sess.run(merged, feed_dict=feed_dict)\n val_writer.add_summary(result, tensorboard_val_index)\n tensorboard_val_index += 1\n\n if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:\n if (X_val is not None) and (y_val is not None):\n tl.logging.info(\"Epoch %d of %d took %fs\" % (epoch + 1, n_epoch, time.time() - start_time))\n if eval_train is True:\n train_loss, train_acc, n_batch = 0, 0, 0\n for X_train_a, y_train_a in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):\n dp_dict = dict_to_one(network.all_drop) # disable noise layers\n feed_dict = {x: X_train_a, y_: y_train_a}\n feed_dict.update(dp_dict)\n if acc is not None:\n err, ac = sess.run([cost, acc], feed_dict=feed_dict)\n train_acc += ac\n else:\n err = sess.run(cost, feed_dict=feed_dict)\n train_loss += err\n n_batch += 1\n tl.logging.info(\" train loss: %f\" % (train_loss / n_batch))\n if acc is not None:\n tl.logging.info(\" train acc: %f\" % (train_acc / n_batch))\n val_loss, val_acc, n_batch = 0, 0, 0\n for X_val_a, y_val_a in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=True):\n dp_dict = dict_to_one(network.all_drop) # disable noise layers\n feed_dict = {x: X_val_a, y_: y_val_a}\n feed_dict.update(dp_dict)\n if acc is not None:\n err, ac = sess.run([cost, acc], feed_dict=feed_dict)\n val_acc += ac\n else:\n err = sess.run(cost, feed_dict=feed_dict)\n val_loss += err\n n_batch += 1\n\n tl.logging.info(\" val loss: %f\" % (val_loss / n_batch))\n\n if acc is not None:\n tl.logging.info(\" val acc: %f\" % (val_acc / n_batch))\n else:\n tl.logging.info(\n \"Epoch %d of %d took %fs, loss %f\" % (epoch + 1, n_epoch, time.time() - start_time, loss_ep)\n )\n tl.logging.info(\"Total training time: %fs\" % (time.time() - start_time_begin))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict(sess, network, X, x, y_op, batch_size=None):\n if batch_size is None:\n dp_dict = dict_to_one(network.all_drop) # disable noise layers\n feed_dict = {\n x: X,\n }\n feed_dict.update(dp_dict)\n return sess.run(y_op, feed_dict=feed_dict)\n else:\n result = None\n for X_a, _ in tl.iterate.minibatches(X, X, batch_size, shuffle=False):\n dp_dict = dict_to_one(network.all_drop)\n feed_dict = {\n x: X_a,\n }\n feed_dict.update(dp_dict)\n result_a = sess.run(y_op, feed_dict=feed_dict)\n if result is None:\n result = result_a\n else:\n result = np.concatenate((result, result_a))\n if result is None:\n if len(X) % batch_size != 0:\n dp_dict = dict_to_one(network.all_drop)\n feed_dict = {\n x: X[-(len(X) % batch_size):, :],\n }\n feed_dict.update(dp_dict)\n result_a = sess.run(y_op, feed_dict=feed_dict)\n result = result_a\n else:\n if len(X) != len(result) and len(X) % batch_size != 0:\n dp_dict = dict_to_one(network.all_drop)\n feed_dict = {\n x: X[-(len(X) % batch_size):, :],\n }\n feed_dict.update(dp_dict)\n result_a = sess.run(y_op, feed_dict=feed_dict)\n result = np.concatenate((result, result_a))\n return result", "response": "Predict the result of given non - time - series network."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninputting the features and labels and return the features and labels after oversampling.", "response": "def class_balancing_oversample(X_train=None, y_train=None, printable=True):\n \"\"\"Input the features and labels, return the features and labels after oversampling.\n\n Parameters\n ----------\n X_train : numpy.array\n The inputs.\n y_train : numpy.array\n The targets.\n\n Examples\n --------\n One X\n\n >>> X_train, y_train = class_balancing_oversample(X_train, y_train, printable=True)\n\n Two X\n\n >>> X, y = tl.utils.class_balancing_oversample(X_train=np.hstack((X1, X2)), y_train=y, printable=False)\n >>> X1 = X[:, 0:5]\n >>> X2 = X[:, 5:]\n\n \"\"\"\n # ======== Classes balancing\n if printable:\n tl.logging.info(\"Classes balancing for training examples...\")\n\n c = Counter(y_train)\n\n if printable:\n tl.logging.info('the occurrence number of each stage: %s' % c.most_common())\n tl.logging.info('the least stage is Label %s have %s instances' % c.most_common()[-1])\n tl.logging.info('the most stage is Label %s have %s instances' % c.most_common(1)[0])\n\n most_num = c.most_common(1)[0][1]\n\n if printable:\n tl.logging.info('most num is %d, all classes tend to be this num' % most_num)\n\n locations = {}\n number = {}\n\n for lab, num in c.most_common(): # find the index from y_train\n number[lab] = num\n locations[lab] = np.where(np.array(y_train) == lab)[0]\n if printable:\n tl.logging.info('convert list(np.array) to dict format')\n X = {} # convert list to dict\n for lab, num in number.items():\n X[lab] = X_train[locations[lab]]\n\n # oversampling\n if printable:\n tl.logging.info('start oversampling')\n for key in X:\n temp = X[key]\n while True:\n if len(X[key]) >= most_num:\n break\n X[key] = np.vstack((X[key], temp))\n if printable:\n tl.logging.info('first features of label 0 > %d' % len(X[0][0]))\n tl.logging.info('the occurrence num of each stage after oversampling')\n for key in X:\n tl.logging.info(\"%s %d\" % (key, len(X[key])))\n if printable:\n tl.logging.info('make each stage have same num of instances')\n for key in X:\n X[key] = X[key][0:most_num, :]\n tl.logging.info(\"%s %d\" % (key, len(X[key])))\n\n # convert dict to list\n if printable:\n tl.logging.info('convert from dict to list format')\n y_train = []\n X_train = np.empty(shape=(0, len(X[0][0])))\n for key in X:\n X_train = np.vstack((X_train, X[key]))\n y_train.extend([key for i in range(len(X[key]))])\n # tl.logging.info(len(X_train), len(y_train))\n c = Counter(y_train)\n if printable:\n tl.logging.info('the occurrence number of each stage after oversampling: %s' % c.most_common())\n # ================ End of Classes balancing\n return X_train, y_train"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_random_int(min_v=0, max_v=10, number=5, seed=None):\n rnd = random.Random()\n if seed:\n rnd = random.Random(seed)\n # return [random.randint(min,max) for p in range(0, number)]\n return [rnd.randint(min_v, max_v) for p in range(0, number)]", "response": "Return a list of random integer by the given range and quantity."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninputs list string returns dictionary", "response": "def list_string_to_dict(string):\n \"\"\"Inputs ``['a', 'b', 'c']``, returns ``{'a': 0, 'b': 1, 'c': 2}``.\"\"\"\n dictionary = {}\n for idx, c in enumerate(string):\n dictionary.update({c: idx})\n return dictionary"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef exit_tensorflow(sess=None, port=6006):\n text = \"[TL] Close tensorboard and nvidia-process if available\"\n text2 = \"[TL] Close tensorboard and nvidia-process not yet supported by this function (tl.ops.exit_tf) on \"\n\n if sess is not None:\n sess.close()\n\n if _platform == \"linux\" or _platform == \"linux2\":\n tl.logging.info('linux: %s' % text)\n os.system('nvidia-smi')\n os.system('fuser ' + port + '/tcp -k') # kill tensorboard 6006\n os.system(\"nvidia-smi | grep python |awk '{print $3}'|xargs kill\") # kill all nvidia-smi python process\n _exit()\n\n elif _platform == \"darwin\":\n tl.logging.info('OS X: %s' % text)\n subprocess.Popen(\n \"lsof -i tcp:\" + str(port) + \" | grep -v PID | awk '{print $2}' | xargs kill\", shell=True\n ) # kill tensorboard\n elif _platform == \"win32\":\n raise NotImplementedError(\"this function is not supported on the Windows platform\")\n\n else:\n tl.logging.info(text2 + _platform)", "response": "Close TensorBoard and Nvidia - process if available."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nopens a TensorBoard in the specified port.", "response": "def open_tensorboard(log_dir='/tmp/tensorflow', port=6006):\n \"\"\"Open Tensorboard.\n\n Parameters\n ----------\n log_dir : str\n Directory where your tensorboard logs are saved\n port : int\n TensorBoard port you want to open, 6006 is tensorboard default\n\n \"\"\"\n text = \"[TL] Open tensorboard, go to localhost:\" + str(port) + \" to access\"\n text2 = \" not yet supported by this function (tl.ops.open_tb)\"\n\n if not tl.files.exists_or_mkdir(log_dir, verbose=False):\n tl.logging.info(\"[TL] Log reportory was created at %s\" % log_dir)\n\n if _platform == \"linux\" or _platform == \"linux2\":\n raise NotImplementedError()\n elif _platform == \"darwin\":\n tl.logging.info('OS X: %s' % text)\n subprocess.Popen(\n sys.prefix + \" | python -m tensorflow.tensorboard --logdir=\" + log_dir + \" --port=\" + str(port), shell=True\n ) # open tensorboard in localhost:6006/ or whatever port you chose\n elif _platform == \"win32\":\n raise NotImplementedError(\"this function is not supported on the Windows platform\")\n else:\n tl.logging.info(_platform + text2)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nclear all the placeholder variables of keep prob and dropout denoising and dropconnect etc.", "response": "def clear_all_placeholder_variables(printable=True):\n \"\"\"Clears all the placeholder variables of keep prob,\n including keeping probabilities of all dropout, denoising, dropconnect etc.\n\n Parameters\n ----------\n printable : boolean\n If True, print all deleted variables.\n\n \"\"\"\n tl.logging.info('clear all .....................................')\n gl = globals().copy()\n for var in gl:\n if var[0] == '_': continue\n if 'func' in str(globals()[var]): continue\n if 'module' in str(globals()[var]): continue\n if 'class' in str(globals()[var]): continue\n\n if printable:\n tl.logging.info(\" clear_all ------- %s\" % str(globals()[var]))\n\n del globals()[var]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_gpu_fraction(gpu_fraction=0.3):\n tl.logging.info(\"[TL]: GPU MEM Fraction %f\" % gpu_fraction)\n gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)\n sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))\n return sess", "response": "Sets the GPU memory fraction for the application."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating a training batch for the Skip - Gram model.", "response": "def generate_skip_gram_batch(data, batch_size, num_skips, skip_window, data_index=0):\n \"\"\"Generate a training batch for the Skip-Gram model.\n\n See `Word2Vec example `__.\n\n Parameters\n ----------\n data : list of data\n To present context, usually a list of integers.\n batch_size : int\n Batch size to return.\n num_skips : int\n How many times to reuse an input to generate a label.\n skip_window : int\n How many words to consider left and right.\n data_index : int\n Index of the context location. This code use `data_index` to instead of yield like ``tl.iterate``.\n\n Returns\n -------\n batch : list of data\n Inputs.\n labels : list of data\n Labels\n data_index : int\n Index of the context location.\n\n Examples\n --------\n Setting num_skips=2, skip_window=1, use the right and left words.\n In the same way, num_skips=4, skip_window=2 means use the nearby 4 words.\n\n >>> data = [1,2,3,4,5,6,7,8,9,10,11]\n >>> batch, labels, data_index = tl.nlp.generate_skip_gram_batch(data=data, batch_size=8, num_skips=2, skip_window=1, data_index=0)\n >>> print(batch)\n [2 2 3 3 4 4 5 5]\n >>> print(labels)\n [[3]\n [1]\n [4]\n [2]\n [5]\n [3]\n [4]\n [6]]\n\n \"\"\"\n # global data_index # you can put data_index outside the function, then\n # modify the global data_index in the function without return it.\n # note: without using yield, this code use data_index to instead.\n\n if batch_size % num_skips != 0:\n raise Exception(\"batch_size should be able to be divided by num_skips.\")\n if num_skips > 2 * skip_window:\n raise Exception(\"num_skips <= 2 * skip_window\")\n batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n span = 2 * skip_window + 1 # [ skip_window target skip_window ]\n buffer = collections.deque(maxlen=span)\n for _ in range(span):\n buffer.append(data[data_index])\n data_index = (data_index + 1) % len(data)\n for i in range(batch_size // num_skips):\n target = skip_window # target label at the center of the buffer\n targets_to_avoid = [skip_window]\n for j in range(num_skips):\n while target in targets_to_avoid:\n target = random.randint(0, span - 1)\n targets_to_avoid.append(target)\n batch[i * num_skips + j] = buffer[skip_window]\n labels[i * num_skips + j, 0] = buffer[target]\n buffer.append(data[data_index])\n data_index = (data_index + 1) % len(data)\n return batch, labels, data_index"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sample(a=None, temperature=1.0):\n if a is None:\n raise Exception(\"a : list of float\")\n b = np.copy(a)\n try:\n if temperature == 1:\n return np.argmax(np.random.multinomial(1, a, 1))\n if temperature is None:\n return np.argmax(a)\n else:\n a = np.log(a) / temperature\n a = np.exp(a) / np.sum(np.exp(a))\n return np.argmax(np.random.multinomial(1, a, 1))\n except Exception:\n # np.set_printoptions(threshold=np.nan)\n # tl.logging.info(a)\n # tl.logging.info(np.sum(a))\n # tl.logging.info(np.max(a))\n # tl.logging.info(np.min(a))\n # exit()\n message = \"For large vocabulary_size, choice a higher temperature\\\n to avoid log error. Hint : use ``sample_top``. \"\n\n warnings.warn(message, Warning)\n # tl.logging.info(a)\n # tl.logging.info(b)\n return np.argmax(np.random.multinomial(1, b, 1))", "response": "Sample an index from a probability array."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsample from top_k probabilities.", "response": "def sample_top(a=None, top_k=10):\n \"\"\"Sample from ``top_k`` probabilities.\n\n Parameters\n ----------\n a : list of float\n List of probabilities.\n top_k : int\n Number of candidates to be considered.\n\n \"\"\"\n if a is None:\n a = []\n\n idx = np.argpartition(a, -top_k)[-top_k:]\n probs = a[idx]\n # tl.logging.info(\"new %f\" % probs)\n probs = probs / np.sum(probs)\n choice = np.random.choice(idx, p=probs)\n return choice"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef process_sentence(sentence, start_word=\"\", end_word=\"\"):\n if start_word is not None:\n process_sentence = [start_word]\n else:\n process_sentence = []\n process_sentence.extend(nltk.tokenize.word_tokenize(sentence.lower()))\n\n if end_word is not None:\n process_sentence.append(end_word)\n return process_sentence", "response": "Takes a string and returns a list of strings that are separated into words."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates the vocabulary of words to word_id.", "response": "def create_vocab(sentences, word_counts_output_file, min_word_count=1):\n \"\"\"Creates the vocabulary of word to word_id.\n\n See ``tutorial_tfrecord3.py``.\n\n The vocabulary is saved to disk in a text file of word counts. The id of each\n word in the file is its corresponding 0-based line number.\n\n Parameters\n ------------\n sentences : list of list of str\n All sentences for creating the vocabulary.\n word_counts_output_file : str\n The file name.\n min_word_count : int\n Minimum number of occurrences for a word.\n\n Returns\n --------\n :class:`SimpleVocabulary`\n The simple vocabulary object, see :class:`Vocabulary` for more.\n\n Examples\n --------\n Pre-process sentences\n\n >>> captions = [\"one two , three\", \"four five five\"]\n >>> processed_capts = []\n >>> for c in captions:\n >>> c = tl.nlp.process_sentence(c, start_word=\"\", end_word=\"\")\n >>> processed_capts.append(c)\n >>> print(processed_capts)\n ...[['', 'one', 'two', ',', 'three', ''], ['', 'four', 'five', 'five', '']]\n\n Create vocabulary\n\n >>> tl.nlp.create_vocab(processed_capts, word_counts_output_file='vocab.txt', min_word_count=1)\n Creating vocabulary.\n Total words: 8\n Words in vocabulary: 8\n Wrote vocabulary file: vocab.txt\n\n Get vocabulary object\n\n >>> vocab = tl.nlp.Vocabulary('vocab.txt', start_word=\"\", end_word=\"\", unk_word=\"\")\n INFO:tensorflow:Initializing vocabulary from file: vocab.txt\n [TL] Vocabulary from vocab.txt : \n vocabulary with 10 words (includes start_word, end_word, unk_word)\n start_id: 2\n end_id: 3\n unk_id: 9\n pad_id: 0\n\n \"\"\"\n tl.logging.info(\"Creating vocabulary.\")\n\n counter = Counter()\n\n for c in sentences:\n counter.update(c)\n # tl.logging.info('c',c)\n tl.logging.info(\" Total words: %d\" % len(counter))\n\n # Filter uncommon words and sort by descending count.\n word_counts = [x for x in counter.items() if x[1] >= min_word_count]\n word_counts.sort(key=lambda x: x[1], reverse=True)\n word_counts = [(\"\", 0)] + word_counts # 1st id should be reserved for padding\n # tl.logging.info(word_counts)\n tl.logging.info(\" Words in vocabulary: %d\" % len(word_counts))\n\n # Write out the word counts file.\n with tf.gfile.FastGFile(word_counts_output_file, \"w\") as f:\n f.write(\"\\n\".join([\"%s %d\" % (w, c) for w, c in word_counts]))\n tl.logging.info(\" Wrote vocabulary file: %s\" % word_counts_output_file)\n\n # Create the vocabulary dictionary.\n reverse_vocab = [x[0] for x in word_counts]\n unk_id = len(reverse_vocab)\n vocab_dict = dict([(x, y) for (y, x) in enumerate(reverse_vocab)])\n vocab = SimpleVocabulary(vocab_dict, unk_id)\n\n return vocab"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads list format context from a file.", "response": "def read_words(filename=\"nietzsche.txt\", replace=None):\n \"\"\"Read list format context from a file.\n\n For customized read_words method, see ``tutorial_generate_text.py``.\n\n Parameters\n ----------\n filename : str\n a file path.\n replace : list of str\n replace original string by target string.\n\n Returns\n -------\n list of str\n The context in a list (split using space).\n \"\"\"\n if replace is None:\n replace = ['\\n', '']\n\n with tf.gfile.GFile(filename, \"r\") as f:\n try: # python 3.4 or older\n context_list = f.read().replace(*replace).split()\n except Exception: # python 3.5\n f.seek(0)\n replace = [x.encode('utf-8') for x in replace]\n context_list = f.read().replace(*replace).split()\n return context_list"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread through an analogy question file and returns its id format.", "response": "def read_analogies_file(eval_file='questions-words.txt', word2id=None):\n \"\"\"Reads through an analogy question file, return its id format.\n\n Parameters\n ----------\n eval_file : str\n The file name.\n word2id : dictionary\n a dictionary that maps word to ID.\n\n Returns\n --------\n numpy.array\n A ``[n_examples, 4]`` numpy array containing the analogy question's word IDs.\n\n Examples\n ---------\n The file should be in this format\n\n >>> : capital-common-countries\n >>> Athens Greece Baghdad Iraq\n >>> Athens Greece Bangkok Thailand\n >>> Athens Greece Beijing China\n >>> Athens Greece Berlin Germany\n >>> Athens Greece Bern Switzerland\n >>> Athens Greece Cairo Egypt\n >>> Athens Greece Canberra Australia\n >>> Athens Greece Hanoi Vietnam\n >>> Athens Greece Havana Cuba\n\n Get the tokenized analogy question data\n\n >>> words = tl.files.load_matt_mahoney_text8_dataset()\n >>> data, count, dictionary, reverse_dictionary = tl.nlp.build_words_dataset(words, vocabulary_size, True)\n >>> analogy_questions = tl.nlp.read_analogies_file(eval_file='questions-words.txt', word2id=dictionary)\n >>> print(analogy_questions)\n [[ 3068 1248 7161 1581]\n [ 3068 1248 28683 5642]\n [ 3068 1248 3878 486]\n ...,\n [ 1216 4309 19982 25506]\n [ 1216 4309 3194 8650]\n [ 1216 4309 140 312]]\n\n \"\"\"\n if word2id is None:\n word2id = {}\n\n questions = []\n questions_skipped = 0\n with open(eval_file, \"rb\") as analogy_f:\n for line in analogy_f:\n if line.startswith(b\":\"): # Skip comments.\n continue\n words = line.strip().lower().split(b\" \") # lowercase\n ids = [word2id.get(w.strip()) for w in words]\n if None in ids or len(ids) != 4:\n questions_skipped += 1\n else:\n questions.append(np.array(ids))\n tl.logging.info(\"Eval analogy file: %s\" % eval_file)\n tl.logging.info(\"Questions: %d\", len(questions))\n tl.logging.info(\"Skipped: %d\", questions_skipped)\n analogy_questions = np.array(questions, dtype=np.int32)\n return analogy_questions"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nbuild a reverse dictionary that maps a word to integer id.", "response": "def build_reverse_dictionary(word_to_id):\n \"\"\"Given a dictionary that maps word to integer id.\n Returns a reverse dictionary that maps a id to word.\n\n Parameters\n ----------\n word_to_id : dictionary\n that maps word to ID.\n\n Returns\n --------\n dictionary\n A dictionary that maps IDs to words.\n\n \"\"\"\n reverse_dictionary = dict(zip(word_to_id.values(), word_to_id.keys()))\n return reverse_dictionary"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef build_words_dataset(words=None, vocabulary_size=50000, printable=True, unk_key='UNK'):\n if words is None:\n raise Exception(\"words : list of str or byte\")\n\n count = [[unk_key, -1]]\n count.extend(collections.Counter(words).most_common(vocabulary_size - 1))\n dictionary = dict()\n for word, _ in count:\n dictionary[word] = len(dictionary)\n data = list()\n unk_count = 0\n for word in words:\n if word in dictionary:\n index = dictionary[word]\n else:\n index = 0 # dictionary['UNK']\n unk_count += 1\n data.append(index)\n count[0][1] = unk_count\n reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n if printable:\n tl.logging.info('Real vocabulary size %d' % len(collections.Counter(words).keys()))\n tl.logging.info('Limited vocabulary size {}'.format(vocabulary_size))\n if len(collections.Counter(words).keys()) < vocabulary_size:\n raise Exception(\n \"len(collections.Counter(words).keys()) >= vocabulary_size , the limited vocabulary_size must be less than or equal to the read vocabulary_size\"\n )\n return data, count, dictionary, reverse_dictionary", "response": "Build the words dictionary and replace rare words with UNK token."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert a list of string or byte words to list of int IDs.", "response": "def words_to_word_ids(data=None, word_to_id=None, unk_key='UNK'):\n \"\"\"Convert a list of string (words) to IDs.\n\n Parameters\n ----------\n data : list of string or byte\n The context in list format\n word_to_id : a dictionary\n that maps word to ID.\n unk_key : str\n Represent the unknown words.\n\n Returns\n --------\n list of int\n A list of IDs to represent the context.\n\n Examples\n --------\n >>> words = tl.files.load_matt_mahoney_text8_dataset()\n >>> vocabulary_size = 50000\n >>> data, count, dictionary, reverse_dictionary = tl.nlp.build_words_dataset(words, vocabulary_size, True)\n >>> context = [b'hello', b'how', b'are', b'you']\n >>> ids = tl.nlp.words_to_word_ids(words, dictionary)\n >>> context = tl.nlp.word_ids_to_words(ids, reverse_dictionary)\n >>> print(ids)\n [6434, 311, 26, 207]\n >>> print(context)\n [b'hello', b'how', b'are', b'you']\n\n References\n ---------------\n - `tensorflow.models.rnn.ptb.reader `__\n\n \"\"\"\n if data is None:\n raise Exception(\"data : list of string or byte\")\n if word_to_id is None:\n raise Exception(\"word_to_id : a dictionary\")\n # if isinstance(data[0], six.string_types):\n # tl.logging.info(type(data[0]))\n # # exit()\n # tl.logging.info(data[0])\n # tl.logging.info(word_to_id)\n # return [word_to_id[str(word)] for word in data]\n # else:\n\n word_ids = []\n for word in data:\n if word_to_id.get(word) is not None:\n word_ids.append(word_to_id[word])\n else:\n word_ids.append(word_to_id[unk_key])\n return word_ids"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_vocab(count=None, name='vocab.txt'):\n if count is None:\n count = []\n\n pwd = os.getcwd()\n vocabulary_size = len(count)\n with open(os.path.join(pwd, name), \"w\") as f:\n for i in xrange(vocabulary_size):\n f.write(\"%s %d\\n\" % (tf.compat.as_text(count[i][0]), count[i][1]))\n tl.logging.info(\"%d vocab saved to %s in %s\" % (vocabulary_size, name, pwd))", "response": "Save the vocabulary to a file."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef basic_tokenizer(sentence, _WORD_SPLIT=re.compile(b\"([.,!?\\\"':;)(])\")):\n words = []\n sentence = tf.compat.as_bytes(sentence)\n for space_separated_fragment in sentence.strip().split():\n words.extend(re.split(_WORD_SPLIT, space_separated_fragment))\n return [w for w in words if w]", "response": "Very basic tokenizer: split the sentence into a list of tokens.\n\n Parameters\n -----------\n sentence : tensorflow.python.platform.gfile.GFile Object\n _WORD_SPLIT : regular expression for word spliting.\n\n\n Examples\n --------\n >>> see create_vocabulary\n >>> from tensorflow.python.platform import gfile\n >>> train_path = \"wmt/giga-fren.release2\"\n >>> with gfile.GFile(train_path + \".en\", mode=\"rb\") as f:\n >>> for line in f:\n >>> tokens = tl.nlp.basic_tokenizer(line)\n >>> tl.logging.info(tokens)\n >>> exit()\n [b'Changing', b'Lives', b'|', b'Changing', b'Society', b'|', b'How',\n b'It', b'Works', b'|', b'Technology', b'Drives', b'Change', b'Home',\n b'|', b'Concepts', b'|', b'Teachers', b'|', b'Search', b'|', b'Overview',\n b'|', b'Credits', b'|', b'HHCC', b'Web', b'|', b'Reference', b'|',\n b'Feedback', b'Virtual', b'Museum', b'of', b'Canada', b'Home', b'Page']\n\n References\n ----------\n - Code from ``/tensorflow/models/rnn/translation/data_utils.py``"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing vocabulary from file.", "response": "def initialize_vocabulary(vocabulary_path):\n \"\"\"Initialize vocabulary from file, return the `word_to_id` (dictionary)\n and `id_to_word` (list).\n\n We assume the vocabulary is stored one-item-per-line, so a file will result in a vocabulary {\"dog\": 0, \"cat\": 1}, and this function will also return the reversed-vocabulary [\"dog\", \"cat\"].\n\n Parameters\n -----------\n vocabulary_path : str\n Path to the file containing the vocabulary.\n\n Returns\n --------\n vocab : dictionary\n a dictionary that maps word to ID.\n rev_vocab : list of int\n a list that maps ID to word.\n\n Examples\n ---------\n >>> Assume 'test' contains\n dog\n cat\n bird\n >>> vocab, rev_vocab = tl.nlp.initialize_vocabulary(\"test\")\n >>> print(vocab)\n >>> {b'cat': 1, b'dog': 0, b'bird': 2}\n >>> print(rev_vocab)\n >>> [b'dog', b'cat', b'bird']\n\n Raises\n -------\n ValueError : if the provided vocabulary_path does not exist.\n\n \"\"\"\n if gfile.Exists(vocabulary_path):\n rev_vocab = []\n with gfile.GFile(vocabulary_path, mode=\"rb\") as f:\n rev_vocab.extend(f.readlines())\n rev_vocab = [tf.compat.as_bytes(line.strip()) for line in rev_vocab]\n vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])\n return vocab, rev_vocab\n else:\n raise ValueError(\"Vocabulary file %s not found.\", vocabulary_path)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts a string to list of token - ids.", "response": "def sentence_to_token_ids(\n sentence, vocabulary, tokenizer=None, normalize_digits=True, UNK_ID=3, _DIGIT_RE=re.compile(br\"\\d\")\n):\n \"\"\"Convert a string to list of integers representing token-ids.\n\n For example, a sentence \"I have a dog\" may become tokenized into\n [\"I\", \"have\", \"a\", \"dog\"] and with vocabulary {\"I\": 1, \"have\": 2,\n \"a\": 4, \"dog\": 7\"} this function will return [1, 2, 4, 7].\n\n Parameters\n -----------\n sentence : tensorflow.python.platform.gfile.GFile Object\n The sentence in bytes format to convert to token-ids, see ``basic_tokenizer()`` and ``data_to_token_ids()``.\n vocabulary : dictionary\n Mmapping tokens to integers.\n tokenizer : function\n A function to use to tokenize each sentence. If None, ``basic_tokenizer`` will be used.\n normalize_digits : boolean\n If true, all digits are replaced by 0.\n\n Returns\n --------\n list of int\n The token-ids for the sentence.\n\n \"\"\"\n if tokenizer:\n words = tokenizer(sentence)\n else:\n words = basic_tokenizer(sentence)\n if not normalize_digits:\n return [vocabulary.get(w, UNK_ID) for w in words]\n # Normalize digits by 0 before looking words up in the vocabulary.\n return [vocabulary.get(re.sub(_DIGIT_RE, b\"0\", w), UNK_ID) for w in words]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef data_to_token_ids(\n data_path, target_path, vocabulary_path, tokenizer=None, normalize_digits=True, UNK_ID=3,\n _DIGIT_RE=re.compile(br\"\\d\")\n):\n \"\"\"Tokenize data file and turn into token-ids using given vocabulary file.\n\n This function loads data line-by-line from data_path, calls the above\n sentence_to_token_ids, and saves the result to target_path. See comment\n for sentence_to_token_ids on the details of token-ids format.\n\n Parameters\n -----------\n data_path : str\n Path to the data file in one-sentence-per-line format.\n target_path : str\n Path where the file with token-ids will be created.\n vocabulary_path : str\n Path to the vocabulary file.\n tokenizer : function\n A function to use to tokenize each sentence. If None, ``basic_tokenizer`` will be used.\n normalize_digits : boolean\n If true, all digits are replaced by 0.\n\n References\n ----------\n - Code from ``/tensorflow/models/rnn/translation/data_utils.py``\n\n \"\"\"\n if not gfile.Exists(target_path):\n tl.logging.info(\"Tokenizing data in %s\" % data_path)\n vocab, _ = initialize_vocabulary(vocabulary_path)\n with gfile.GFile(data_path, mode=\"rb\") as data_file:\n with gfile.GFile(target_path, mode=\"w\") as tokens_file:\n counter = 0\n for line in data_file:\n counter += 1\n if counter % 100000 == 0:\n tl.logging.info(\" tokenizing line %d\" % counter)\n token_ids = sentence_to_token_ids(\n line, vocab, tokenizer, normalize_digits, UNK_ID=UNK_ID, _DIGIT_RE=_DIGIT_RE\n )\n tokens_file.write(\" \".join([str(tok) for tok in token_ids]) + \"\\n\")\n else:\n tl.logging.info(\"Target path %s exists\" % target_path)", "response": "Tokenize data file and turn into token - ids using given tokenizer."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef moses_multi_bleu(hypotheses, references, lowercase=False):\n if np.size(hypotheses) == 0:\n return np.float32(0.0)\n\n # Get MOSES multi-bleu script\n try:\n multi_bleu_path, _ = urllib.request.urlretrieve(\n \"https://raw.githubusercontent.com/moses-smt/mosesdecoder/\"\n \"master/scripts/generic/multi-bleu.perl\"\n )\n os.chmod(multi_bleu_path, 0o755)\n except Exception: # pylint: disable=W0702\n tl.logging.info(\"Unable to fetch multi-bleu.perl script, using local.\")\n metrics_dir = os.path.dirname(os.path.realpath(__file__))\n bin_dir = os.path.abspath(os.path.join(metrics_dir, \"..\", \"..\", \"bin\"))\n multi_bleu_path = os.path.join(bin_dir, \"tools/multi-bleu.perl\")\n\n # Dump hypotheses and references to tempfiles\n hypothesis_file = tempfile.NamedTemporaryFile()\n hypothesis_file.write(\"\\n\".join(hypotheses).encode(\"utf-8\"))\n hypothesis_file.write(b\"\\n\")\n hypothesis_file.flush()\n reference_file = tempfile.NamedTemporaryFile()\n reference_file.write(\"\\n\".join(references).encode(\"utf-8\"))\n reference_file.write(b\"\\n\")\n reference_file.flush()\n\n # Calculate BLEU using multi-bleu script\n with open(hypothesis_file.name, \"r\") as read_pred:\n bleu_cmd = [multi_bleu_path]\n if lowercase:\n bleu_cmd += [\"-lc\"]\n bleu_cmd += [reference_file.name]\n try:\n bleu_out = subprocess.check_output(bleu_cmd, stdin=read_pred, stderr=subprocess.STDOUT)\n bleu_out = bleu_out.decode(\"utf-8\")\n bleu_score = re.search(r\"BLEU = (.+?),\", bleu_out).group(1)\n bleu_score = float(bleu_score)\n except subprocess.CalledProcessError as error:\n if error.output is not None:\n tl.logging.warning(\"multi-bleu.perl script returned non-zero exit code\")\n tl.logging.warning(error.output)\n bleu_score = np.float32(0.0)\n\n # Close temp files\n hypothesis_file.close()\n reference_file.close()\n\n return np.float32(bleu_score)", "response": "Calculate the bleu score for hypotheses and references using the MOSES multi - bLEU script."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the integer id of a word string.", "response": "def word_to_id(self, word):\n \"\"\"Returns the integer id of a word string.\"\"\"\n if word in self._vocab:\n return self._vocab[word]\n else:\n return self._unk_id"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the integer word id of a word string.", "response": "def word_to_id(self, word):\n \"\"\"Returns the integer word id of a word string.\"\"\"\n if word in self.vocab:\n return self.vocab[word]\n else:\n return self.unk_id"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the word string of an integer word id.", "response": "def id_to_word(self, word_id):\n \"\"\"Returns the word string of an integer word id.\"\"\"\n if word_id >= len(self.reverse_vocab):\n return self.reverse_vocab[self.unk_id]\n else:\n return self.reverse_vocab[word_id]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef main_restore_embedding_layer():\n # Step 1: Build the embedding matrix and load the existing embedding matrix.\n vocabulary_size = 50000\n embedding_size = 128\n model_file_name = \"model_word2vec_50k_128\"\n batch_size = None\n\n print(\"Load existing embedding matrix and dictionaries\")\n all_var = tl.files.load_npy_to_any(name=model_file_name + '.npy')\n data = all_var['data']\n count = all_var['count']\n dictionary = all_var['dictionary']\n reverse_dictionary = all_var['reverse_dictionary']\n\n tl.nlp.save_vocab(count, name='vocab_' + model_file_name + '.txt')\n\n del all_var, data, count\n\n load_params = tl.files.load_npz(name=model_file_name + '.npz')\n\n x = tf.placeholder(tf.int32, shape=[batch_size])\n\n emb_net = tl.layers.EmbeddingInputlayer(x, vocabulary_size, embedding_size, name='emb')\n\n # sess.run(tf.global_variables_initializer())\n sess.run(tf.global_variables_initializer())\n\n tl.files.assign_params(sess, [load_params[0]], emb_net)\n\n emb_net.print_params()\n emb_net.print_layers()\n\n # Step 2: Input word(s), output the word vector(s).\n word = b'hello'\n word_id = dictionary[word]\n print('word_id:', word_id)\n\n words = [b'i', b'am', b'tensor', b'layer']\n word_ids = tl.nlp.words_to_word_ids(words, dictionary, _UNK)\n context = tl.nlp.word_ids_to_words(word_ids, reverse_dictionary)\n print('word_ids:', word_ids)\n print('context:', context)\n\n vector = sess.run(emb_net.outputs, feed_dict={x: [word_id]})\n print('vector:', vector.shape)\n\n vectors = sess.run(emb_net.outputs, feed_dict={x: word_ids})\n print('vectors:', vectors.shape)", "response": "Restore the embedding layer."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates text by Synced sequence input and output.", "response": "def main_lstm_generate_text():\n \"\"\"Generate text by Synced sequence input and output.\"\"\"\n # rnn model and update (describtion: see tutorial_ptb_lstm.py)\n init_scale = 0.1\n learning_rate = 1.0\n max_grad_norm = 5\n sequence_length = 20\n hidden_size = 200\n max_epoch = 4\n max_max_epoch = 100\n lr_decay = 0.9\n batch_size = 20\n\n top_k_list = [1, 3, 5, 10]\n print_length = 30\n\n model_file_name = \"model_generate_text.npz\"\n\n # ===== Prepare Data\n words = customized_read_words(input_fpath=\"data/trump/trump_text.txt\")\n\n vocab = tl.nlp.create_vocab([words], word_counts_output_file='vocab.txt', min_word_count=1)\n vocab = tl.nlp.Vocabulary('vocab.txt', unk_word=\"\")\n vocab_size = vocab.unk_id + 1\n train_data = [vocab.word_to_id(word) for word in words]\n\n # Set the seed to generate sentence.\n seed = \"it is a\"\n # seed = basic_clean_str(seed).split()\n seed = nltk.tokenize.word_tokenize(seed)\n print('seed : %s' % seed)\n\n sess = tf.InteractiveSession()\n\n # ===== Define model\n input_data = tf.placeholder(tf.int32, [batch_size, sequence_length])\n targets = tf.placeholder(tf.int32, [batch_size, sequence_length])\n # Testing (Evaluation), for generate text\n input_data_test = tf.placeholder(tf.int32, [1, 1])\n\n def inference(x, is_train, sequence_length, reuse=None):\n \"\"\"If reuse is True, the inferences use the existing parameters,\n then different inferences share the same parameters.\n \"\"\"\n print(\"\\nsequence_length: %d, is_train: %s, reuse: %s\" % (sequence_length, is_train, reuse))\n rnn_init = tf.random_uniform_initializer(-init_scale, init_scale)\n with tf.variable_scope(\"model\", reuse=reuse):\n network = EmbeddingInputlayer(x, vocab_size, hidden_size, rnn_init, name='embedding')\n network = RNNLayer(\n network, cell_fn=tf.contrib.rnn.BasicLSTMCell, cell_init_args={\n 'forget_bias': 0.0,\n 'state_is_tuple': True\n }, n_hidden=hidden_size, initializer=rnn_init, n_steps=sequence_length, return_last=False,\n return_seq_2d=True, name='lstm1'\n )\n lstm1 = network\n network = DenseLayer(network, vocab_size, W_init=rnn_init, b_init=rnn_init, act=None, name='output')\n return network, lstm1\n\n # Inference for Training\n network, lstm1 = inference(input_data, is_train=True, sequence_length=sequence_length, reuse=None)\n # Inference for generate text, sequence_length=1\n network_test, lstm1_test = inference(input_data_test, is_train=False, sequence_length=1, reuse=True)\n y_linear = network_test.outputs\n y_soft = tf.nn.softmax(y_linear)\n\n # y_id = tf.argmax(tf.nn.softmax(y), 1)\n\n # ===== Define train ops\n def loss_fn(outputs, targets, batch_size, sequence_length):\n # Returns the cost function of Cross-entropy of two sequences, implement\n # softmax internally.\n # outputs : 2D tensor [n_examples, n_outputs]\n # targets : 2D tensor [n_examples, n_outputs]\n # n_examples = batch_size * sequence_length\n # so\n # cost is the averaged cost of each mini-batch (concurrent process).\n loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(\n [outputs], [tf.reshape(targets, [-1])], [tf.ones([batch_size * sequence_length])]\n )\n cost = tf.reduce_sum(loss) / batch_size\n return cost\n\n # Cost for Training\n cost = loss_fn(network.outputs, targets, batch_size, sequence_length)\n\n # Truncated Backpropagation for training\n with tf.variable_scope('learning_rate'):\n lr = tf.Variable(0.0, trainable=False)\n # You can get all trainable parameters as follow.\n # tvars = tf.trainable_variables()\n # Alternatively, you can specify the parameters for training as follw.\n # tvars = network.all_params $ all parameters\n # tvars = network.all_params[1:] $ parameters except embedding matrix\n # Train the whole network.\n tvars = network.all_params\n grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), max_grad_norm)\n optimizer = tf.train.GradientDescentOptimizer(lr)\n train_op = optimizer.apply_gradients(zip(grads, tvars))\n\n # ===== Training\n sess.run(tf.global_variables_initializer())\n\n print(\"\\nStart learning a model to generate text\")\n for i in range(max_max_epoch):\n # decrease the learning_rate after ``max_epoch``, by multipling lr_decay.\n new_lr_decay = lr_decay**max(i - max_epoch, 0.0)\n sess.run(tf.assign(lr, learning_rate * new_lr_decay))\n\n print(\"Epoch: %d/%d Learning rate: %.8f\" % (i + 1, max_max_epoch, sess.run(lr)))\n epoch_size = ((len(train_data) // batch_size) - 1) // sequence_length\n\n start_time = time.time()\n costs = 0.0\n iters = 0\n # reset all states at the begining of every epoch\n state1 = tl.layers.initialize_rnn_state(lstm1.initial_state)\n for step, (x, y) in enumerate(tl.iterate.ptb_iterator(train_data, batch_size, sequence_length)):\n _cost, state1, _ = sess.run(\n [cost, lstm1.final_state, train_op], feed_dict={\n input_data: x,\n targets: y,\n lstm1.initial_state: state1\n }\n )\n costs += _cost\n iters += sequence_length\n\n if step % (epoch_size // 10) == 1:\n print(\n \"%.3f perplexity: %.3f speed: %.0f wps\" %\n (step * 1.0 / epoch_size, np.exp(costs / iters), iters * batch_size / (time.time() - start_time))\n )\n train_perplexity = np.exp(costs / iters)\n # print(\"Epoch: %d Train Perplexity: %.3f\" % (i + 1, train_perplexity))\n print(\"Epoch: %d/%d Train Perplexity: %.3f\" % (i + 1, max_max_epoch, train_perplexity))\n\n # for diversity in diversity_list:\n # testing: sample from top k words\n for top_k in top_k_list:\n # Testing, generate some text from a given seed.\n state1 = tl.layers.initialize_rnn_state(lstm1_test.initial_state)\n # state2 = tl.layers.initialize_rnn_state(lstm2_test.initial_state)\n outs_id = [vocab.word_to_id(w) for w in seed]\n # feed the seed to initialize the state for generation.\n for ids in outs_id[:-1]:\n a_id = np.asarray(ids).reshape(1, 1)\n state1 = sess.run(\n [lstm1_test.final_state], feed_dict={\n input_data_test: a_id,\n lstm1_test.initial_state: state1\n }\n )\n # feed the last word in seed, and start to generate sentence.\n a_id = outs_id[-1]\n for _ in range(print_length):\n a_id = np.asarray(a_id).reshape(1, 1)\n out, state1 = sess.run(\n [y_soft, lstm1_test.final_state], feed_dict={\n input_data_test: a_id,\n lstm1_test.initial_state: state1\n }\n )\n # Without sampling\n # a_id = np.argmax(out[0])\n # Sample from all words, if vocab_size is large,\n # this may have numeric error.\n # a_id = tl.nlp.sample(out[0], diversity)\n # Sample from the top k words.\n a_id = tl.nlp.sample_top(out[0], top_k=top_k)\n outs_id.append(a_id)\n sentence = [vocab.id_to_word(w) for w in outs_id]\n sentence = \" \".join(sentence)\n # print(diversity, ':', sentence)\n print(top_k, ':', sentence)\n\n print(\"Save model\")\n tl.files.save_npz(network_test.all_params, name=model_file_name)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef createAndStartSwarm(client, clientInfo=\"\", clientKey=\"\", params=\"\",\n minimumWorkers=None, maximumWorkers=None,\n alreadyRunning=False):\n \"\"\"Create and start a swarm job.\n\n Args:\n client - A string identifying the calling client. There is a small limit\n for the length of the value. See ClientJobsDAO.CLIENT_MAX_LEN.\n clientInfo - JSON encoded dict of client specific information.\n clientKey - Foreign key. Limited in length, see ClientJobsDAO._initTables.\n params - JSON encoded dict of the parameters for the job. This can be\n fetched out of the database by the worker processes based on the jobID.\n minimumWorkers - The minimum workers to allocate to the swarm. Set to None\n to use the default.\n maximumWorkers - The maximum workers to allocate to the swarm. Set to None\n to use the swarm default. Set to 0 to use the maximum scheduler value.\n alreadyRunning - Insert a job record for an already running process. Used\n for testing.\n \"\"\"\n if minimumWorkers is None:\n minimumWorkers = Configuration.getInt(\n \"nupic.hypersearch.minWorkersPerSwarm\")\n if maximumWorkers is None:\n maximumWorkers = Configuration.getInt(\n \"nupic.hypersearch.maxWorkersPerSwarm\")\n\n return ClientJobsDAO.get().jobInsert(\n client=client,\n cmdLine=\"$HYPERSEARCH\",\n clientInfo=clientInfo,\n clientKey=clientKey,\n alreadyRunning=alreadyRunning,\n params=params,\n minimumWorkers=minimumWorkers,\n maximumWorkers=maximumWorkers,\n jobType=ClientJobsDAO.JOB_TYPE_HS)", "response": "Create and start a new job."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nretrieve the Engine - level model params from a Swarm model .", "response": "def getSwarmModelParams(modelID):\n \"\"\"Retrieve the Engine-level model params from a Swarm model\n\n Args:\n modelID - Engine-level model ID of the Swarm model\n\n Returns:\n JSON-encoded string containing Model Params\n \"\"\"\n\n # TODO: the use of nupic.frameworks.opf.helpers.loadExperimentDescriptionScriptFromDir when\n # retrieving module params results in a leakage of pf_base_descriptionNN and\n # pf_descriptionNN module imports for every call to getSwarmModelParams, so\n # the leakage is unlimited when getSwarmModelParams is called by a\n # long-running process. An alternate solution is to execute the guts of\n # this function's logic in a seprate process (via multiprocessing module).\n\n cjDAO = ClientJobsDAO.get()\n\n (jobID, description) = cjDAO.modelsGetFields(\n modelID,\n [\"jobId\", \"genDescription\"])\n\n (baseDescription,) = cjDAO.jobGetFields(jobID, [\"genBaseDescription\"])\n\n # Construct a directory with base.py and description.py for loading model\n # params, and use nupic.frameworks.opf.helpers to extract model params from\n # those files\n descriptionDirectory = tempfile.mkdtemp()\n try:\n baseDescriptionFilePath = os.path.join(descriptionDirectory, \"base.py\")\n with open(baseDescriptionFilePath, mode=\"wb\") as f:\n f.write(baseDescription)\n\n descriptionFilePath = os.path.join(descriptionDirectory, \"description.py\")\n with open(descriptionFilePath, mode=\"wb\") as f:\n f.write(description)\n\n expIface = helpers.getExperimentDescriptionInterfaceFromModule(\n helpers.loadExperimentDescriptionScriptFromDir(descriptionDirectory))\n\n return json.dumps(\n dict(\n modelConfig=expIface.getModelDescription(),\n inferenceArgs=expIface.getModelControl().get(\"inferenceArgs\", None)))\n finally:\n shutil.rmtree(descriptionDirectory, ignore_errors=True)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nenabling the diagnostic feature for debugging unexpected concurrency in .", "response": "def enableConcurrencyChecks(maxConcurrency, raiseException=True):\n \"\"\" Enable the diagnostic feature for debugging unexpected concurrency in\n acquiring ConnectionWrapper instances.\n\n NOTE: This MUST be done early in your application's execution, BEFORE any\n accesses to ConnectionFactory or connection policies from your application\n (including imports and sub-imports of your app).\n\n Parameters:\n ----------------------------------------------------------------\n maxConcurrency: A non-negative integer that represents the maximum expected\n number of outstanding connections. When this value is\n exceeded, useful information will be logged and, depending\n on the value of the raiseException arg,\n ConcurrencyExceededError may be raised.\n raiseException: If true, ConcurrencyExceededError will be raised when\n maxConcurrency is exceeded.\n \"\"\"\n global g_max_concurrency, g_max_concurrency_raise_exception\n\n assert maxConcurrency >= 0\n\n g_max_concurrency = maxConcurrency\n g_max_concurrency_raise_exception = raiseException\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _getCommonSteadyDBArgsDict():\n\n return dict(\n creator = pymysql,\n host = Configuration.get('nupic.cluster.database.host'),\n port = int(Configuration.get('nupic.cluster.database.port')),\n user = Configuration.get('nupic.cluster.database.user'),\n passwd = Configuration.get('nupic.cluster.database.passwd'),\n charset = 'utf8',\n use_unicode = True,\n setsession = ['SET AUTOCOMMIT = 1'])", "response": "Returns a dictionary of arguments for the common SteadyDBConnection\n constructor."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting a logger for the given class in this module.", "response": "def _getLogger(cls, logLevel=None):\n \"\"\" Gets a logger for the given class in this module\n \"\"\"\n logger = logging.getLogger(\n \".\".join(['com.numenta', _MODULE_NAME, cls.__name__]))\n\n if logLevel is not None:\n logger.setLevel(logLevel)\n\n return logger"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nacquire a ConnectionWrapper instance that represents a connection containing the specified nupic. cluster. database. Connections and returns it.", "response": "def get(cls):\n \"\"\" Acquire a ConnectionWrapper instance that represents a connection\n to the SQL server per nupic.cluster.database.* configuration settings.\n\n NOTE: caller is responsible for calling the ConnectionWrapper instance's\n release() method after using the connection in order to release resources.\n Better yet, use the returned ConnectionWrapper instance in a Context Manager\n statement for automatic invocation of release():\n Example:\n # If using Jython 2.5.x, first import with_statement at the very top of\n your script (don't need this import for Jython/Python 2.6.x and later):\n from __future__ import with_statement\n # Then:\n from nupic.database.Connection import ConnectionFactory\n # Then use it like this\n with ConnectionFactory.get() as conn:\n conn.cursor.execute(\"SELECT ...\")\n conn.cursor.fetchall()\n conn.cursor.execute(\"INSERT ...\")\n\n WARNING: DO NOT close the underlying connection or cursor as it may be\n shared by other modules in your process. ConnectionWrapper's release()\n method will do the right thing.\n\n Parameters:\n ----------------------------------------------------------------\n retval: A ConnectionWrapper instance. NOTE: Caller is responsible\n for releasing resources as described above.\n \"\"\"\n if cls._connectionPolicy is None:\n logger = _getLogger(cls)\n logger.info(\"Creating db connection policy via provider %r\",\n cls._connectionPolicyInstanceProvider)\n cls._connectionPolicy = cls._connectionPolicyInstanceProvider()\n\n logger.debug(\"Created connection policy: %r\", cls._connectionPolicy)\n\n return cls._connectionPolicy.acquireConnection()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _createDefaultPolicy(cls):\n logger = _getLogger(cls)\n\n logger.debug(\n \"Creating database connection policy: platform=%r; pymysql.VERSION=%r\",\n platform.system(), pymysql.VERSION)\n\n if platform.system() == \"Java\":\n # NOTE: PooledDB doesn't seem to work under Jython\n # NOTE: not appropriate for multi-threaded applications.\n # TODO: this was fixed in Webware DBUtils r8228, so once\n # we pick up a realease with this fix, we should use\n # PooledConnectionPolicy for both Jython and Python.\n policy = SingleSharedConnectionPolicy()\n else:\n policy = PooledConnectionPolicy()\n\n return policy", "response": "Create the default database connection policy instance."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreleasing the database connection and cursor and release resources.", "response": "def release(self):\n \"\"\" Release the database connection and cursor\n\n The receiver of the Connection instance MUST call this method in order\n to reclaim resources\n \"\"\"\n\n self._logger.debug(\"Releasing: %r\", self)\n\n # Discard self from set of outstanding instances\n if self._addedToInstanceSet:\n try:\n self._clsOutstandingInstances.remove(self)\n except:\n self._logger.exception(\n \"Failed to remove self from _clsOutstandingInstances: %r;\", self)\n raise\n\n self._releaser(dbConn=self.dbConn, cursor=self.cursor)\n\n self.__class__._clsNumOutstanding -= 1\n assert self._clsNumOutstanding >= 0, \\\n \"_clsNumOutstanding=%r\" % (self._clsNumOutstanding,)\n\n self._releaser = None\n self.cursor = None\n self.dbConn = None\n self._creationTracebackString = None\n self._addedToInstanceSet = False\n self._logger = None\n return"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck for concurrency violation and add self to _clsOutstandingInstances.", "response": "def _trackInstanceAndCheckForConcurrencyViolation(self):\n \"\"\" Check for concurrency violation and add self to\n _clsOutstandingInstances.\n\n ASSUMPTION: Called from constructor BEFORE _clsNumOutstanding is\n incremented\n \"\"\"\n global g_max_concurrency, g_max_concurrency_raise_exception\n\n assert g_max_concurrency is not None\n assert self not in self._clsOutstandingInstances, repr(self)\n\n # Populate diagnostic info\n self._creationTracebackString = traceback.format_stack()\n\n # Check for concurrency violation\n if self._clsNumOutstanding >= g_max_concurrency:\n # NOTE: It's possible for _clsNumOutstanding to be greater than\n # len(_clsOutstandingInstances) if concurrency check was enabled after\n # unrelease allocations.\n errorMsg = (\"With numOutstanding=%r, exceeded concurrency limit=%r \"\n \"when requesting %r. OTHER TRACKED UNRELEASED \"\n \"INSTANCES (%s): %r\") % (\n self._clsNumOutstanding, g_max_concurrency, self,\n len(self._clsOutstandingInstances), self._clsOutstandingInstances,)\n\n self._logger.error(errorMsg)\n\n if g_max_concurrency_raise_exception:\n raise ConcurrencyExceededError(errorMsg)\n\n\n # Add self to tracked instance set\n self._clsOutstandingInstances.add(self)\n self._addedToInstanceSet = True\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef close(self):\n self._logger.info(\"Closing\")\n if self._conn is not None:\n self._conn.close()\n self._conn = None\n else:\n self._logger.warning(\n \"close() called, but connection policy was alredy closed\")\n return", "response": "Closes the connection to the shared database."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a ConnectionWrapper instance that can be used to acquire a connection to the database.", "response": "def acquireConnection(self):\n \"\"\" Get a Connection instance.\n\n Parameters:\n ----------------------------------------------------------------\n retval: A ConnectionWrapper instance. NOTE: Caller\n is responsible for calling the ConnectionWrapper\n instance's release() method or use it in a context manager\n expression (with ... as:) to release resources.\n \"\"\"\n self._logger.debug(\"Acquiring connection\")\n\n # Check connection and attempt to re-establish it if it died (this is\n # what PooledDB does)\n self._conn._ping_check()\n connWrap = ConnectionWrapper(dbConn=self._conn,\n cursor=self._conn.cursor(),\n releaser=self._releaseConnection,\n logger=self._logger)\n return connWrap"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef close(self):\n self._logger.info(\"Closing\")\n\n if self._pool is not None:\n self._pool.close()\n self._pool = None\n else:\n self._logger.warning(\n \"close() called, but connection policy was alredy closed\")\n return", "response": "Closes the connection pool and the database connection pool."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a ConnectionWrapper instance that can be used to acquire a connection from the pool.", "response": "def acquireConnection(self):\n \"\"\" Get a connection from the pool.\n\n Parameters:\n ----------------------------------------------------------------\n retval: A ConnectionWrapper instance. NOTE: Caller\n is responsible for calling the ConnectionWrapper\n instance's release() method or use it in a context manager\n expression (with ... as:) to release resources.\n \"\"\"\n self._logger.debug(\"Acquiring connection\")\n\n dbConn = self._pool.connection(shareable=False)\n connWrap = ConnectionWrapper(dbConn=dbConn,\n cursor=dbConn.cursor(),\n releaser=self._releaseConnection,\n logger=self._logger)\n return connWrap"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncloses the connection policy instance.", "response": "def close(self):\n \"\"\" Close the policy instance. \"\"\"\n self._logger.info(\"Closing\")\n\n if self._opened:\n self._opened = False\n else:\n self._logger.warning(\n \"close() called, but connection policy was alredy closed\")\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a Connection instance and returns it.", "response": "def acquireConnection(self):\n \"\"\" Create a Connection instance.\n\n Parameters:\n ----------------------------------------------------------------\n retval: A ConnectionWrapper instance. NOTE: Caller\n is responsible for calling the ConnectionWrapper\n instance's release() method or use it in a context manager\n expression (with ... as:) to release resources.\n \"\"\"\n self._logger.debug(\"Acquiring connection\")\n\n dbConn = SteadyDB.connect(** _getCommonSteadyDBArgsDict())\n connWrap = ConnectionWrapper(dbConn=dbConn,\n cursor=dbConn.cursor(),\n releaser=self._releaseConnection,\n logger=self._logger)\n return connWrap"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrelease the connection and cursor", "response": "def _releaseConnection(self, dbConn, cursor):\n \"\"\" Release database connection and cursor; passed as a callback to\n ConnectionWrapper\n \"\"\"\n self._logger.debug(\"Releasing connection\")\n\n # Close the cursor\n cursor.close()\n\n # ... then close the database connection\n dbConn.close()\n return"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getSpec(cls):\n ns = dict(\n description=KNNAnomalyClassifierRegion.__doc__,\n singleNodeOnly=True,\n inputs=dict(\n spBottomUpOut=dict(\n description=\"\"\"The output signal generated from the bottom-up inputs\n from lower levels.\"\"\",\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=False,\n isDefaultInput=True,\n requireSplitterMap=False),\n\n tpTopDownOut=dict(\n description=\"\"\"The top-down inputsignal, generated from\n feedback from upper levels\"\"\",\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=False,\n isDefaultInput=True,\n requireSplitterMap=False),\n\n tpLrnActiveStateT=dict(\n description=\"\"\"Active cells in the learn state at time T from TM.\n This is used to classify on.\"\"\",\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=False,\n isDefaultInput=True,\n requireSplitterMap=False),\n\n sequenceIdIn=dict(\n description=\"Sequence ID\",\n dataType='UInt64',\n count=1,\n required=False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n ),\n\n outputs=dict(\n ),\n\n parameters=dict(\n trainRecords=dict(\n description='Number of records to wait for training',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Create'),\n\n anomalyThreshold=dict(\n description='Threshold used to classify anomalies.',\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Create'),\n\n cacheSize=dict(\n description='Number of records to store in cache.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Create'),\n\n classificationVectorType=dict(\n description=\"\"\"Vector type to use when classifying.\n 1 - Vector Column with Difference (TM and SP)\n \"\"\",\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=1,\n accessMode='ReadWrite'),\n\n activeColumnCount=dict(\n description=\"\"\"Number of active columns in a given step. Typically\n equivalent to SP.numActiveColumnsPerInhArea\"\"\",\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=40,\n accessMode='ReadWrite'),\n\n classificationMaxDist=dict(\n description=\"\"\"Maximum distance a sample can be from an anomaly\n in the classifier to be labeled as an anomaly.\n\n Ex: With rawOverlap distance, a value of 0.65 means that the points\n must be at most a distance 0.65 apart from each other. This\n translates to they must be at least 35% similar.\"\"\",\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=0.65,\n accessMode='Create'\n )\n ),\n commands=dict(\n getLabels=dict(description=\n \"Returns a list of label dicts with properties ROWID and labels.\"\n \"ROWID corresponds to the records id and labels is a list of \"\n \"strings representing the records labels. Takes additional \"\n \"integer properties start and end representing the range that \"\n \"will be returned.\"),\n\n addLabel=dict(description=\n \"Takes parameters start, end and labelName. Adds the label \"\n \"labelName to the records from start to end. This will recalculate \"\n \"labels from end to the most recent record.\"),\n\n removeLabels=dict(description=\n \"Takes additional parameters start, end, labelFilter. Start and \"\n \"end correspond to range to remove the label. Remove labels from \"\n \"each record with record ROWID in range from start to end, \"\n \"noninclusive of end. Removes all records if labelFilter is None, \"\n \"otherwise only removes the labels eqaul to labelFilter.\")\n )\n )\n\n ns['parameters'].update(KNNClassifierRegion.getSpec()['parameters'])\n\n return ns", "response": "Returns a dictionary of all the properties of the KNNAnomalyClassifierRegion class."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute(self, inputs, outputs):\n record = self._constructClassificationRecord(inputs)\n\n #Classify this point after waiting the classification delay\n if record.ROWID >= self.getParameter('trainRecords'):\n self._classifyState(record)\n\n #Save new classification record and keep history as moving window\n self._recordsCache.append(record)\n while len(self._recordsCache) > self.cacheSize:\n self._recordsCache.pop(0)\n\n self.labelResults = record.anomalyLabel\n\n self._iteration += 1", "response": "This method is called by the runtime engine to process one input sample."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconstructing a _HTMClassificationRecord based on the state of the model and the state of the model.", "response": "def _constructClassificationRecord(self, inputs):\n \"\"\"\n Construct a _HTMClassificationRecord based on the state of the model\n passed in through the inputs.\n\n Types for self.classificationVectorType:\n 1 - TM active cells in learn state\n 2 - SP columns concatenated with error from TM column predictions and SP\n \"\"\"\n # Count the number of unpredicted columns\n allSPColumns = inputs[\"spBottomUpOut\"]\n activeSPColumns = allSPColumns.nonzero()[0]\n\n score = anomaly.computeRawAnomalyScore(activeSPColumns,\n self._prevPredictedColumns)\n\n spSize = len(allSPColumns)\n\n\n allTPCells = inputs['tpTopDownOut']\n tpSize = len(inputs['tpLrnActiveStateT'])\n\n classificationVector = numpy.array([])\n\n if self.classificationVectorType == 1:\n # Classification Vector: [---TM Cells---]\n classificationVector = numpy.zeros(tpSize)\n activeCellMatrix = inputs[\"tpLrnActiveStateT\"].reshape(tpSize, 1)\n activeCellIdx = numpy.where(activeCellMatrix > 0)[0]\n if activeCellIdx.shape[0] > 0:\n classificationVector[numpy.array(activeCellIdx, dtype=numpy.uint16)] = 1\n elif self.classificationVectorType == 2:\n # Classification Vecotr: [---SP---|---(TM-SP)----]\n classificationVector = numpy.zeros(spSize+spSize)\n if activeSPColumns.shape[0] > 0:\n classificationVector[activeSPColumns] = 1.0\n\n errorColumns = numpy.setdiff1d(self._prevPredictedColumns,\n activeSPColumns)\n if errorColumns.shape[0] > 0:\n errorColumnIndexes = ( numpy.array(errorColumns, dtype=numpy.uint16) +\n spSize )\n classificationVector[errorColumnIndexes] = 1.0\n else:\n raise TypeError(\"Classification vector type must be either 'tpc' or\"\n \" 'sp_tpe', current value is %s\" % (self.classificationVectorType))\n\n # Store the state for next time step\n numPredictedCols = len(self._prevPredictedColumns)\n predictedColumns = allTPCells.nonzero()[0]\n self._prevPredictedColumns = copy.deepcopy(predictedColumns)\n\n if self._anomalyVectorLength is None:\n self._anomalyVectorLength = len(classificationVector)\n\n result = _CLAClassificationRecord(\n ROWID=self._iteration, #__numRunCalls called\n #at beginning of model.run\n anomalyScore=score,\n anomalyVector=classificationVector.nonzero()[0].tolist(),\n anomalyLabel=[]\n )\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _addRecordToKNN(self, record):\n knn = self._knnclassifier._knn\n\n prototype_idx = self._knnclassifier.getParameter('categoryRecencyList')\n category = self._labelListToCategoryNumber(record.anomalyLabel)\n\n # If record is already in the classifier, overwrite its labeling\n if record.ROWID in prototype_idx:\n knn.prototypeSetCategory(record.ROWID, category)\n return\n\n # Learn this pattern in the knn\n pattern = self._getStateAnomalyVector(record)\n rowID = record.ROWID\n knn.learn(pattern, category, rowID=rowID)", "response": "Adds the record to the KNN classifier."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _deleteRecordsFromKNN(self, recordsToDelete):\n prototype_idx = self._knnclassifier.getParameter('categoryRecencyList')\n\n idsToDelete = ([r.ROWID for r in recordsToDelete if\n not r.setByUser and r.ROWID in prototype_idx])\n\n nProtos = self._knnclassifier._knn._numPatterns\n self._knnclassifier._knn.removeIds(idsToDelete)\n assert self._knnclassifier._knn._numPatterns == nProtos - len(idsToDelete)", "response": "Removes the given records from the KNN."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndeletes any stored records within the given range from the KNN.", "response": "def _deleteRangeFromKNN(self, start=0, end=None):\n \"\"\"\n Removes any stored records within the range from start to\n end. Noninclusive of end.\n\n parameters\n ------------\n start - integer representing the ROWID of the start of the deletion range,\n end - integer representing the ROWID of the end of the deletion range,\n if None, it will default to end.\n \"\"\"\n prototype_idx = numpy.array(\n self._knnclassifier.getParameter('categoryRecencyList'))\n\n if end is None:\n end = prototype_idx.max() + 1\n\n idsIdxToDelete = numpy.logical_and(prototype_idx >= start,\n prototype_idx < end)\n idsToDelete = prototype_idx[idsIdxToDelete]\n\n nProtos = self._knnclassifier._knn._numPatterns\n self._knnclassifier._knn.removeIds(idsToDelete.tolist())\n assert self._knnclassifier._knn._numPatterns == nProtos - len(idsToDelete)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _recomputeRecordFromKNN(self, record):\n inputs = {\n \"categoryIn\": [None],\n \"bottomUpIn\": self._getStateAnomalyVector(record),\n }\n\n outputs = {\"categoriesOut\": numpy.zeros((1,)),\n \"bestPrototypeIndices\":numpy.zeros((1,)),\n \"categoryProbabilitiesOut\":numpy.zeros((1,))}\n\n # Only use points before record to classify and after the wait period.\n classifier_indexes = numpy.array(\n self._knnclassifier.getParameter('categoryRecencyList'))\n valid_idx = numpy.where(\n (classifier_indexes >= self.getParameter('trainRecords')) &\n (classifier_indexes < record.ROWID)\n )[0].tolist()\n\n if len(valid_idx) == 0:\n return None\n\n self._knnclassifier.setParameter('inferenceMode', None, True)\n self._knnclassifier.setParameter('learningMode', None, False)\n self._knnclassifier.compute(inputs, outputs)\n self._knnclassifier.setParameter('learningMode', None, True)\n\n classifier_distances = self._knnclassifier.getLatestDistances()\n valid_distances = classifier_distances[valid_idx]\n if valid_distances.min() <= self._classificationMaxDist:\n classifier_indexes_prev = classifier_indexes[valid_idx]\n rowID = classifier_indexes_prev[valid_distances.argmin()]\n indexID = numpy.where(classifier_indexes == rowID)[0][0]\n category = self._knnclassifier.getCategoryList()[indexID]\n return category\n return None", "response": "Recomputes the record from KNN and returns the classified labeling of record"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting a label to a category number.", "response": "def _labelToCategoryNumber(self, label):\n \"\"\"\n Since the KNN Classifier stores categories as numbers, we must store each\n label as a number. This method converts from a label to a unique number.\n Each label is assigned a unique bit so multiple labels may be assigned to\n a single record.\n \"\"\"\n if label not in self.saved_categories:\n self.saved_categories.append(label)\n return pow(2, self.saved_categories.index(label))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _categoryToLabelList(self, category):\n if category is None:\n return []\n\n labelList = []\n labelNum = 0\n while category > 0:\n if category % 2 == 1:\n labelList.append(self.saved_categories[labelNum])\n labelNum += 1\n category = category >> 1\n return labelList", "response": "Converts a category number into a list of labels"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _getStateAnomalyVector(self, state):\n vector = numpy.zeros(self._anomalyVectorLength)\n vector[state.anomalyVector] = 1\n return vector", "response": "Returns a state s anomaly vector converting it from spare to dense\n "} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the labels on classified points within the specified range.", "response": "def getLabels(self, start=None, end=None):\n \"\"\"\n Get the labels on classified points within range start to end. Not inclusive\n of end.\n\n :returns: (dict) with format:\n\n ::\n\n {\n 'isProcessing': boolean,\n 'recordLabels': list of results\n }\n\n ``isProcessing`` - currently always false as recalculation blocks; used if\n reprocessing of records is still being performed;\n\n Each item in ``recordLabels`` is of format:\n \n ::\n \n {\n 'ROWID': id of the row,\n 'labels': list of strings\n }\n\n \"\"\"\n if len(self._recordsCache) == 0:\n return {\n 'isProcessing': False,\n 'recordLabels': []\n }\n try:\n start = int(start)\n except Exception:\n start = 0\n\n try:\n end = int(end)\n except Exception:\n end = self._recordsCache[-1].ROWID\n\n if end <= start:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for 'getLabels'.\",\n debugInfo={\n 'requestRange': {\n 'startRecordID': start,\n 'endRecordID': end\n },\n 'numRecordsStored': len(self._recordsCache)\n })\n\n results = {\n 'isProcessing': False,\n 'recordLabels': []\n }\n\n ROWIDX = numpy.array(\n self._knnclassifier.getParameter('categoryRecencyList'))\n validIdx = numpy.where((ROWIDX >= start) & (ROWIDX < end))[0].tolist()\n categories = self._knnclassifier.getCategoryList()\n for idx in validIdx:\n row = dict(\n ROWID=int(ROWIDX[idx]),\n labels=self._categoryToLabelList(categories[idx]))\n results['recordLabels'].append(row)\n\n return results"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds a label to each record in the classifier s internal cache.", "response": "def addLabel(self, start, end, labelName):\n \"\"\"\n Add the label labelName to each record with record ROWID in range from\n ``start`` to ``end``, noninclusive of end.\n\n This will recalculate all points from end to the last record stored in the\n internal cache of this classifier.\n\n :param start: (int) start index \n :param end: (int) end index (noninclusive)\n :param labelName: (string) label name\n \"\"\"\n if len(self._recordsCache) == 0:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for 'addLabel'. \"\n \"Model has no saved records.\")\n\n try:\n start = int(start)\n except Exception:\n start = 0\n\n try:\n end = int(end)\n except Exception:\n end = int(self._recordsCache[-1].ROWID)\n\n startID = self._recordsCache[0].ROWID\n\n clippedStart = max(0, start - startID)\n clippedEnd = max(0, min( len( self._recordsCache) , end - startID))\n\n if clippedEnd <= clippedStart:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for 'addLabel'.\",\n debugInfo={\n 'requestRange': {\n 'startRecordID': start,\n 'endRecordID': end\n },\n 'clippedRequestRange': {\n 'startRecordID': clippedStart,\n 'endRecordID': clippedEnd\n },\n 'validRange': {\n 'startRecordID': startID,\n 'endRecordID': self._recordsCache[len(self._recordsCache)-1].ROWID\n },\n 'numRecordsStored': len(self._recordsCache)\n })\n\n # Add label to range [clippedStart, clippedEnd)\n for state in self._recordsCache[clippedStart:clippedEnd]:\n if labelName not in state.anomalyLabel:\n state.anomalyLabel.append(labelName)\n state.setByUser = True\n self._addRecordToKNN(state)\n\n assert len(self.saved_categories) > 0\n\n # Recompute [end, ...)\n for state in self._recordsCache[clippedEnd:]:\n self._classifyState(state)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving all records from the specified start to end.", "response": "def removeLabels(self, start=None, end=None, labelFilter=None):\n \"\"\"\n Remove labels from each record with record ROWID in range from\n ``start`` to ``end``, noninclusive of end. Removes all records if \n ``labelFilter`` is None, otherwise only removes the labels equal to \n ``labelFilter``.\n\n This will recalculate all points from end to the last record stored in the\n internal cache of this classifier.\n \n :param start: (int) start index \n :param end: (int) end index (noninclusive)\n :param labelFilter: (string) label filter\n \"\"\"\n if len(self._recordsCache) == 0:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for \"\n \"'removeLabels'. Model has no saved records.\")\n\n try:\n start = int(start)\n except Exception:\n start = 0\n\n try:\n end = int(end)\n except Exception:\n end = self._recordsCache[-1].ROWID\n\n startID = self._recordsCache[0].ROWID\n\n clippedStart = 0 if start is None else max(0, start - startID)\n clippedEnd = len(self._recordsCache) if end is None else \\\n max(0, min( len( self._recordsCache) , end - startID))\n\n if clippedEnd <= clippedStart:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for \"\n \"'removeLabels'.\", debugInfo={\n 'requestRange': {\n 'startRecordID': start,\n 'endRecordID': end\n },\n 'clippedRequestRange': {\n 'startRecordID': clippedStart,\n 'endRecordID': clippedEnd\n },\n 'validRange': {\n 'startRecordID': startID,\n 'endRecordID': self._recordsCache[len(self._recordsCache)-1].ROWID\n },\n 'numRecordsStored': len(self._recordsCache)\n })\n\n # Remove records within the cache\n recordsToDelete = []\n for state in self._recordsCache[clippedStart:clippedEnd]:\n if labelFilter is not None:\n if labelFilter in state.anomalyLabel:\n state.anomalyLabel.remove(labelFilter)\n else:\n state.anomalyLabel = []\n state.setByUser = False\n recordsToDelete.append(state)\n self._deleteRecordsFromKNN(recordsToDelete)\n\n # Remove records not in cache\n self._deleteRangeFromKNN(start, end)\n\n # Recompute [clippedEnd, ...)\n for state in self._recordsCache[clippedEnd:]:\n self._classifyState(state)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef match(self, record):\n '''\n Returns True if the record matches any of the provided filters\n '''\n\n for field, meta in self.filterDict.iteritems():\n index = meta['index']\n categories = meta['categories']\n for category in categories:\n # Record might be blank, handle this\n if not record:\n continue\n if record[index].find(category) != -1:\n '''\n This field contains the string we're searching for\n so we'll keep the records\n '''\n return True\n\n # None of the categories were found in this record\n return False", "response": "Returns True if the record matches any of the filters\n Returns False otherwise"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef replace(self, columnIndex, bitmap):\n return super(_SparseMatrixCorticalColumnAdapter, self).replaceSparseRow(\n columnIndex, bitmap\n )", "response": "Replaces the row with the given bitmap."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update(self, columnIndex, vector):\n return super(_SparseMatrixCorticalColumnAdapter, self).setRowFromDense(\n columnIndex, vector\n )", "response": "Updates the row with the given vector."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting the local area density. Invalidates the numActiveColumnsPerInhArea parameter.", "response": "def setLocalAreaDensity(self, localAreaDensity):\n \"\"\"\n Sets the local area density. Invalidates the 'numActiveColumnsPerInhArea'\n parameter\n \n :param localAreaDensity: (float) value to set\n \"\"\"\n assert(localAreaDensity > 0 and localAreaDensity <= 1)\n self._localAreaDensity = localAreaDensity\n self._numActiveColumnsPerInhArea = 0"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget the potential for a given column.", "response": "def getPotential(self, columnIndex, potential):\n \"\"\"\n :param columnIndex: (int) column index to get potential for.\n :param potential: (list) will be overwritten with column potentials. Must \n match the number of inputs.\n \"\"\"\n assert(columnIndex < self._numColumns)\n potential[:] = self._potentialPools[columnIndex]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the potential mapping for a given column.", "response": "def setPotential(self, columnIndex, potential):\n \"\"\"\n Sets the potential mapping for a given column. ``potential`` size must match \n the number of inputs, and must be greater than ``stimulusThreshold``.\n \n :param columnIndex: (int) column index to set potential for.\n :param potential: (list) value to set.\n \"\"\"\n assert(columnIndex < self._numColumns)\n\n potentialSparse = numpy.where(potential > 0)[0]\n if len(potentialSparse) < self._stimulusThreshold:\n raise Exception(\"This is likely due to a \" +\n \"value of stimulusThreshold that is too large relative \" +\n \"to the input size.\")\n\n self._potentialPools.replace(columnIndex, potentialSparse)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getPermanence(self, columnIndex, permanence):\n assert(columnIndex < self._numColumns)\n permanence[:] = self._permanences[columnIndex]", "response": "Returns the permanence values for a given column."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef setPermanence(self, columnIndex, permanence):\n assert(columnIndex < self._numColumns)\n self._updatePermanencesForColumn(permanence, columnIndex, raisePerm=False)", "response": "Sets the permanence values for a given column."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a generator that yields the connected synapses for a given column.", "response": "def getConnectedSynapses(self, columnIndex, connectedSynapses):\n \"\"\"\n :param connectedSynapses: (list) will be overwritten\n :returns: (iter) the connected synapses for a given column.\n ``connectedSynapses`` size must match the number of inputs\"\"\"\n assert(columnIndex < self._numColumns)\n connectedSynapses[:] = self._connectedSynapses[columnIndex]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef stripUnlearnedColumns(self, activeArray):\n neverLearned = numpy.where(self._activeDutyCycles == 0)[0]\n activeArray[neverLearned] = 0", "response": "Removes unlearned columns from the set of active columns."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _updateMinDutyCycles(self):\n if self._globalInhibition or self._inhibitionRadius > self._numInputs:\n self._updateMinDutyCyclesGlobal()\n else:\n self._updateMinDutyCyclesLocal()", "response": "Updates the minimum duty cycles defining normal activity for a column."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating the minimum duty cycles for the current resource in a global fashion.", "response": "def _updateMinDutyCyclesGlobal(self):\n \"\"\"\n Updates the minimum duty cycles in a global fashion. Sets the minimum duty\n cycles for the overlap all columns to be a percent of the maximum in the\n region, specified by minPctOverlapDutyCycle. Functionality it is equivalent\n to _updateMinDutyCyclesLocal, but this function exploits the globality of\n the computation to perform it in a straightforward, and efficient manner.\n \"\"\"\n self._minOverlapDutyCycles.fill(\n self._minPctOverlapDutyCycles * self._overlapDutyCycles.max()\n )"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate the minimum duty cycles for the entire sequence.", "response": "def _updateMinDutyCyclesLocal(self):\n \"\"\"\n Updates the minimum duty cycles. The minimum duty cycles are determined\n locally. Each column's minimum duty cycles are set to be a percent of the\n maximum duty cycles in the column's neighborhood. Unlike\n _updateMinDutyCyclesGlobal, here the values can be quite different for\n different columns.\n \"\"\"\n for column in xrange(self._numColumns):\n neighborhood = self._getColumnNeighborhood(column)\n\n maxActiveDuty = self._activeDutyCycles[neighborhood].max()\n maxOverlapDuty = self._overlapDutyCycles[neighborhood].max()\n\n self._minOverlapDutyCycles[column] = (maxOverlapDuty *\n self._minPctOverlapDutyCycles)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nupdate the duty cycles for each column.", "response": "def _updateDutyCycles(self, overlaps, activeColumns):\n \"\"\"\n Updates the duty cycles for each column. The OVERLAP duty cycle is a moving\n average of the number of inputs which overlapped with the each column. The\n ACTIVITY duty cycles is a moving average of the frequency of activation for\n each column.\n\n Parameters:\n ----------------------------\n :param overlaps:\n An array containing the overlap score for each column.\n The overlap score for a column is defined as the number\n of synapses in a \"connected state\" (connected synapses)\n that are connected to input bits which are turned on.\n :param activeColumns:\n An array containing the indices of the active columns,\n the sparse set of columns which survived inhibition\n \"\"\"\n overlapArray = numpy.zeros(self._numColumns, dtype=realDType)\n activeArray = numpy.zeros(self._numColumns, dtype=realDType)\n overlapArray[overlaps > 0] = 1\n activeArray[activeColumns] = 1\n\n period = self._dutyCyclePeriod\n if (period > self._iterationNum):\n period = self._iterationNum\n\n self._overlapDutyCycles = self._updateDutyCyclesHelper(\n self._overlapDutyCycles,\n overlapArray,\n period\n )\n\n self._activeDutyCycles = self._updateDutyCyclesHelper(\n self._activeDutyCycles,\n activeArray,\n period\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _updateInhibitionRadius(self):\n if self._globalInhibition:\n self._inhibitionRadius = int(self._columnDimensions.max())\n return\n\n avgConnectedSpan = numpy.average(\n [self._avgConnectedSpanForColumnND(i)\n for i in xrange(self._numColumns)]\n )\n columnsPerInput = self._avgColumnsPerInput()\n diameter = avgConnectedSpan * columnsPerInput\n radius = (diameter - 1) / 2.0\n radius = max(1.0, radius)\n self._inhibitionRadius = int(radius + 0.5)", "response": "Update the inhibition radius of the inhibition of the input sequence."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _avgColumnsPerInput(self):\n #TODO: extend to support different number of dimensions for inputs and\n # columns\n numDim = max(self._columnDimensions.size, self._inputDimensions.size)\n colDim = numpy.ones(numDim)\n colDim[:self._columnDimensions.size] = self._columnDimensions\n\n inputDim = numpy.ones(numDim)\n inputDim[:self._inputDimensions.size] = self._inputDimensions\n\n columnsPerInput = colDim.astype(realDType) / inputDim\n return numpy.average(columnsPerInput)", "response": "Returns the average number of columns per input."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _avgConnectedSpanForColumn1D(self, columnIndex):\n assert(self._inputDimensions.size == 1)\n connected = self._connectedSynapses[columnIndex].nonzero()[0]\n if connected.size == 0:\n return 0\n else:\n return max(connected) - min(connected) + 1", "response": "This function calculates the average connected span for a single column."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _avgConnectedSpanForColumn2D(self, columnIndex):\n assert(self._inputDimensions.size == 2)\n connected = self._connectedSynapses[columnIndex]\n (rows, cols) = connected.reshape(self._inputDimensions).nonzero()\n if rows.size == 0 and cols.size == 0:\n return 0\n rowSpan = rows.max() - rows.min() + 1\n colSpan = cols.max() - cols.min() + 1\n return numpy.average([rowSpan, colSpan])", "response": "Returns the average connected span for a given column."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _bumpUpWeakColumns(self):\n weakColumns = numpy.where(self._overlapDutyCycles\n < self._minOverlapDutyCycles)[0]\n for columnIndex in weakColumns:\n perm = self._permanences[columnIndex].astype(realDType)\n maskPotential = numpy.where(self._potentialPools[columnIndex] > 0)[0]\n perm[maskPotential] += self._synPermBelowStimulusInc\n self._updatePermanencesForColumn(perm, columnIndex, raisePerm=False)", "response": "This method increments the permanence values of synapses of columns whose activity level is too low."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nraise an exception if the permanence value of each input bit is too large relative to the input size.", "response": "def _raisePermanenceToThreshold(self, perm, mask):\n \"\"\"\n This method ensures that each column has enough connections to input bits\n to allow it to become active. Since a column must have at least\n 'self._stimulusThreshold' overlaps in order to be considered during the\n inhibition phase, columns without such minimal number of connections, even\n if all the input bits they are connected to turn on, have no chance of\n obtaining the minimum threshold. For such columns, the permanence values\n are increased until the minimum number of connections are formed.\n\n\n Parameters:\n ----------------------------\n :param perm: An array of permanence values for a column. The array is\n \"dense\", i.e. it contains an entry for each input bit, even\n if the permanence value is 0.\n :param mask: the indices of the columns whose permanences need to be\n raised.\n \"\"\"\n if len(mask) < self._stimulusThreshold:\n raise Exception(\"This is likely due to a \" +\n \"value of stimulusThreshold that is too large relative \" +\n \"to the input size. [len(mask) < self._stimulusThreshold]\")\n\n numpy.clip(perm, self._synPermMin, self._synPermMax, out=perm)\n while True:\n numConnected = numpy.nonzero(\n perm > self._synPermConnected - PERMANENCE_EPSILON)[0].size\n\n if numConnected >= self._stimulusThreshold:\n return\n perm[mask] += self._synPermBelowStimulusInc"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _updatePermanencesForColumn(self, perm, columnIndex, raisePerm=True):\n maskPotential = numpy.where(self._potentialPools[columnIndex] > 0)[0]\n if raisePerm:\n self._raisePermanenceToThreshold(perm, maskPotential)\n perm[perm < self._synPermTrimThreshold] = 0\n numpy.clip(perm, self._synPermMin, self._synPermMax, out=perm)\n newConnected = numpy.where(perm >=\n self._synPermConnected - PERMANENCE_EPSILON)[0]\n self._permanences.update(columnIndex, perm)\n self._connectedSynapses.replace(columnIndex, newConnected)\n self._connectedCounts[columnIndex] = newConnected.size", "response": "This method updates the permanence matrix with the values of a column in the given permanence matrix."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a randomized permanence value for a synapses that is initialized in a connected state.", "response": "def _initPermConnected(self):\n \"\"\"\n Returns a randomly generated permanence value for a synapses that is\n initialized in a connected state. The basic idea here is to initialize\n permanence values very close to synPermConnected so that a small number of\n learning steps could make it disconnected or connected.\n\n Note: experimentation was done a long time ago on the best way to initialize\n permanence values, but the history for this particular scheme has been lost.\n \"\"\"\n p = self._synPermConnected + (\n self._synPermMax - self._synPermConnected)*self._random.getReal64()\n\n # Ensure we don't have too much unnecessary precision. A full 64 bits of\n # precision causes numerical stability issues across platforms and across\n # implementations\n p = int(p*100000) / 100000.0\n return p"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a randomized permanence value for a synapses that is to be initialized in a non - connected state.", "response": "def _initPermNonConnected(self):\n \"\"\"\n Returns a randomly generated permanence value for a synapses that is to be\n initialized in a non-connected state.\n \"\"\"\n p = self._synPermConnected * self._random.getReal64()\n\n # Ensure we don't have too much unnecessary precision. A full 64 bits of\n # precision causes numerical stability issues across platforms and across\n # implementations\n p = int(p*100000) / 100000.0\n return p"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes the permanences of a column.", "response": "def _initPermanence(self, potential, connectedPct):\n \"\"\"\n Initializes the permanences of a column. The method\n returns a 1-D array the size of the input, where each entry in the\n array represents the initial permanence value between the input bit\n at the particular index in the array, and the column represented by\n the 'index' parameter.\n\n Parameters:\n ----------------------------\n :param potential: A numpy array specifying the potential pool of the column.\n Permanence values will only be generated for input bits\n corresponding to indices for which the mask value is 1.\n :param connectedPct: A value between 0 or 1 governing the chance, for each\n permanence, that the initial permanence value will\n be a value that is considered connected.\n \"\"\"\n # Determine which inputs bits will start out as connected\n # to the inputs. Initially a subset of the input bits in a\n # column's potential pool will be connected. This number is\n # given by the parameter \"connectedPct\"\n perm = numpy.zeros(self._numInputs, dtype=realDType)\n for i in xrange(self._numInputs):\n if (potential[i] < 1):\n continue\n\n if (self._random.getReal64() <= connectedPct):\n perm[i] = self._initPermConnected()\n else:\n perm[i] = self._initPermNonConnected()\n\n # Clip off low values. Since we use a sparse representation\n # to store the permanence values this helps reduce memory\n # requirements.\n perm[perm < self._synPermTrimThreshold] = 0\n\n return perm"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nmapping a column to its respective input index, keeping to the topology of the region. It takes the index of the column as an argument and determines what is the index of the flattened input vector that is to be the center of the column's potential pool. It distributes the columns over the inputs uniformly. The return value is an integer representing the index of the input bit. Examples of the expected output of this method: * If the topology is one dimensional, and the column index is 0, this method will return the input index 0. If the column index is 1, and there are 3 columns over 7 inputs, this method will return the input index 3. * If the topology is two dimensional, with column dimensions [3, 5] and input dimensions [7, 11], and the column index is 3, the method returns input index 8. Parameters: ---------------------------- :param index: The index identifying a column in the permanence, potential and connectivity matrices. :param wrapAround: A boolean value indicating that boundaries should be ignored.", "response": "def _mapColumn(self, index):\n \"\"\"\n Maps a column to its respective input index, keeping to the topology of\n the region. It takes the index of the column as an argument and determines\n what is the index of the flattened input vector that is to be the center of\n the column's potential pool. It distributes the columns over the inputs\n uniformly. The return value is an integer representing the index of the\n input bit. Examples of the expected output of this method:\n * If the topology is one dimensional, and the column index is 0, this\n method will return the input index 0. If the column index is 1, and there\n are 3 columns over 7 inputs, this method will return the input index 3.\n * If the topology is two dimensional, with column dimensions [3, 5] and\n input dimensions [7, 11], and the column index is 3, the method\n returns input index 8.\n\n Parameters:\n ----------------------------\n :param index: The index identifying a column in the permanence, potential\n and connectivity matrices.\n :param wrapAround: A boolean value indicating that boundaries should be\n ignored.\n \"\"\"\n columnCoords = numpy.unravel_index(index, self._columnDimensions)\n columnCoords = numpy.array(columnCoords, dtype=realDType)\n ratios = columnCoords / self._columnDimensions\n inputCoords = self._inputDimensions * ratios\n inputCoords += 0.5 * self._inputDimensions / self._columnDimensions\n inputCoords = inputCoords.astype(int)\n inputIndex = numpy.ravel_multi_index(inputCoords, self._inputDimensions)\n return inputIndex"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _mapPotential(self, index):\n\n centerInput = self._mapColumn(index)\n columnInputs = self._getInputNeighborhood(centerInput).astype(uintType)\n\n # Select a subset of the receptive field to serve as the\n # the potential pool\n numPotential = int(columnInputs.size * self._potentialPct + 0.5)\n selectedInputs = numpy.empty(numPotential, dtype=uintType)\n self._random.sample(columnInputs, selectedInputs)\n\n potential = numpy.zeros(self._numInputs, dtype=uintType)\n potential[selectedInputs] = 1\n\n return potential", "response": "This method maps a column to its input bits."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating boost factors when global inhibition is used", "response": "def _updateBoostFactorsGlobal(self):\n \"\"\"\n Update boost factors when global inhibition is used\n \"\"\"\n # When global inhibition is enabled, the target activation level is\n # the sparsity of the spatial pooler\n if (self._localAreaDensity > 0):\n targetDensity = self._localAreaDensity\n else:\n inhibitionArea = ((2 * self._inhibitionRadius + 1)\n ** self._columnDimensions.size)\n inhibitionArea = min(self._numColumns, inhibitionArea)\n targetDensity = float(self._numActiveColumnsPerInhArea) / inhibitionArea\n targetDensity = min(targetDensity, 0.5)\n\n self._boostFactors = numpy.exp(\n (targetDensity - self._activeDutyCycles) * self._boostStrength)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates boost factors when local inhibition is used", "response": "def _updateBoostFactorsLocal(self):\n \"\"\"\n Update boost factors when local inhibition is used\n \"\"\"\n # Determine the target activation level for each column\n # The targetDensity is the average activeDutyCycles of the neighboring\n # columns of each column.\n targetDensity = numpy.zeros(self._numColumns, dtype=realDType)\n for i in xrange(self._numColumns):\n maskNeighbors = self._getColumnNeighborhood(i)\n targetDensity[i] = numpy.mean(self._activeDutyCycles[maskNeighbors])\n\n self._boostFactors = numpy.exp(\n (targetDensity - self._activeDutyCycles) * self._boostStrength)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nperform inhibition. This method calculates the necessary values needed to actually perform inhibition and then delegates the task of picking the active columns to helper functions. Parameters: ---------------------------- :param overlaps: an array containing the overlap score for each column. The overlap score for a column is defined as the number of synapses in a \"connected state\" (connected synapses) that are connected to input bits which are turned on.", "response": "def _inhibitColumns(self, overlaps):\n \"\"\"\n Performs inhibition. This method calculates the necessary values needed to\n actually perform inhibition and then delegates the task of picking the\n active columns to helper functions.\n\n Parameters:\n ----------------------------\n :param overlaps: an array containing the overlap score for each column.\n The overlap score for a column is defined as the number\n of synapses in a \"connected state\" (connected synapses)\n that are connected to input bits which are turned on.\n \"\"\"\n # determine how many columns should be selected in the inhibition phase.\n # This can be specified by either setting the 'numActiveColumnsPerInhArea'\n # parameter or the 'localAreaDensity' parameter when initializing the class\n if (self._localAreaDensity > 0):\n density = self._localAreaDensity\n else:\n inhibitionArea = ((2*self._inhibitionRadius + 1)\n ** self._columnDimensions.size)\n inhibitionArea = min(self._numColumns, inhibitionArea)\n density = float(self._numActiveColumnsPerInhArea) / inhibitionArea\n density = min(density, 0.5)\n\n if self._globalInhibition or \\\n self._inhibitionRadius > max(self._columnDimensions):\n return self._inhibitColumnsGlobal(overlaps, density)\n else:\n return self._inhibitColumnsLocal(overlaps, density)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _inhibitColumnsGlobal(self, overlaps, density):\n #calculate num active per inhibition area\n numActive = int(density * self._numColumns)\n\n # Calculate winners using stable sort algorithm (mergesort)\n # for compatibility with C++\n sortedWinnerIndices = numpy.argsort(overlaps, kind='mergesort')\n\n # Enforce the stimulus threshold\n start = len(sortedWinnerIndices) - numActive\n while start < len(sortedWinnerIndices):\n i = sortedWinnerIndices[start]\n if overlaps[i] >= self._stimulusThreshold:\n break\n else:\n start += 1\n\n return sortedWinnerIndices[start:][::-1]", "response": "This function performs global inhibition."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms local inhibition. Local inhibition is performed on a column by column basis. Each column observes the overlaps of its neighbors and is selected if its overlap score is within the top 'numActive' in its local neighborhood. At most half of the columns in a local neighborhood are allowed to be active. Columns with an overlap score below the 'stimulusThreshold' are always inhibited. :param overlaps: an array containing the overlap score for each column. The overlap score for a column is defined as the number of synapses in a \"connected state\" (connected synapses) that are connected to input bits which are turned on. :param density: The fraction of columns to survive inhibition. This value is only an intended target. Since the surviving columns are picked in a local fashion, the exact fraction of surviving columns is likely to vary. @return list with indices of the winning columns", "response": "def _inhibitColumnsLocal(self, overlaps, density):\n \"\"\"\n Performs local inhibition. Local inhibition is performed on a column by\n column basis. Each column observes the overlaps of its neighbors and is\n selected if its overlap score is within the top 'numActive' in its local\n neighborhood. At most half of the columns in a local neighborhood are\n allowed to be active. Columns with an overlap score below the\n 'stimulusThreshold' are always inhibited.\n\n :param overlaps: an array containing the overlap score for each column.\n The overlap score for a column is defined as the number\n of synapses in a \"connected state\" (connected synapses)\n that are connected to input bits which are turned on.\n :param density: The fraction of columns to survive inhibition. This\n value is only an intended target. Since the surviving\n columns are picked in a local fashion, the exact fraction\n of surviving columns is likely to vary.\n @return list with indices of the winning columns\n \"\"\"\n\n activeArray = numpy.zeros(self._numColumns, dtype=\"bool\")\n\n for column, overlap in enumerate(overlaps):\n if overlap >= self._stimulusThreshold:\n neighborhood = self._getColumnNeighborhood(column)\n neighborhoodOverlaps = overlaps[neighborhood]\n\n numBigger = numpy.count_nonzero(neighborhoodOverlaps > overlap)\n\n # When there is a tie, favor neighbors that are already selected as\n # active.\n ties = numpy.where(neighborhoodOverlaps == overlap)\n tiedNeighbors = neighborhood[ties]\n numTiesLost = numpy.count_nonzero(activeArray[tiedNeighbors])\n\n numActive = int(0.5 + density * len(neighborhood))\n if numBigger + numTiesLost < numActive:\n activeArray[column] = True\n\n return activeArray.nonzero()[0]"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _getColumnNeighborhood(self, centerColumn):\n if self._wrapAround:\n return topology.wrappingNeighborhood(centerColumn,\n self._inhibitionRadius,\n self._columnDimensions)\n\n else:\n return topology.neighborhood(centerColumn,\n self._inhibitionRadius,\n self._columnDimensions)", "response": "Returns a neighborhood of the columns in the specified center."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a neighborhood of inputs.", "response": "def _getInputNeighborhood(self, centerInput):\n \"\"\"\n Gets a neighborhood of inputs.\n\n Simply calls topology.wrappingNeighborhood or topology.neighborhood.\n\n A subclass can insert different topology behavior by overriding this method.\n\n :param centerInput (int)\n The center of the neighborhood.\n\n @returns (1D numpy array of integers)\n The inputs in the neighborhood.\n \"\"\"\n if self._wrapAround:\n return topology.wrappingNeighborhood(centerInput,\n self._potentialRadius,\n self._inputDimensions)\n else:\n return topology.neighborhood(centerInput,\n self._potentialRadius,\n self._inputDimensions)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing the random seed", "response": "def _seed(self, seed=-1):\n \"\"\"\n Initialize the random seed\n \"\"\"\n if seed != -1:\n self._random = NupicRandom(seed)\n else:\n self._random = NupicRandom()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef handleGetValue(self, topContainer):\n value = self.__referenceDict if self.__referenceDict is not None else topContainer\n for key in self.__dictKeyChain:\n value = value[key]\n\n return value", "response": "This method overrides ValueGetterBase s pure virtual method to return the referenced value."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning an array of the specified data type.", "response": "def Array(dtype, size=None, ref=False):\n \"\"\"Factory function that creates typed Array or ArrayRef objects\n\n dtype - the data type of the array (as string).\n Supported types are: Byte, Int16, UInt16, Int32, UInt32, Int64, UInt64, Real32, Real64\n\n size - the size of the array. Must be positive integer.\n \"\"\"\n\n def getArrayType(self):\n \"\"\"A little function to replace the getType() method of arrays\n\n It returns a string representation of the array element type instead of the\n integer value (NTA_BasicType enum) returned by the origianl array\n \"\"\"\n return self._dtype\n\n\n # ArrayRef can't be allocated\n if ref:\n assert size is None\n\n index = basicTypes.index(dtype)\n if index == -1:\n raise Exception('Invalid data type: ' + dtype)\n if size and size <= 0:\n raise Exception('Array size must be positive')\n suffix = 'ArrayRef' if ref else 'Array'\n arrayFactory = getattr(engine_internal, dtype + suffix)\n arrayFactory.getType = getArrayType\n\n if size:\n a = arrayFactory(size)\n else:\n a = arrayFactory()\n\n a._dtype = basicTypes[index]\n return a"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn list of input names in the", "response": "def getInputNames(self):\n \"\"\"\n Returns list of input names in spec.\n \"\"\"\n inputs = self.getSpec().inputs\n return [inputs.getByIndex(i)[0] for i in xrange(inputs.getCount())]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getOutputNames(self):\n outputs = self.getSpec().outputs\n return [outputs.getByIndex(i)[0] for i in xrange(outputs.getCount())]", "response": "Returns list of output names in the\n "} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the set and get functions to set or get the parameter.", "response": "def _getParameterMethods(self, paramName):\n \"\"\"Returns functions to set/get the parameter. These are\n the strongly typed functions get/setParameterUInt32, etc.\n The return value is a pair:\n setfunc, getfunc\n If the parameter is not available on this region, setfunc/getfunc\n are None. \"\"\"\n if paramName in self._paramTypeCache:\n return self._paramTypeCache[paramName]\n try:\n # Catch the error here. We will re-throw in getParameter or\n # setParameter with a better error message than we could generate here\n paramSpec = self.getSpec().parameters.getByName(paramName)\n except:\n return (None, None)\n dataType = paramSpec.dataType\n dataTypeName = basicTypes[dataType]\n count = paramSpec.count\n if count == 1:\n # Dynamically generate the proper typed get/setParameter\n x = 'etParameter' + dataTypeName\n try:\n g = getattr(self, 'g' + x) # get the typed getParameter method\n s = getattr(self, 's' + x) # get the typed setParameter method\n except AttributeError:\n raise Exception(\"Internal error: unknown parameter type %s\" %\n dataTypeName)\n info = (s, g)\n else:\n if dataTypeName == \"Byte\":\n info = (self.setParameterString, self.getParameterString)\n else:\n helper = _ArrayParameterHelper(self, dataType)\n info = (self.setParameterArray, helper.getParameterArray)\n\n self._paramTypeCache[paramName] = info\n return info"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getParameter(self, paramName):\n (setter, getter) = self._getParameterMethods(paramName)\n if getter is None:\n import exceptions\n raise exceptions.Exception(\n \"getParameter -- parameter name '%s' does not exist in region %s of type %s\"\n % (paramName, self.name, self.type))\n return getter(paramName)", "response": "Get a parameter value from the current object"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the value of a parameter in the current region.", "response": "def setParameter(self, paramName, value):\n \"\"\"Set parameter value\"\"\"\n (setter, getter) = self._getParameterMethods(paramName)\n if setter is None:\n import exceptions\n raise exceptions.Exception(\n \"setParameter -- parameter name '%s' does not exist in region %s of type %s\"\n % (paramName, self.name, self.type))\n setter(paramName, value)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the collection of regions in a network", "response": "def _getRegions(self):\n \"\"\"Get the collection of regions in a network\n\n This is a tricky one. The collection of regions returned from\n from the internal network is a collection of internal regions.\n The desired collection is a collelcion of net.Region objects\n that also points to this network (net.network) and not to\n the internal network. To achieve that a CollectionWrapper\n class is used with a custom makeRegion() function (see bellow)\n as a value wrapper. The CollectionWrapper class wraps each value in the\n original collection with the result of the valueWrapper.\n \"\"\"\n\n def makeRegion(name, r):\n \"\"\"Wrap a engine region with a nupic.engine_internal.Region\n\n Also passes the containing nupic.engine_internal.Network network in _network. This\n function is passed a value wrapper to the CollectionWrapper\n \"\"\"\n r = Region(r, self)\n #r._network = self\n return r\n\n regions = CollectionWrapper(engine_internal.Network.getRegions(self), makeRegion)\n return regions"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getRegionsByType(self, regionClass):\n regions = []\n\n for region in self.regions.values():\n if type(region.getSelf()) is regionClass:\n regions.append(region)\n\n return regions", "response": "Returns all regions of a given class"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getSpec(cls):\n ns = dict(\n description=SDRClassifierRegion.__doc__,\n singleNodeOnly=True,\n\n inputs=dict(\n actValueIn=dict(\n description=\"Actual value of the field to predict. Only taken \"\n \"into account if the input has no category field.\",\n dataType=\"Real32\",\n count=0,\n required=False,\n regionLevel=False,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n bucketIdxIn=dict(\n description=\"Active index of the encoder bucket for the \"\n \"actual value of the field to predict. Only taken \"\n \"into account if the input has no category field.\",\n dataType=\"UInt64\",\n count=0,\n required=False,\n regionLevel=False,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n categoryIn=dict(\n description='Vector of categories of the input sample',\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n bottomUpIn=dict(\n description='Belief values over children\\'s groups',\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=False,\n isDefaultInput=True,\n requireSplitterMap=False),\n\n predictedActiveCells=dict(\n description=\"The cells that are active and predicted\",\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n sequenceIdIn=dict(\n description=\"Sequence ID\",\n dataType='UInt64',\n count=1,\n required=False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n ),\n\n outputs=dict(\n categoriesOut=dict(\n description='Classification results',\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False,\n requireSplitterMap=False),\n\n actualValues=dict(\n description='Classification results',\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False,\n requireSplitterMap=False),\n\n probabilities=dict(\n description='Classification results',\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False,\n requireSplitterMap=False),\n ),\n\n parameters=dict(\n learningMode=dict(\n description='Boolean (0/1) indicating whether or not a region '\n 'is in learning mode.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=1,\n accessMode='ReadWrite'),\n\n inferenceMode=dict(\n description='Boolean (0/1) indicating whether or not a region '\n 'is in inference mode.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='ReadWrite'),\n\n maxCategoryCount=dict(\n description='The maximal number of categories the '\n 'classifier will distinguish between.',\n dataType='UInt32',\n required=True,\n count=1,\n constraints='',\n # arbitrarily large value\n defaultValue=2000,\n accessMode='Create'),\n\n steps=dict(\n description='Comma separated list of the desired steps of '\n 'prediction that the classifier should learn',\n dataType=\"Byte\",\n count=0,\n constraints='',\n defaultValue='0',\n accessMode='Create'),\n\n alpha=dict(\n description='The alpha is the learning rate of the classifier.'\n 'lower alpha results in longer term memory and slower '\n 'learning',\n dataType=\"Real32\",\n count=1,\n constraints='',\n defaultValue=0.001,\n accessMode='Create'),\n\n implementation=dict(\n description='The classifier implementation to use.',\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints='enum: py, cpp'),\n\n verbosity=dict(\n description='An integer that controls the verbosity level, '\n '0 means no verbose output, increasing integers '\n 'provide more verbosity.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='ReadWrite'),\n ),\n commands=dict()\n )\n\n return ns", "response": "Returns a new SDRClassifierRegion object that is used by the getSpec method of the base class."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninitializes the internal state of the internal state of the internal state of the internal state.", "response": "def initialize(self):\n \"\"\"\n Overrides :meth:`nupic.bindings.regions.PyRegion.PyRegion.initialize`.\n\n Is called once by NuPIC before the first call to compute().\n Initializes self._sdrClassifier if it is not already initialized.\n \"\"\"\n if self._sdrClassifier is None:\n self._sdrClassifier = SDRClassifierFactory.create(\n steps=self.stepsList,\n alpha=self.alpha,\n verbosity=self.verbosity,\n implementation=self.implementation,\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef writeToProto(self, proto):\n proto.implementation = self.implementation\n proto.steps = self.steps\n proto.alpha = self.alpha\n proto.verbosity = self.verbosity\n proto.maxCategoryCount = self.maxCategoryCount\n proto.learningMode = self.learningMode\n proto.inferenceMode = self.inferenceMode\n proto.recordNum = self.recordNum\n\n self._sdrClassifier.write(proto.sdrClassifier)", "response": "Writes the state of this capnproto object to the specified proto object."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef readFromProto(cls, proto):\n instance = cls()\n\n instance.implementation = proto.implementation\n instance.steps = proto.steps\n instance.stepsList = [int(i) for i in proto.steps.split(\",\")]\n instance.alpha = proto.alpha\n instance.verbosity = proto.verbosity\n instance.maxCategoryCount = proto.maxCategoryCount\n\n instance._sdrClassifier = SDRClassifierFactory.read(proto)\n\n instance.learningMode = proto.learningMode\n instance.inferenceMode = proto.inferenceMode\n instance.recordNum = proto.recordNum\n\n return instance", "response": "Reads state from proto object."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef compute(self, inputs, outputs):\n\n # This flag helps to prevent double-computation, in case the deprecated\n # customCompute() method is being called in addition to compute() called\n # when network.run() is called\n self._computeFlag = True\n\n patternNZ = inputs[\"bottomUpIn\"].nonzero()[0]\n\n if self.learningMode:\n # An input can potentially belong to multiple categories.\n # If a category value is < 0, it means that the input does not belong to\n # that category.\n categories = [category for category in inputs[\"categoryIn\"]\n if category >= 0]\n\n if len(categories) > 0:\n # Allow to train on multiple input categories.\n bucketIdxList = []\n actValueList = []\n for category in categories:\n bucketIdxList.append(int(category))\n if \"actValueIn\" not in inputs:\n actValueList.append(int(category))\n else:\n actValueList.append(float(inputs[\"actValueIn\"]))\n\n classificationIn = {\"bucketIdx\": bucketIdxList,\n \"actValue\": actValueList}\n else:\n # If the input does not belong to a category, i.e. len(categories) == 0,\n # then look for bucketIdx and actValueIn.\n if \"bucketIdxIn\" not in inputs:\n raise KeyError(\"Network link missing: bucketIdxOut -> bucketIdxIn\")\n if \"actValueIn\" not in inputs:\n raise KeyError(\"Network link missing: actValueOut -> actValueIn\")\n\n classificationIn = {\"bucketIdx\": int(inputs[\"bucketIdxIn\"]),\n \"actValue\": float(inputs[\"actValueIn\"])}\n else:\n # Use Dummy classification input, because this param is required even for\n # inference mode. Because learning is off, the classifier is not learning\n # this dummy input. Inference only here.\n classificationIn = {\"actValue\": 0, \"bucketIdx\": 0}\n\n # Perform inference if self.inferenceMode is True\n # Train classifier if self.learningMode is True\n clResults = self._sdrClassifier.compute(recordNum=self.recordNum,\n patternNZ=patternNZ,\n classification=classificationIn,\n learn=self.learningMode,\n infer=self.inferenceMode)\n\n # fill outputs with clResults\n if clResults is not None and len(clResults) > 0:\n outputs['actualValues'][:len(clResults[\"actualValues\"])] = \\\n clResults[\"actualValues\"]\n\n for step in self.stepsList:\n stepIndex = self.stepsList.index(step)\n categoryOut = clResults[\"actualValues\"][clResults[step].argmax()]\n outputs['categoriesOut'][stepIndex] = categoryOut\n\n # Flatten the rest of the output. For example:\n # Original dict {1 : [0.1, 0.3, 0.2, 0.7]\n # 4 : [0.2, 0.4, 0.3, 0.5]}\n # becomes: [0.1, 0.3, 0.2, 0.7, 0.2, 0.4, 0.3, 0.5]\n stepProbabilities = clResults[step]\n for categoryIndex in xrange(self.maxCategoryCount):\n flatIndex = categoryIndex + stepIndex * self.maxCategoryCount\n if categoryIndex < len(stepProbabilities):\n outputs['probabilities'][flatIndex] = \\\n stepProbabilities[categoryIndex]\n else:\n outputs['probabilities'][flatIndex] = 0.0\n\n self.recordNum += 1", "response": "This method processes one input sample and returns the output of the compute method."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the number of elements in the output.", "response": "def getOutputElementCount(self, outputName):\n \"\"\"\n Overrides :meth:`nupic.bindings.regions.PyRegion.PyRegion.getOutputElementCount`.\n \"\"\"\n if outputName == \"categoriesOut\":\n return len(self.stepsList)\n elif outputName == \"probabilities\":\n return len(self.stepsList) * self.maxCategoryCount\n elif outputName == \"actualValues\":\n return self.maxCategoryCount\n else:\n raise ValueError(\"Unknown output {}.\".format(outputName))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run(self):\n # -----------------------------------------------------------------------\n # Load the experiment's description.py module\n descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(\n self._experimentDir)\n expIface = helpers.getExperimentDescriptionInterfaceFromModule(\n descriptionPyModule)\n expIface.normalizeStreamSources()\n\n modelDescription = expIface.getModelDescription()\n self._modelControl = expIface.getModelControl()\n\n # -----------------------------------------------------------------------\n # Create the input data stream for this task\n streamDef = self._modelControl['dataset']\n\n from nupic.data.stream_reader import StreamReader\n readTimeout = 0\n\n self._inputSource = StreamReader(streamDef, isBlocking=False,\n maxTimeout=readTimeout)\n\n\n # -----------------------------------------------------------------------\n #Get field statistics from the input source\n fieldStats = self._getFieldStats()\n # -----------------------------------------------------------------------\n # Construct the model instance\n self._model = ModelFactory.create(modelDescription)\n self._model.setFieldStatistics(fieldStats)\n self._model.enableLearning()\n self._model.enableInference(self._modelControl.get(\"inferenceArgs\", None))\n\n # -----------------------------------------------------------------------\n # Instantiate the metrics\n self.__metricMgr = MetricsManager(self._modelControl.get('metrics',None),\n self._model.getFieldInfo(),\n self._model.getInferenceType())\n\n self.__loggedMetricPatterns = self._modelControl.get(\"loggedMetrics\", [])\n\n self._optimizedMetricLabel = self.__getOptimizedMetricLabel()\n self._reportMetricLabels = matchPatterns(self._reportKeyPatterns,\n self._getMetricLabels())\n\n\n # -----------------------------------------------------------------------\n # Initialize periodic activities (e.g., for model result updates)\n self._periodic = self._initPeriodicActivities()\n\n # -----------------------------------------------------------------------\n # Create our top-level loop-control iterator\n numIters = self._modelControl.get('iterationCount', -1)\n\n # Are we asked to turn off learning for a certain # of iterations near the\n # end?\n learningOffAt = None\n iterationCountInferOnly = self._modelControl.get('iterationCountInferOnly', 0)\n if iterationCountInferOnly == -1:\n self._model.disableLearning()\n elif iterationCountInferOnly > 0:\n assert numIters > iterationCountInferOnly, \"when iterationCountInferOnly \" \\\n \"is specified, iterationCount must be greater than \" \\\n \"iterationCountInferOnly.\"\n learningOffAt = numIters - iterationCountInferOnly\n\n self.__runTaskMainLoop(numIters, learningOffAt=learningOffAt)\n\n # -----------------------------------------------------------------------\n # Perform final operations for model\n self._finalize()\n\n return (self._cmpReason, None)", "response": "Runs the OPF Model"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _finalize(self):\n\n self._logger.info(\n \"Finished: modelID=%r; %r records processed. Performing final activities\",\n self._modelID, self._currentRecordIndex + 1)\n\n # =========================================================================\n # Dump the experiment metrics at the end of the task\n # =========================================================================\n self._updateModelDBResults()\n\n # =========================================================================\n # Check if the current model is the best. Create a milestone if necessary\n # If the model has been killed, it is not a candidate for \"best model\",\n # and its output cache should be destroyed\n # =========================================================================\n if not self._isKilled:\n self.__updateJobResults()\n else:\n self.__deleteOutputCache(self._modelID)\n\n # =========================================================================\n # Close output stream, if necessary\n # =========================================================================\n if self._predictionLogger:\n self._predictionLogger.close()\n\n # =========================================================================\n # Close input stream, if necessary\n # =========================================================================\n if self._inputSource: \n self._inputSource.close()", "response": "Run final activities after a model has been run."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a checkpoint from the current model and store it in the Models DB", "response": "def __createModelCheckpoint(self):\n \"\"\" Create a checkpoint from the current model, and store it in a dir named\n after checkpoint GUID, and finally store the GUID in the Models DB \"\"\"\n\n if self._model is None or self._modelCheckpointGUID is None:\n return\n\n # Create an output store, if one doesn't exist already\n if self._predictionLogger is None:\n self._createPredictionLogger()\n\n predictions = StringIO.StringIO()\n self._predictionLogger.checkpoint(\n checkpointSink=predictions,\n maxRows=int(Configuration.get('nupic.model.checkpoint.maxPredictionRows')))\n\n self._model.save(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))\n self._jobsDAO.modelSetFields(modelID,\n {'modelCheckpointId':str(self._modelCheckpointGUID)},\n ignoreUnchanged=True)\n\n self._logger.info(\"Checkpointed Hypersearch Model: modelID: %r, \"\n \"checkpointID: %r\", self._modelID, checkpointID)\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndeletes the stored checkpoint for the specified modelID.", "response": "def __deleteModelCheckpoint(self, modelID):\n \"\"\"\n Delete the stored checkpoint for the specified modelID. This function is\n called if the current model is now the best model, making the old model's\n checkpoint obsolete\n\n Parameters:\n -----------------------------------------------------------------------\n modelID: The modelID for the checkpoint to delete. This is NOT the\n unique checkpointID\n \"\"\"\n\n checkpointID = \\\n self._jobsDAO.modelsGetFields(modelID, ['modelCheckpointId'])[0]\n\n if checkpointID is None:\n return\n\n try:\n shutil.rmtree(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))\n except:\n self._logger.warn(\"Failed to delete model checkpoint %s. \"\\\n \"Assuming that another worker has already deleted it\",\n checkpointID)\n return\n\n self._jobsDAO.modelSetFields(modelID,\n {'modelCheckpointId':None},\n ignoreUnchanged=True)\n return"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _createPredictionLogger(self):\n # Write results to a file\n self._predictionLogger = BasicPredictionLogger(\n fields=self._model.getFieldInfo(),\n experimentDir=self._experimentDir,\n label = \"hypersearch-worker\",\n inferenceType=self._model.getInferenceType())\n\n if self.__loggedMetricPatterns:\n metricLabels = self.__metricMgr.getMetricLabels()\n loggedMetrics = matchPatterns(self.__loggedMetricPatterns, metricLabels)\n self._predictionLogger.setLoggedMetrics(loggedMetrics)", "response": "Creates the model s PredictionLogger object which is an interface to write\n model results to a permanent storage location."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __getOptimizedMetricLabel(self):\n matchingKeys = matchPatterns([self._optimizeKeyPattern],\n self._getMetricLabels())\n\n if len(matchingKeys) == 0:\n raise Exception(\"None of the generated metrics match the specified \"\n \"optimization pattern: %s. Available metrics are %s\" % \\\n (self._optimizeKeyPattern, self._getMetricLabels()))\n elif len(matchingKeys) > 1:\n raise Exception(\"The specified optimization pattern '%s' matches more \"\n \"than one metric: %s\" % (self._optimizeKeyPattern, matchingKeys))\n\n return matchingKeys[0]", "response": "Returns the label for the metric being optimized. This function returns the label for the metric being optimized over the base model."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the model s record in with the current results and updates the model s record in the Model database.", "response": "def _updateModelDBResults(self):\n \"\"\" Retrieves the current results and updates the model's record in\n the Model database.\n \"\"\"\n\n # -----------------------------------------------------------------------\n # Get metrics\n metrics = self._getMetrics()\n\n # -----------------------------------------------------------------------\n # Extract report metrics that match the requested report REs\n reportDict = dict([(k,metrics[k]) for k in self._reportMetricLabels])\n\n # -----------------------------------------------------------------------\n # Extract the report item that matches the optimize key RE\n # TODO cache optimizedMetricLabel sooner\n metrics = self._getMetrics()\n optimizeDict = dict()\n if self._optimizeKeyPattern is not None:\n optimizeDict[self._optimizedMetricLabel] = \\\n metrics[self._optimizedMetricLabel]\n\n # -----------------------------------------------------------------------\n # Update model results\n results = json.dumps((metrics , optimizeDict))\n self._jobsDAO.modelUpdateResults(self._modelID, results=results,\n metricValue=optimizeDict.values()[0],\n numRecords=(self._currentRecordIndex + 1))\n\n self._logger.debug(\n \"Model Results: modelID=%s; numRecords=%s; results=%s\" % \\\n (self._modelID, self._currentRecordIndex + 1, results))\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating the results field of the job.", "response": "def __updateJobResultsPeriodic(self):\n \"\"\"\n Periodic check to see if this is the best model. This should only have an\n effect if this is the *first* model to report its progress\n \"\"\"\n if self._isBestModelStored and not self._isBestModel:\n return\n\n while True:\n jobResultsStr = self._jobsDAO.jobGetFields(self._jobID, ['results'])[0]\n if jobResultsStr is None:\n jobResults = {}\n else:\n self._isBestModelStored = True\n if not self._isBestModel:\n return\n\n jobResults = json.loads(jobResultsStr)\n\n bestModel = jobResults.get('bestModel', None)\n bestMetric = jobResults.get('bestValue', None)\n isSaved = jobResults.get('saved', False)\n\n # If there is a best model, and it is not the same as the current model\n # we should wait till we have processed all of our records to see if\n # we are the the best\n if (bestModel is not None) and (self._modelID != bestModel):\n self._isBestModel = False\n return\n\n # Make sure prediction output stream is ready before we present our model\n # as \"bestModel\"; sometimes this takes a long time, so update the model's\n # timestamp to help avoid getting orphaned\n self.__flushPredictionCache()\n self._jobsDAO.modelUpdateTimestamp(self._modelID)\n\n metrics = self._getMetrics()\n\n jobResults['bestModel'] = self._modelID\n jobResults['bestValue'] = metrics[self._optimizedMetricLabel]\n jobResults['metrics'] = metrics\n jobResults['saved'] = False\n\n newResults = json.dumps(jobResults)\n\n isUpdated = self._jobsDAO.jobSetFieldIfEqual(self._jobID,\n fieldName='results',\n curValue=jobResultsStr,\n newValue=newResults)\n if isUpdated or (not isUpdated and newResults==jobResultsStr):\n self._isBestModel = True\n break"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck if the current best model for the job is better than the stored best model for the job.", "response": "def __checkIfBestCompletedModel(self):\n \"\"\"\n Reads the current \"best model\" for the job and returns whether or not the\n current model is better than the \"best model\" stored for the job\n\n Returns: (isBetter, storedBest, origResultsStr)\n\n isBetter:\n True if the current model is better than the stored \"best model\"\n storedResults:\n A dict of the currently stored results in the jobs table record\n origResultsStr:\n The json-encoded string that currently resides in the \"results\" field\n of the jobs record (used to create atomicity)\n \"\"\"\n\n jobResultsStr = self._jobsDAO.jobGetFields(self._jobID, ['results'])[0]\n\n if jobResultsStr is None:\n jobResults = {}\n else:\n jobResults = json.loads(jobResultsStr)\n\n isSaved = jobResults.get('saved', False)\n bestMetric = jobResults.get('bestValue', None)\n\n currentMetric = self._getMetrics()[self._optimizedMetricLabel]\n self._isBestModel = (not isSaved) \\\n or (currentMetric < bestMetric)\n\n\n\n return self._isBestModel, jobResults, jobResultsStr"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __updateJobResults(self):\n isSaved = False\n while True:\n self._isBestModel, jobResults, jobResultsStr = \\\n self.__checkIfBestCompletedModel()\n\n # -----------------------------------------------------------------------\n # If the current model is the best:\n # 1) Save the model's predictions\n # 2) Checkpoint the model state\n # 3) Update the results for the job\n if self._isBestModel:\n\n # Save the current model and its results\n if not isSaved:\n self.__flushPredictionCache()\n self._jobsDAO.modelUpdateTimestamp(self._modelID)\n self.__createModelCheckpoint()\n self._jobsDAO.modelUpdateTimestamp(self._modelID)\n isSaved = True\n\n # Now record the model as the best for the job\n prevBest = jobResults.get('bestModel', None)\n prevWasSaved = jobResults.get('saved', False)\n\n # If the current model is the best, it shouldn't already be checkpointed\n if prevBest == self._modelID:\n assert not prevWasSaved\n\n metrics = self._getMetrics()\n\n jobResults['bestModel'] = self._modelID\n jobResults['bestValue'] = metrics[self._optimizedMetricLabel]\n jobResults['metrics'] = metrics\n jobResults['saved'] = True\n\n isUpdated = self._jobsDAO.jobSetFieldIfEqual(self._jobID,\n fieldName='results',\n curValue=jobResultsStr,\n newValue=json.dumps(jobResults))\n if isUpdated:\n if prevWasSaved:\n self.__deleteOutputCache(prevBest)\n self._jobsDAO.modelUpdateTimestamp(self._modelID)\n self.__deleteModelCheckpoint(prevBest)\n self._jobsDAO.modelUpdateTimestamp(self._modelID)\n\n self._logger.info(\"Model %d chosen as best model\", self._modelID)\n break\n\n # -----------------------------------------------------------------------\n # If the current model is not the best, delete its outputs\n else:\n # NOTE: we update model timestamp around these occasionally-lengthy\n # operations to help prevent the model from becoming orphaned\n self.__deleteOutputCache(self._modelID)\n self._jobsDAO.modelUpdateTimestamp(self._modelID)\n self.__deleteModelCheckpoint(self._modelID)\n self._jobsDAO.modelUpdateTimestamp(self._modelID)\n break", "response": "Update the results of the current model and the best model."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _writePrediction(self, result):\n self.__predictionCache.append(result)\n\n if self._isBestModel:\n self.__flushPredictionCache()", "response": "Writes the prediction results to the prediction cache."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __flushPredictionCache(self):\n\n if not self.__predictionCache:\n return\n\n # Create an output store, if one doesn't exist already\n if self._predictionLogger is None:\n self._createPredictionLogger()\n\n startTime = time.time()\n self._predictionLogger.writeRecords(self.__predictionCache,\n progressCB=self.__writeRecordsCallback)\n self._logger.info(\"Flushed prediction cache; numrows=%s; elapsed=%s sec.\",\n len(self.__predictionCache), time.time() - startTime)\n self.__predictionCache.clear()", "response": "Flushes the prediction cache to the prediction output stream instance."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndeleting the output cache associated with the given modelID.", "response": "def __deleteOutputCache(self, modelID):\n \"\"\"\n Delete's the output cache associated with the given modelID. This actually\n clears up the resources associated with the cache, rather than deleting al\n the records in the cache\n\n Parameters:\n -----------------------------------------------------------------------\n modelID: The id of the model whose output cache is being deleted\n\n \"\"\"\n\n # If this is our output, we should close the connection\n if modelID == self._modelID and self._predictionLogger is not None:\n self._predictionLogger.close()\n del self.__predictionCache\n self._predictionLogger = None\n self.__predictionCache = None"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _initPeriodicActivities(self):\n\n # Activity to update the metrics for this model\n # in the models table\n updateModelDBResults = PeriodicActivityRequest(repeating=True,\n period=100,\n cb=self._updateModelDBResults)\n\n updateJobResults = PeriodicActivityRequest(repeating=True,\n period=100,\n cb=self.__updateJobResultsPeriodic)\n\n checkCancelation = PeriodicActivityRequest(repeating=True,\n period=50,\n cb=self.__checkCancelation)\n\n checkMaturity = PeriodicActivityRequest(repeating=True,\n period=10,\n cb=self.__checkMaturity)\n\n\n # Do an initial update of the job record after 2 iterations to make\n # sure that it is populated with something without having to wait too long\n updateJobResultsFirst = PeriodicActivityRequest(repeating=False,\n period=2,\n cb=self.__updateJobResultsPeriodic)\n\n\n periodicActivities = [updateModelDBResults,\n updateJobResultsFirst,\n updateJobResults,\n checkCancelation]\n\n if self._isMaturityEnabled:\n periodicActivities.append(checkMaturity)\n\n return PeriodicActivityMgr(requestedActivities=periodicActivities)", "response": "Creates and returns a PeriodicActivityMgr instance initialized with the current model and job records."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __checkCancelation(self):\n\n # Update a hadoop job counter at least once every 600 seconds so it doesn't\n # think our map task is dead\n print >>sys.stderr, \"reporter:counter:HypersearchWorker,numRecords,50\"\n\n # See if the job got cancelled\n jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['cancel'])[0]\n if jobCancel:\n self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED\n self._isCanceled = True\n self._logger.info(\"Model %s canceled because Job %s was stopped.\",\n self._modelID, self._jobID)\n else:\n stopReason = self._jobsDAO.modelsGetFields(self._modelID, ['engStop'])[0]\n\n if stopReason is None:\n pass\n\n elif stopReason == ClientJobsDAO.STOP_REASON_KILLED:\n self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED\n self._isKilled = True\n self._logger.info(\"Model %s canceled because it was killed by hypersearch\",\n self._modelID)\n\n elif stopReason == ClientJobsDAO.STOP_REASON_STOPPED:\n self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED\n self._isCanceled = True\n self._logger.info(\"Model %s stopped because hypersearch ended\", self._modelID)\n else:\n raise RuntimeError (\"Unexpected stop reason encountered: %s\" % (stopReason))", "response": "Check if the cancelation flag has been set for this model in the Model DB"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck the performance of the current model and see if the error is leveled off.", "response": "def __checkMaturity(self):\n \"\"\" Save the current metric value and see if the model's performance has\n 'leveled off.' We do this by looking at some number of previous number of\n recordings \"\"\"\n\n if self._currentRecordIndex+1 < self._MIN_RECORDS_TO_BE_BEST:\n return\n\n # If we are already mature, don't need to check anything\n if self._isMature:\n return\n\n metric = self._getMetrics()[self._optimizedMetricLabel]\n self._metricRegression.addPoint(x=self._currentRecordIndex, y=metric)\n\n # Perform a linear regression to see if the error is leveled off\n #pctChange = self._metricRegression.getPctChange()\n #if pctChange is not None and abs(pctChange ) <= self._MATURITY_MAX_CHANGE:\n pctChange, absPctChange = self._metricRegression.getPctChanges()\n if pctChange is not None and absPctChange <= self._MATURITY_MAX_CHANGE:\n self._jobsDAO.modelSetFields(self._modelID,\n {'engMatured':True})\n\n # TODO: Don't stop if we are currently the best model. Also, if we\n # are still running after maturity, we have to periodically check to\n # see if we are still the best model. As soon we lose to some other\n # model, then we should stop at that point.\n self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED\n self._isMature = True\n\n self._logger.info(\"Model %d has matured (pctChange=%s, n=%d). \\n\"\\\n \"Scores = %s\\n\"\\\n \"Stopping execution\",self._modelID, pctChange,\n self._MATURITY_NUM_POINTS,\n self._metricRegression._window)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the current model as orphaned.", "response": "def __setAsOrphaned(self):\n \"\"\"\n Sets the current model as orphaned. This is called when the scheduler is\n about to kill the process to reallocate the worker to a different process.\n \"\"\"\n cmplReason = ClientJobsDAO.CMPL_REASON_ORPHAN\n cmplMessage = \"Killed by Scheduler\"\n self._jobsDAO.modelSetCompleted(self._modelID, cmplReason, cmplMessage)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting the state of the job record to the DB.", "response": "def writeStateToDB(self):\n \"\"\"Update the state in the job record with our local changes (if any).\n If we don't have the latest state in our priorStateJSON, then re-load\n in the latest state and return False. If we were successful writing out\n our changes, return True\n\n Parameters:\n ---------------------------------------------------------------------\n retval: True if we were successful writing out our changes\n False if our priorState is not the latest that was in the DB.\n In this case, we will re-load our state from the DB\n \"\"\"\n # If no changes, do nothing\n if not self._dirty:\n return True\n\n # Set the update time\n self._state['lastUpdateTime'] = time.time()\n newStateJSON = json.dumps(self._state)\n success = self._hsObj._cjDAO.jobSetFieldIfEqual(self._hsObj._jobID,\n 'engWorkerState', str(newStateJSON), str(self._priorStateJSON))\n\n if success:\n self.logger.debug(\"Success changing hsState to: \\n%s \" % \\\n (pprint.pformat(self._state, indent=4)))\n self._priorStateJSON = newStateJSON\n\n # If no success, read in the current state from the DB\n else:\n self.logger.debug(\"Failed to change hsState to: \\n%s \" % \\\n (pprint.pformat(self._state, indent=4)))\n\n self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(self._hsObj._jobID,\n ['engWorkerState'])[0]\n self._state = json.loads(self._priorStateJSON)\n\n self.logger.info(\"New hsState has been set by some other worker to: \"\n \" \\n%s\" % (pprint.pformat(self._state, indent=4)))\n\n return success"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the statistics for each field in a fast swarm.", "response": "def getFieldContributions(self):\n \"\"\"Return the field contributions statistics.\n\n Parameters:\n ---------------------------------------------------------------------\n retval: Dictionary where the keys are the field names and the values\n are how much each field contributed to the best score.\n \"\"\"\n\n #in the fast swarm, there is only 1 sprint and field contributions are\n #not defined\n if self._hsObj._fixedFields is not None:\n return dict(), dict()\n # Get the predicted field encoder name\n predictedEncoderName = self._hsObj._predictedFieldEncoder\n\n # -----------------------------------------------------------------------\n # Collect all the single field scores\n fieldScores = []\n for swarmId, info in self._state['swarms'].iteritems():\n encodersUsed = swarmId.split('.')\n if len(encodersUsed) != 1:\n continue\n field = self.getEncoderNameFromKey(encodersUsed[0])\n bestScore = info['bestErrScore']\n\n # If the bestScore is None, this swarm hasn't completed yet (this could\n # happen if we're exiting because of maxModels), so look up the best\n # score so far\n if bestScore is None:\n (_modelId, bestScore) = \\\n self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)\n\n fieldScores.append((bestScore, field))\n\n\n # -----------------------------------------------------------------------\n # If we only have 1 field that was tried in the first sprint, then use that\n # as the base and get the contributions from the fields in the next sprint.\n if self._hsObj._searchType == HsSearchType.legacyTemporal:\n assert(len(fieldScores)==1)\n (baseErrScore, baseField) = fieldScores[0]\n\n for swarmId, info in self._state['swarms'].iteritems():\n encodersUsed = swarmId.split('.')\n if len(encodersUsed) != 2:\n continue\n\n fields = [self.getEncoderNameFromKey(name) for name in encodersUsed]\n fields.remove(baseField)\n\n fieldScores.append((info['bestErrScore'], fields[0]))\n\n # The first sprint tried a bunch of fields, pick the worst performing one\n # (within the top self._hsObj._maxBranching ones) as the base\n else:\n fieldScores.sort(reverse=True)\n\n # If maxBranching was specified, pick the worst performing field within\n # the top maxBranching+1 fields as our base, which will give that field\n # a contribution of 0.\n if self._hsObj._maxBranching > 0 \\\n and len(fieldScores) > self._hsObj._maxBranching:\n baseErrScore = fieldScores[-self._hsObj._maxBranching-1][0]\n else:\n baseErrScore = fieldScores[0][0]\n\n\n # -----------------------------------------------------------------------\n # Prepare and return the fieldContributions dict\n pctFieldContributionsDict = dict()\n absFieldContributionsDict = dict()\n\n # If we have no base score, can't compute field contributions. This can\n # happen when we exit early due to maxModels or being cancelled\n if baseErrScore is not None:\n\n # If the base error score is 0, we can't compute a percent difference\n # off of it, so move it to a very small float\n if abs(baseErrScore) < 0.00001:\n baseErrScore = 0.00001\n for (errScore, field) in fieldScores:\n if errScore is not None:\n pctBetter = (baseErrScore - errScore) * 100.0 / baseErrScore\n else:\n pctBetter = 0.0\n errScore = baseErrScore # for absFieldContribution\n\n pctFieldContributionsDict[field] = pctBetter\n absFieldContributionsDict[field] = baseErrScore - errScore\n\n self.logger.debug(\"FieldContributions: %s\" % (pctFieldContributionsDict))\n return pctFieldContributionsDict, absFieldContributionsDict"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getAllSwarms(self, sprintIdx):\n swarmIds = []\n for swarmId, info in self._state['swarms'].iteritems():\n if info['sprintIdx'] == sprintIdx:\n swarmIds.append(swarmId)\n\n return swarmIds", "response": "Returns the list of all active swarms in the given sprint."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getCompletedSwarms(self):\n swarmIds = []\n for swarmId, info in self._state['swarms'].iteritems():\n if info['status'] == 'completed':\n swarmIds.append(swarmId)\n\n return swarmIds", "response": "Returns the list of all completed swarms."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getCompletingSwarms(self):\n swarmIds = []\n for swarmId, info in self._state['swarms'].iteritems():\n if info['status'] == 'completing':\n swarmIds.append(swarmId)\n\n return swarmIds", "response": "Returns the list of all completing swarms."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the best model ID and the errScore from the given sprint.", "response": "def bestModelInSprint(self, sprintIdx):\n \"\"\"Return the best model ID and it's errScore from the given sprint,\n which may still be in progress. This returns the best score from all models\n in the sprint which have matured so far.\n\n Parameters:\n ---------------------------------------------------------------------\n retval: (modelId, errScore)\n \"\"\"\n # Get all the swarms in this sprint\n swarms = self.getAllSwarms(sprintIdx)\n\n # Get the best model and score from each swarm\n bestModelId = None\n bestErrScore = numpy.inf\n for swarmId in swarms:\n (modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)\n if errScore < bestErrScore:\n bestModelId = modelId\n bestErrScore = errScore\n\n return (bestModelId, bestErrScore)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef setSwarmState(self, swarmId, newStatus):\n assert (newStatus in ['active', 'completing', 'completed', 'killed'])\n\n # Set the swarm status\n swarmInfo = self._state['swarms'][swarmId]\n if swarmInfo['status'] == newStatus:\n return\n\n # If some other worker noticed it as completed, setting it to completing\n # is obviously old information....\n if swarmInfo['status'] == 'completed' and newStatus == 'completing':\n return\n\n self._dirty = True\n swarmInfo['status'] = newStatus\n if newStatus == 'completed':\n (modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)\n swarmInfo['bestModelId'] = modelId\n swarmInfo['bestErrScore'] = errScore\n\n # If no longer active, remove it from the activeSwarms entry\n if newStatus != 'active' and swarmId in self._state['activeSwarms']:\n self._state['activeSwarms'].remove(swarmId)\n\n # If new status is 'killed', kill off any running particles in that swarm\n if newStatus=='killed':\n self._hsObj.killSwarmParticles(swarmId)\n\n # In case speculative particles are enabled, make sure we generate a new\n # swarm at this time if all of the swarms in the current sprint have\n # completed. This will insure that we don't mark the sprint as completed\n # before we've created all the possible swarms.\n sprintIdx = swarmInfo['sprintIdx']\n self.isSprintActive(sprintIdx)\n\n # Update the sprint status. Check all the swarms that belong to this sprint.\n # If they are all completed, the sprint is completed.\n sprintInfo = self._state['sprints'][sprintIdx]\n\n statusCounts = dict(active=0, completing=0, completed=0, killed=0)\n bestModelIds = []\n bestErrScores = []\n for info in self._state['swarms'].itervalues():\n if info['sprintIdx'] != sprintIdx:\n continue\n statusCounts[info['status']] += 1\n if info['status'] == 'completed':\n bestModelIds.append(info['bestModelId'])\n bestErrScores.append(info['bestErrScore'])\n\n if statusCounts['active'] > 0:\n sprintStatus = 'active'\n elif statusCounts['completing'] > 0:\n sprintStatus = 'completing'\n else:\n sprintStatus = 'completed'\n sprintInfo['status'] = sprintStatus\n\n # If the sprint is complete, get the best model from all of its swarms and\n # store that as the sprint best\n if sprintStatus == 'completed':\n if len(bestErrScores) > 0:\n whichIdx = numpy.array(bestErrScores).argmin()\n sprintInfo['bestModelId'] = bestModelIds[whichIdx]\n sprintInfo['bestErrScore'] = bestErrScores[whichIdx]\n else:\n # This sprint was empty, most likely because all particles were\n # killed. Give it a huge error score\n sprintInfo['bestModelId'] = 0\n sprintInfo['bestErrScore'] = numpy.inf\n\n\n # See if our best err score got NO BETTER as compared to a previous\n # sprint. If so, stop exploring subsequent sprints (lastGoodSprint\n # is no longer None).\n bestPrior = numpy.inf\n for idx in range(sprintIdx):\n if self._state['sprints'][idx]['status'] == 'completed':\n (_, errScore) = self.bestModelInCompletedSprint(idx)\n if errScore is None:\n errScore = numpy.inf\n else:\n errScore = numpy.inf\n if errScore < bestPrior:\n bestPrior = errScore\n\n if sprintInfo['bestErrScore'] >= bestPrior:\n self._state['lastGoodSprint'] = sprintIdx-1\n\n # If ALL sprints up to the last good one are done, the search is now over\n if self._state['lastGoodSprint'] is not None \\\n and not self.anyGoodSprintsActive():\n self._state['searchOver'] = True", "response": "Change the given swarm s state to newStatus."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning True if there are any more good sprints still being explored.", "response": "def anyGoodSprintsActive(self):\n \"\"\"Return True if there are any more good sprints still being explored.\n A 'good' sprint is one that is earlier than where we detected an increase\n in error from sprint to subsequent sprint.\n \"\"\"\n if self._state['lastGoodSprint'] is not None:\n goodSprints = self._state['sprints'][0:self._state['lastGoodSprint']+1]\n else:\n goodSprints = self._state['sprints']\n\n for sprint in goodSprints:\n if sprint['status'] == 'active':\n anyActiveSprints = True\n break\n else:\n anyActiveSprints = False\n\n return anyActiveSprints"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef isSprintCompleted(self, sprintIdx):\n numExistingSprints = len(self._state['sprints'])\n if sprintIdx >= numExistingSprints:\n return False\n\n return (self._state['sprints'][sprintIdx]['status'] == 'completed')", "response": "Return True if the given sprint has completed."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef killUselessSwarms(self):\n # Get number of existing sprints\n numExistingSprints = len(self._state['sprints'])\n\n # Should we bother killing useless swarms?\n if self._hsObj._searchType == HsSearchType.legacyTemporal:\n if numExistingSprints <= 2:\n return\n else:\n if numExistingSprints <= 1:\n return\n\n # Form completedSwarms as a list of tuples, each tuple contains:\n # (swarmName, swarmState, swarmBestErrScore)\n # ex. completedSwarms:\n # [('a', {...}, 1.4),\n # ('b', {...}, 2.0),\n # ('c', {...}, 3.0)]\n completedSwarms = self.getCompletedSwarms()\n completedSwarms = [(swarm, self._state[\"swarms\"][swarm],\n self._state[\"swarms\"][swarm][\"bestErrScore\"]) \\\n for swarm in completedSwarms]\n\n # Form the completedMatrix. Each row corresponds to a sprint. Each row\n # contains the list of swarm tuples that belong to that sprint, sorted\n # by best score. Each swarm tuple contains (swarmName, swarmState,\n # swarmBestErrScore).\n # ex. completedMatrix:\n # [(('a', {...}, 1.4), ('b', {...}, 2.0), ('c', {...}, 3.0)),\n # (('a.b', {...}, 3.0), ('b.c', {...}, 4.0))]\n completedMatrix = [[] for i in range(numExistingSprints)]\n for swarm in completedSwarms:\n completedMatrix[swarm[1][\"sprintIdx\"]].append(swarm)\n for sprint in completedMatrix:\n sprint.sort(key=itemgetter(2))\n\n # Form activeSwarms as a list of tuples, each tuple contains:\n # (swarmName, swarmState, swarmBestErrScore)\n # Include all activeSwarms and completingSwarms\n # ex. activeSwarms:\n # [('d', {...}, 1.4),\n # ('e', {...}, 2.0),\n # ('f', {...}, 3.0)]\n activeSwarms = self.getActiveSwarms()\n # Append the completing swarms\n activeSwarms.extend(self.getCompletingSwarms())\n activeSwarms = [(swarm, self._state[\"swarms\"][swarm],\n self._state[\"swarms\"][swarm][\"bestErrScore\"]) \\\n for swarm in activeSwarms]\n\n # Form the activeMatrix. Each row corresponds to a sprint. Each row\n # contains the list of swarm tuples that belong to that sprint, sorted\n # by best score. Each swarm tuple contains (swarmName, swarmState,\n # swarmBestErrScore)\n # ex. activeMatrix:\n # [(('d', {...}, 1.4), ('e', {...}, 2.0), ('f', {...}, 3.0)),\n # (('d.e', {...}, 3.0), ('e.f', {...}, 4.0))]\n activeMatrix = [[] for i in range(numExistingSprints)]\n for swarm in activeSwarms:\n activeMatrix[swarm[1][\"sprintIdx\"]].append(swarm)\n for sprint in activeMatrix:\n sprint.sort(key=itemgetter(2))\n\n\n # Figure out which active swarms to kill\n toKill = []\n for i in range(1, numExistingSprints):\n for swarm in activeMatrix[i]:\n curSwarmEncoders = swarm[0].split(\".\")\n\n # If previous sprint is complete, get the best swarm and kill all active\n # sprints that are not supersets\n if(len(activeMatrix[i-1])==0):\n # If we are trying all possible 3 field combinations, don't kill any\n # off in sprint 2\n if i==2 and (self._hsObj._tryAll3FieldCombinations or \\\n self._hsObj._tryAll3FieldCombinationsWTimestamps):\n pass\n else:\n bestInPrevious = completedMatrix[i-1][0]\n bestEncoders = bestInPrevious[0].split('.')\n for encoder in bestEncoders:\n if not encoder in curSwarmEncoders:\n toKill.append(swarm)\n\n # if there are more than two completed encoders sets that are complete and\n # are worse than at least one active swarm in the previous sprint. Remove\n # any combinations that have any pair of them since they cannot have the best encoder.\n #elif(len(completedMatrix[i-1])>1):\n # for completedSwarm in completedMatrix[i-1]:\n # activeMatrix[i-1][0][2] 0:\n print \"ParseMe: Killing encoders:\" + str(toKill)\n\n for swarm in toKill:\n self.setSwarmState(swarm[0], \"killed\")\n\n return", "response": "See if we can kill off some speculative swarms. If an earlier sprint\n has finally completed, we can now tell which fields should *really* be present\n in the sprints we've already started due to speculation, and kill off the\n swarms that should not have been included."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef isSprintActive(self, sprintIdx):\n\n while True:\n numExistingSprints = len(self._state['sprints'])\n\n # If this sprint already exists, see if it is active\n if sprintIdx <= numExistingSprints-1:\n\n # With speculation off, it's simple, just return whether or not the\n # asked for sprint has active status\n if not self._hsObj._speculativeParticles:\n active = (self._state['sprints'][sprintIdx]['status'] == 'active')\n return (active, False)\n\n # With speculation on, if the sprint is still marked active, we also\n # need to see if it's time to add a new swarm to it.\n else:\n active = (self._state['sprints'][sprintIdx]['status'] == 'active')\n if not active:\n return (active, False)\n\n # See if all of the existing swarms are at capacity (have all the\n # workers they need):\n activeSwarmIds = self.getActiveSwarms(sprintIdx)\n swarmSizes = [self._hsObj._resultsDB.getParticleInfos(swarmId,\n matured=False)[0] for swarmId in activeSwarmIds]\n notFullSwarms = [len(swarm) for swarm in swarmSizes \\\n if len(swarm) < self._hsObj._minParticlesPerSwarm]\n\n # If some swarms have room return that the swarm is active.\n if len(notFullSwarms) > 0:\n return (True, False)\n\n # If the existing swarms are at capacity, we will fall through to the\n # logic below which tries to add a new swarm to the sprint.\n\n # Stop creating new sprints?\n if self._state['lastGoodSprint'] is not None:\n return (False, True)\n\n # if fixedFields is set, we are running a fast swarm and only run sprint0\n if self._hsObj._fixedFields is not None:\n return (False, True)\n\n # ----------------------------------------------------------------------\n # Get the best model (if there is one) from the prior sprint. That gives\n # us the base encoder set for the next sprint. For sprint zero make sure\n # it does not take the last sprintidx because of wrapping.\n if sprintIdx > 0 \\\n and self._state['sprints'][sprintIdx-1]['status'] == 'completed':\n (bestModelId, _) = self.bestModelInCompletedSprint(sprintIdx-1)\n (particleState, _, _, _, _) = self._hsObj._resultsDB.getParticleInfo(\n bestModelId)\n bestSwarmId = particleState['swarmId']\n baseEncoderSets = [bestSwarmId.split('.')]\n\n # If there is no best model yet, then use all encoder sets from the prior\n # sprint that were not killed\n else:\n bestSwarmId = None\n particleState = None\n # Build up more combinations, using ALL of the sets in the current\n # sprint.\n baseEncoderSets = []\n for swarmId in self.getNonKilledSwarms(sprintIdx-1):\n baseEncoderSets.append(swarmId.split('.'))\n\n # ----------------------------------------------------------------------\n # Which encoders should we add to the current base set?\n encoderAddSet = []\n\n # If we have constraints on how many fields we carry forward into\n # subsequent sprints (either nupic.hypersearch.max.field.branching or\n # nupic.hypersearch.min.field.contribution was set), then be more\n # picky about which fields we add in.\n limitFields = False\n if self._hsObj._maxBranching > 0 \\\n or self._hsObj._minFieldContribution >= 0:\n if self._hsObj._searchType == HsSearchType.temporal or \\\n self._hsObj._searchType == HsSearchType.classification:\n if sprintIdx >= 1:\n limitFields = True\n baseSprintIdx = 0\n elif self._hsObj._searchType == HsSearchType.legacyTemporal:\n if sprintIdx >= 2:\n limitFields = True\n baseSprintIdx = 1\n else:\n raise RuntimeError(\"Unimplemented search type %s\" % \\\n (self._hsObj._searchType))\n\n\n # Only add top _maxBranching encoders to the swarms?\n if limitFields:\n\n # Get field contributions to filter added fields\n pctFieldContributions, absFieldContributions = \\\n self.getFieldContributions()\n toRemove = []\n self.logger.debug(\"FieldContributions min: %s\" % \\\n (self._hsObj._minFieldContribution))\n for fieldname in pctFieldContributions:\n if pctFieldContributions[fieldname] < self._hsObj._minFieldContribution:\n self.logger.debug(\"FieldContributions removing: %s\" % (fieldname))\n toRemove.append(self.getEncoderKeyFromName(fieldname))\n else:\n self.logger.debug(\"FieldContributions keeping: %s\" % (fieldname))\n\n\n # Grab the top maxBranching base sprint swarms.\n swarms = self._state[\"swarms\"]\n sprintSwarms = [(swarm, swarms[swarm][\"bestErrScore\"]) \\\n for swarm in swarms if swarms[swarm][\"sprintIdx\"] == baseSprintIdx]\n sprintSwarms = sorted(sprintSwarms, key=itemgetter(1))\n if self._hsObj._maxBranching > 0:\n sprintSwarms = sprintSwarms[0:self._hsObj._maxBranching]\n\n # Create encoder set to generate further swarms.\n for swarm in sprintSwarms:\n swarmEncoders = swarm[0].split(\".\")\n for encoder in swarmEncoders:\n if not encoder in encoderAddSet:\n encoderAddSet.append(encoder)\n encoderAddSet = [encoder for encoder in encoderAddSet \\\n if not str(encoder) in toRemove]\n\n # If no limit on the branching or min contribution, simply use all of the\n # encoders.\n else:\n encoderAddSet = self._hsObj._encoderNames\n\n\n # -----------------------------------------------------------------------\n # Build up the new encoder combinations for the next sprint.\n newSwarmIds = set()\n\n # See if the caller wants to try more extensive field combinations with\n # 3 fields.\n if (self._hsObj._searchType == HsSearchType.temporal \\\n or self._hsObj._searchType == HsSearchType.legacyTemporal) \\\n and sprintIdx == 2 \\\n and (self._hsObj._tryAll3FieldCombinations or \\\n self._hsObj._tryAll3FieldCombinationsWTimestamps):\n\n if self._hsObj._tryAll3FieldCombinations:\n newEncoders = set(self._hsObj._encoderNames)\n if self._hsObj._predictedFieldEncoder in newEncoders:\n newEncoders.remove(self._hsObj._predictedFieldEncoder)\n else:\n # Just make sure the timestamp encoders are part of the mix\n newEncoders = set(encoderAddSet)\n if self._hsObj._predictedFieldEncoder in newEncoders:\n newEncoders.remove(self._hsObj._predictedFieldEncoder)\n for encoder in self._hsObj._encoderNames:\n if encoder.endswith('_timeOfDay') or encoder.endswith('_weekend') \\\n or encoder.endswith('_dayOfWeek'):\n newEncoders.add(encoder)\n\n allCombos = list(itertools.combinations(newEncoders, 2))\n for combo in allCombos:\n newSet = list(combo)\n newSet.append(self._hsObj._predictedFieldEncoder)\n newSet.sort()\n newSwarmId = '.'.join(newSet)\n if newSwarmId not in self._state['swarms']:\n newSwarmIds.add(newSwarmId)\n\n # If a speculative sprint, only add the first encoder, if not add\n # all of them.\n if (len(self.getActiveSwarms(sprintIdx-1)) > 0):\n break\n\n # Else, we only build up by adding 1 new encoder to the best combination(s)\n # we've seen from the prior sprint\n else:\n for baseEncoderSet in baseEncoderSets:\n for encoder in encoderAddSet:\n if encoder not in self._state['blackListedEncoders'] \\\n and encoder not in baseEncoderSet:\n newSet = list(baseEncoderSet)\n newSet.append(encoder)\n newSet.sort()\n newSwarmId = '.'.join(newSet)\n if newSwarmId not in self._state['swarms']:\n newSwarmIds.add(newSwarmId)\n\n # If a speculative sprint, only add the first encoder, if not add\n # all of them.\n if (len(self.getActiveSwarms(sprintIdx-1)) > 0):\n break\n\n\n # ----------------------------------------------------------------------\n # Sort the new swarm Ids\n newSwarmIds = sorted(newSwarmIds)\n\n # If no more swarms can be found for this sprint...\n if len(newSwarmIds) == 0:\n # if sprint is not an empty sprint return that it is active but do not\n # add anything to it.\n if len(self.getAllSwarms(sprintIdx)) > 0:\n return (True, False)\n\n # If this is an empty sprint and we couldn't find any new swarms to\n # add (only bad fields are remaining), the search is over\n else:\n return (False, True)\n\n # Add this sprint and the swarms that are in it to our state\n self._dirty = True\n\n # Add in the new sprint if necessary\n if len(self._state[\"sprints\"]) == sprintIdx:\n self._state['sprints'].append({'status': 'active',\n 'bestModelId': None,\n 'bestErrScore': None})\n\n # Add in the new swarm(s) to the sprint\n for swarmId in newSwarmIds:\n self._state['swarms'][swarmId] = {'status': 'active',\n 'bestModelId': None,\n 'bestErrScore': None,\n 'sprintIdx': sprintIdx}\n\n # Update the list of active swarms\n self._state['activeSwarms'] = self.getActiveSwarms()\n\n # Try to set new state\n success = self.writeStateToDB()\n\n # Return result if successful\n if success:\n return (True, False)", "response": "Returns True if the given sprint is active."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a encoder to the encoder list", "response": "def addEncoder(self, name, encoder):\n \"\"\"\n Adds one encoder.\n\n :param name: (string) name of encoder, should be unique\n :param encoder: (:class:`.Encoder`) the encoder to add\n \"\"\"\n self.encoders.append((name, encoder, self.width))\n for d in encoder.getDescription():\n self.description.append((d[0], d[1] + self.width))\n self.width += encoder.getWidth()"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef invariant(self):\n # Verify the description and singleNodeOnly attributes\n assert isinstance(self.description, str)\n assert isinstance(self.singleNodeOnly, bool)\n\n # Make sure that all items dicts are really dicts\n assert isinstance(self.inputs, dict)\n assert isinstance(self.outputs, dict)\n assert isinstance(self.parameters, dict)\n assert isinstance(self.commands, dict)\n\n # Verify all item dicts\n hasDefaultInput = False\n for k, v in self.inputs.items():\n assert isinstance(k, str)\n assert isinstance(v, InputSpec)\n v.invariant()\n if v.isDefaultInput:\n assert not hasDefaultInput\n hasDefaultInput = True\n\n\n hasDefaultOutput = False\n for k, v in self.outputs.items():\n assert isinstance(k, str)\n assert isinstance(v, OutputSpec)\n v.invariant()\n if v.isDefaultOutput:\n assert not hasDefaultOutput\n hasDefaultOutput = True\n\n for k, v in self.parameters.items():\n assert isinstance(k, str)\n assert isinstance(v, ParameterSpec)\n v.invariant()\n\n for k, v in self.commands.items():\n assert isinstance(k, str)\n assert isinstance(v, CommandSpec)\n v.invariant()", "response": "Verify the validity of the node spec object."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting the node spec to a plain dict of basic types", "response": "def toDict(self):\n \"\"\"Convert the information of the node spec to a plain dict of basic types\n\n The description and singleNodeOnly attributes are placed directly in\n the result dicts. The inputs, outputs, parameters and commands dicts\n contain Spec item objects (InputSpec, OutputSpec, etc). Each such object\n is converted also to a plain dict using the internal items2dict() function\n (see bellow).\n \"\"\"\n\n def items2dict(items):\n \"\"\"Convert a dict of node spec items to a plain dict\n\n Each node spec item object will be converted to a dict of its\n attributes. The entire items dict will become a dict of dicts (same keys).\n \"\"\"\n d = {}\n for k, v in items.items():\n d[k] = v.__dict__\n\n return d\n\n self.invariant()\n return dict(description=self.description,\n singleNodeOnly=self.singleNodeOnly,\n inputs=items2dict(self.inputs),\n outputs=items2dict(self.outputs),\n parameters=items2dict(self.parameters),\n commands=items2dict(self.commands))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating the best model for a given job.", "response": "def updateResultsForJob(self, forceUpdate=True):\n \"\"\" Chooses the best model for a given job.\n\n Parameters\n -----------------------------------------------------------------------\n forceUpdate: (True/False). If True, the update will ignore all the\n restrictions on the minimum time to update and the minimum\n number of records to update. This should typically only be\n set to true if the model has completed running\n \"\"\"\n updateInterval = time.time() - self._lastUpdateAttemptTime\n if updateInterval < self._MIN_UPDATE_INTERVAL and not forceUpdate:\n return\n\n self.logger.info(\"Attempting model selection for jobID=%d: time=%f\"\\\n \" lastUpdate=%f\"%(self._jobID,\n time.time(),\n self._lastUpdateAttemptTime))\n\n timestampUpdated = self._cjDB.jobUpdateSelectionSweep(self._jobID,\n self._MIN_UPDATE_INTERVAL)\n if not timestampUpdated:\n self.logger.info(\"Unable to update selection sweep timestamp: jobID=%d\" \\\n \" updateTime=%f\"%(self._jobID, self._lastUpdateAttemptTime))\n if not forceUpdate:\n return\n\n self._lastUpdateAttemptTime = time.time()\n self.logger.info(\"Succesfully updated selection sweep timestamp jobid=%d updateTime=%f\"\\\n %(self._jobID, self._lastUpdateAttemptTime))\n\n minUpdateRecords = self._MIN_UPDATE_THRESHOLD\n\n jobResults = self._getJobResults()\n if forceUpdate or jobResults is None:\n minUpdateRecords = 0\n\n candidateIDs, bestMetric = self._cjDB.modelsGetCandidates(self._jobID, minUpdateRecords)\n\n self.logger.info(\"Candidate models=%s, metric=%s, jobID=%s\"\\\n %(candidateIDs, bestMetric, self._jobID))\n\n if len(candidateIDs) == 0:\n return\n\n self._jobUpdateCandidate(candidateIDs[0], bestMetric, results=jobResults)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef createEncoder():\n consumption_encoder = ScalarEncoder(21, 0.0, 100.0, n=50, name=\"consumption\",\n clipInput=True)\n time_encoder = DateEncoder(timeOfDay=(21, 9.5), name=\"timestamp_timeOfDay\")\n\n encoder = MultiEncoder()\n encoder.addEncoder(\"consumption\", consumption_encoder)\n encoder.addEncoder(\"timestamp\", time_encoder)\n\n return encoder", "response": "Create the encoder instance for our test and return it."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a Network instance from a RecordStream instance.", "response": "def createNetwork(dataSource):\n \"\"\"Create the Network instance.\n\n The network has a sensor region reading data from `dataSource` and passing\n the encoded representation to an SPRegion. The SPRegion output is passed to\n a TMRegion.\n\n :param dataSource: a RecordStream instance to get data from\n :returns: a Network instance ready to run\n \"\"\"\n network = Network()\n\n # Our input is sensor data from the gym file. The RecordSensor region\n # allows us to specify a file record stream as the input source via the\n # dataSource attribute.\n network.addRegion(\"sensor\", \"py.RecordSensor\",\n json.dumps({\"verbosity\": _VERBOSITY}))\n sensor = network.regions[\"sensor\"].getSelf()\n # The RecordSensor needs to know how to encode the input values\n sensor.encoder = createEncoder()\n # Specify the dataSource as a file record stream instance\n sensor.dataSource = dataSource\n\n # Create the spatial pooler region\n SP_PARAMS[\"inputWidth\"] = sensor.encoder.getWidth()\n network.addRegion(\"spatialPoolerRegion\", \"py.SPRegion\", json.dumps(SP_PARAMS))\n\n # Link the SP region to the sensor input\n network.link(\"sensor\", \"spatialPoolerRegion\", \"UniformLink\", \"\")\n network.link(\"sensor\", \"spatialPoolerRegion\", \"UniformLink\", \"\",\n srcOutput=\"resetOut\", destInput=\"resetIn\")\n network.link(\"spatialPoolerRegion\", \"sensor\", \"UniformLink\", \"\",\n srcOutput=\"spatialTopDownOut\", destInput=\"spatialTopDownIn\")\n network.link(\"spatialPoolerRegion\", \"sensor\", \"UniformLink\", \"\",\n srcOutput=\"temporalTopDownOut\", destInput=\"temporalTopDownIn\")\n\n # Add the TMRegion on top of the SPRegion\n network.addRegion(\"temporalPoolerRegion\", \"py.TMRegion\",\n json.dumps(TM_PARAMS))\n\n network.link(\"spatialPoolerRegion\", \"temporalPoolerRegion\", \"UniformLink\", \"\")\n network.link(\"temporalPoolerRegion\", \"spatialPoolerRegion\", \"UniformLink\", \"\",\n srcOutput=\"topDownOut\", destInput=\"topDownIn\")\n\n # Add the AnomalyLikelihoodRegion on top of the TMRegion\n network.addRegion(\"anomalyLikelihoodRegion\", \"py.AnomalyLikelihoodRegion\",\n json.dumps({}))\n \n network.link(\"temporalPoolerRegion\", \"anomalyLikelihoodRegion\", \"UniformLink\",\n \"\", srcOutput=\"anomalyScore\", destInput=\"rawAnomalyScore\")\n network.link(\"sensor\", \"anomalyLikelihoodRegion\", \"UniformLink\", \"\",\n srcOutput=\"sourceOut\", destInput=\"metricValue\")\n \n\n spatialPoolerRegion = network.regions[\"spatialPoolerRegion\"]\n\n # Make sure learning is enabled\n spatialPoolerRegion.setParameter(\"learningMode\", True)\n # We want temporal anomalies so disable anomalyMode in the SP. This mode is\n # used for computing anomalies in a non-temporal model.\n spatialPoolerRegion.setParameter(\"anomalyMode\", False)\n\n temporalPoolerRegion = network.regions[\"temporalPoolerRegion\"]\n\n # Enable topDownMode to get the predicted columns output\n temporalPoolerRegion.setParameter(\"topDownMode\", True)\n # Make sure learning is enabled (this is the default)\n temporalPoolerRegion.setParameter(\"learningMode\", True)\n # Enable inference mode so we get predictions\n temporalPoolerRegion.setParameter(\"inferenceMode\", True)\n # Enable anomalyMode to compute the anomaly score to be passed to the anomaly\n # likelihood region. \n temporalPoolerRegion.setParameter(\"anomalyMode\", True)\n\n return network"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef runNetwork(network, writer):\n sensorRegion = network.regions[\"sensor\"]\n spatialPoolerRegion = network.regions[\"spatialPoolerRegion\"]\n temporalPoolerRegion = network.regions[\"temporalPoolerRegion\"]\n anomalyLikelihoodRegion = network.regions[\"anomalyLikelihoodRegion\"]\n\n prevPredictedColumns = []\n\n for i in xrange(_NUM_RECORDS):\n # Run the network for a single iteration\n network.run(1)\n\n # Write out the anomaly likelihood along with the record number and consumption\n # value.\n consumption = sensorRegion.getOutputData(\"sourceOut\")[0]\n anomalyScore = temporalPoolerRegion.getOutputData(\"anomalyScore\")[0]\n anomalyLikelihood = anomalyLikelihoodRegion.getOutputData(\"anomalyLikelihood\")[0]\n writer.writerow((i, consumption, anomalyScore, anomalyLikelihood))", "response": "Run the network and write output to writer."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nvalidating the experiment control dictionary for the experiment context", "response": "def __validateExperimentControl(self, control):\n \"\"\" Validates control dictionary for the experiment context\"\"\"\n # Validate task list\n taskList = control.get('tasks', None)\n if taskList is not None:\n taskLabelsList = []\n\n for task in taskList:\n validateOpfJsonValue(task, \"opfTaskSchema.json\")\n validateOpfJsonValue(task['taskControl'], \"opfTaskControlSchema.json\")\n\n taskLabel = task['taskLabel']\n\n assert isinstance(taskLabel, types.StringTypes), \\\n \"taskLabel type: %r\" % type(taskLabel)\n assert len(taskLabel) > 0, \"empty string taskLabel not is allowed\"\n\n taskLabelsList.append(taskLabel.lower())\n\n taskLabelDuplicates = filter(lambda x: taskLabelsList.count(x) > 1,\n taskLabelsList)\n assert len(taskLabelDuplicates) == 0, \\\n \"Duplcate task labels are not allowed: %s\" % taskLabelDuplicates\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nnormalize the source of a stream into a non - absolute path.", "response": "def normalizeStreamSource(self, stream):\n \"\"\"\n TODO: document\n :param stream:\n \"\"\"\n # The stream source in the task might be in several formats, so we need\n # to make sure it gets converted into an absolute path:\n source = stream['source'][len(FILE_SCHEME):]\n # If the source is already an absolute path, we won't use pkg_resources,\n # we'll just trust that the path exists. If not, it's a user problem.\n if os.path.isabs(source):\n sourcePath = source\n else:\n # First we'll check to see if this path exists within the nupic.datafiles\n # package resource.\n sourcePath = resource_filename(\"nupic.datafiles\", source)\n if not os.path.exists(sourcePath):\n # If this path doesn't exist as a package resource, the last option will\n # be to treat it as a relative path to the current working directory.\n sourcePath = os.path.join(os.getcwd(), source)\n stream['source'] = FILE_SCHEME + sourcePath"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting Nupic environment to OPF", "response": "def convertNupicEnvToOPF(self):\n \"\"\"\n TODO: document\n \"\"\"\n\n # We need to create a task structure, most of which is taken verbatim\n # from the Nupic control dict\n task = dict(self.__control)\n\n task.pop('environment')\n inferenceArgs = task.pop('inferenceArgs')\n task['taskLabel'] = 'DefaultTask'\n\n # Create the iterationCycle element that will be placed inside the\n # taskControl.\n iterationCount = task.get('iterationCount', -1)\n iterationCountInferOnly = task.pop('iterationCountInferOnly', 0)\n if iterationCountInferOnly == -1:\n iterationCycle = [IterationPhaseSpecInferOnly(1000, inferenceArgs=inferenceArgs)]\n elif iterationCountInferOnly > 0:\n assert iterationCount > 0, \"When iterationCountInferOnly is specified, \"\\\n \"iterationCount must also be specified and not be -1\"\n iterationCycle = [IterationPhaseSpecLearnAndInfer(iterationCount\n -iterationCountInferOnly, inferenceArgs=inferenceArgs),\n IterationPhaseSpecInferOnly(iterationCountInferOnly, inferenceArgs=inferenceArgs)]\n else:\n iterationCycle = [IterationPhaseSpecLearnAndInfer(1000, inferenceArgs=inferenceArgs)]\n\n\n taskControl = dict(metrics = task.pop('metrics'),\n loggedMetrics = task.pop('loggedMetrics'),\n iterationCycle = iterationCycle)\n task['taskControl'] = taskControl\n\n # Create the new control\n self.__control = dict(environment = OpfEnvironment.Nupic,\n tasks = [task])"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef createNetwork(dataSource):\n network = Network()\n\n # Our input is sensor data from the gym file. The RecordSensor region\n # allows us to specify a file record stream as the input source via the\n # dataSource attribute.\n network.addRegion(\"sensor\", \"py.RecordSensor\",\n json.dumps({\"verbosity\": _VERBOSITY}))\n sensor = network.regions[\"sensor\"].getSelf()\n # The RecordSensor needs to know how to encode the input values\n sensor.encoder = createEncoder()\n # Specify the dataSource as a file record stream instance\n sensor.dataSource = dataSource\n\n # CUSTOM REGION\n # Add path to custom region to PYTHONPATH\n # NOTE: Before using a custom region, please modify your PYTHONPATH\n # export PYTHONPATH=\":$PYTHONPATH\"\n # In this demo, we have modified it using sys.path.append since we need it to\n # have an effect on this program.\n sys.path.append(os.path.dirname(os.path.abspath(__file__)))\n \n from custom_region.identity_region import IdentityRegion\n\n # Add custom region class to the network\n Network.registerRegion(IdentityRegion)\n\n # Create a custom region\n network.addRegion(\"identityRegion\", \"py.IdentityRegion\",\n json.dumps({\n \"dataWidth\": sensor.encoder.getWidth(),\n }))\n\n # Link the Identity region to the sensor output\n network.link(\"sensor\", \"identityRegion\", \"UniformLink\", \"\")\n\n network.initialize()\n\n return network", "response": "Create a Network instance."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrun the network and write output to writer. Run the network and write output to writer.", "response": "def runNetwork(network, writer):\n \"\"\"Run the network and write output to writer.\n\n :param network: a Network instance to run\n :param writer: a csv.writer instance to write output to\n \"\"\"\n identityRegion = network.regions[\"identityRegion\"]\n\n for i in xrange(_NUM_RECORDS):\n # Run the network for a single iteration\n network.run(1)\n\n # Write out the record number and encoding\n encoding = identityRegion.getOutputData(\"out\")\n writer.writerow((i, encoding))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating a set of possible report keys for an experiment s results dict.", "response": "def _appendReportKeys(keys, prefix, results):\n \"\"\"\n Generate a set of possible report keys for an experiment's results.\n A report key is a string of key names separated by colons, each key being one\n level deeper into the experiment results dict. For example, 'key1:key2'.\n\n This routine is called recursively to build keys that are multiple levels\n deep from the results dict.\n\n Parameters:\n -----------------------------------------------------------\n keys: Set of report keys accumulated so far\n prefix: prefix formed so far, this is the colon separated list of key\n names that led up to the dict passed in results\n results: dictionary of results at this level.\n \"\"\"\n\n allKeys = results.keys()\n allKeys.sort()\n for key in allKeys:\n if hasattr(results[key], 'keys'):\n _appendReportKeys(keys, \"%s%s:\" % (prefix, key), results[key])\n else:\n keys.add(\"%s%s\" % (prefix, key))"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a list of all report keys that match one of the regular expressions passed in reportKeys.", "response": "def _matchReportKeys(reportKeyREs=[], allReportKeys=[]):\n \"\"\"\n Extract all items from the 'allKeys' list whose key matches one of the regular\n expressions passed in 'reportKeys'.\n\n Parameters:\n ----------------------------------------------------------------------------\n reportKeyREs: List of regular expressions\n allReportKeys: List of all keys\n\n retval: list of keys from allReportKeys that match the regular expressions\n in 'reportKeyREs'\n If an invalid regular expression was included in 'reportKeys',\n then BadKeyError() is raised\n \"\"\"\n\n matchingReportKeys = []\n\n # Extract the report items of interest\n for keyRE in reportKeyREs:\n # Find all keys that match this regular expression\n matchObj = re.compile(keyRE)\n found = False\n for keyName in allReportKeys:\n match = matchObj.match(keyName)\n if match and match.end() == len(keyName):\n matchingReportKeys.append(keyName)\n found = True\n if not found:\n raise _BadKeyError(keyRE)\n\n return matchingReportKeys"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _getReportItem(itemName, results):\n\n subKeys = itemName.split(':')\n subResults = results\n for subKey in subKeys:\n subResults = subResults[subKey]\n\n return subResults", "response": "Get a specific item from the results dict."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef filterResults(allResults, reportKeys, optimizeKey=None):\n\n # Init return values\n optimizeDict = dict()\n\n # Get all available report key names for this experiment\n allReportKeys = set()\n _appendReportKeys(keys=allReportKeys, prefix='', results=allResults)\n\n #----------------------------------------------------------------------------\n # Extract the report items that match the regular expressions passed in reportKeys\n matchingKeys = _matchReportKeys(reportKeys, allReportKeys)\n\n # Extract the values of the desired items\n reportDict = dict()\n for keyName in matchingKeys:\n value = _getReportItem(keyName, allResults)\n reportDict[keyName] = value\n\n\n # -------------------------------------------------------------------------\n # Extract the report item that matches the regular expression passed in\n # optimizeKey\n if optimizeKey is not None:\n matchingKeys = _matchReportKeys([optimizeKey], allReportKeys)\n if len(matchingKeys) == 0:\n raise _BadKeyError(optimizeKey)\n elif len(matchingKeys) > 1:\n raise _BadOptimizeKeyError(optimizeKey, matchingKeys)\n optimizeKeyFullName = matchingKeys[0]\n\n # Get the value of the optimize metric\n value = _getReportItem(optimizeKeyFullName, allResults)\n optimizeDict[optimizeKeyFullName] = value\n reportDict[optimizeKeyFullName] = value\n\n # Return info\n return(reportDict, optimizeDict)", "response": "Given the complete set of results generated by an experiment and a list of report keys and a single key and a single value of the value of that key in the results dict and a single key in the results dictionary and a single key in the results dictionary."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _handleModelRunnerException(jobID, modelID, jobsDAO, experimentDir, logger,\n e):\n \"\"\" Perform standard handling of an exception that occurs while running\n a model.\n\n Parameters:\n -------------------------------------------------------------------------\n jobID: ID for this hypersearch job in the jobs table\n modelID: model ID\n jobsDAO: ClientJobsDAO instance\n experimentDir: directory containing the experiment\n logger: the logger to use\n e: the exception that occurred\n retval: (completionReason, completionMsg)\n \"\"\"\n\n msg = StringIO.StringIO()\n print >>msg, \"Exception occurred while running model %s: %r (%s)\" % (\n modelID, e, type(e))\n traceback.print_exc(None, msg)\n\n completionReason = jobsDAO.CMPL_REASON_ERROR\n completionMsg = msg.getvalue()\n logger.error(completionMsg)\n\n # Write results to the model database for the error case. Ignore\n # InvalidConnectionException, as this is usually caused by orphaned models\n #\n # TODO: do we really want to set numRecords to 0? Last updated value might\n # be useful for debugging\n if type(e) is not InvalidConnectionException:\n jobsDAO.modelUpdateResults(modelID, results=None, numRecords=0)\n\n # TODO: Make sure this wasn't the best model in job. If so, set the best\n # appropriately\n\n # If this was an exception that should mark the job as failed, do that\n # now.\n if type(e) == JobFailException:\n workerCmpReason = jobsDAO.jobGetFields(jobID,\n ['workerCompletionReason'])[0]\n if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:\n jobsDAO.jobSetFields(jobID, fields=dict(\n cancel=True,\n workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,\n workerCompletionMsg = \": \".join(str(i) for i in e.args)),\n useConnectionID=False,\n ignoreUnchanged=True)\n\n return (completionReason, completionMsg)", "response": "This function handles exceptions raised while running a hypersearch model."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef runModelGivenBaseAndParams(modelID, jobID, baseDescription, params,\n predictedField, reportKeys, optimizeKey, jobsDAO,\n modelCheckpointGUID, logLevel=None, predictionCacheMaxRecords=None):\n \"\"\" This creates an experiment directory with a base.py description file\n created from 'baseDescription' and a description.py generated from the\n given params dict and then runs the experiment.\n\n Parameters:\n -------------------------------------------------------------------------\n modelID: ID for this model in the models table\n jobID: ID for this hypersearch job in the jobs table\n baseDescription: Contents of a description.py with the base experiment\n description\n params: Dictionary of specific parameters to override within\n the baseDescriptionFile.\n predictedField: Name of the input field for which this model is being\n optimized\n reportKeys: Which metrics of the experiment to store into the\n results dict of the model's database entry\n optimizeKey: Which metric we are optimizing for\n jobsDAO Jobs data access object - the interface to the\n jobs database which has the model's table.\n modelCheckpointGUID: A persistent, globally-unique identifier for\n constructing the model checkpoint key\n logLevel: override logging level to this value, if not None\n\n retval: (completionReason, completionMsg)\n \"\"\"\n from nupic.swarming.ModelRunner import OPFModelRunner\n\n # The logger for this method\n logger = logging.getLogger('com.numenta.nupic.hypersearch.utils')\n\n\n # --------------------------------------------------------------------------\n # Create a temp directory for the experiment and the description files\n experimentDir = tempfile.mkdtemp()\n try:\n logger.info(\"Using experiment directory: %s\" % (experimentDir))\n\n # Create the decription.py from the overrides in params\n paramsFilePath = os.path.join(experimentDir, 'description.py')\n paramsFile = open(paramsFilePath, 'wb')\n paramsFile.write(_paramsFileHead())\n\n items = params.items()\n items.sort()\n for (key,value) in items:\n quotedKey = _quoteAndEscape(key)\n if isinstance(value, basestring):\n\n paramsFile.write(\" %s : '%s',\\n\" % (quotedKey , value))\n else:\n paramsFile.write(\" %s : %s,\\n\" % (quotedKey , value))\n\n paramsFile.write(_paramsFileTail())\n paramsFile.close()\n\n\n # Write out the base description\n baseParamsFile = open(os.path.join(experimentDir, 'base.py'), 'wb')\n baseParamsFile.write(baseDescription)\n baseParamsFile.close()\n\n\n # Store the experiment's sub-description file into the model table\n # for reference\n fd = open(paramsFilePath)\n expDescription = fd.read()\n fd.close()\n jobsDAO.modelSetFields(modelID, {'genDescription': expDescription})\n\n\n # Run the experiment now\n try:\n runner = OPFModelRunner(\n modelID=modelID,\n jobID=jobID,\n predictedField=predictedField,\n experimentDir=experimentDir,\n reportKeyPatterns=reportKeys,\n optimizeKeyPattern=optimizeKey,\n jobsDAO=jobsDAO,\n modelCheckpointGUID=modelCheckpointGUID,\n logLevel=logLevel,\n predictionCacheMaxRecords=predictionCacheMaxRecords)\n\n signal.signal(signal.SIGINT, runner.handleWarningSignal)\n\n (completionReason, completionMsg) = runner.run()\n\n except InvalidConnectionException:\n raise\n except Exception, e:\n\n (completionReason, completionMsg) = _handleModelRunnerException(jobID,\n modelID, jobsDAO, experimentDir, logger, e)\n\n finally:\n # delete our temporary directory tree\n shutil.rmtree(experimentDir)\n signal.signal(signal.SIGINT, signal.default_int_handler)\n\n # Return completion reason and msg\n return (completionReason, completionMsg)", "response": "This function creates a temporary directory and runs the model given the base description and parameters."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a clipped version of obj suitable for printing.", "response": "def clippedObj(obj, maxElementSize=64):\n \"\"\"\n Return a clipped version of obj suitable for printing, This\n is useful when generating log messages by printing data structures, but\n don't want the message to be too long.\n\n If passed in a dict, list, or namedtuple, each element of the structure's\n string representation will be limited to 'maxElementSize' characters. This\n will return a new object where the string representation of each element\n has been truncated to fit within maxElementSize.\n \"\"\"\n\n # Is it a named tuple?\n if hasattr(obj, '_asdict'):\n obj = obj._asdict()\n\n\n # Printing a dict?\n if isinstance(obj, dict):\n objOut = dict()\n for key,val in obj.iteritems():\n objOut[key] = clippedObj(val)\n\n # Printing a list?\n elif hasattr(obj, '__iter__'):\n objOut = []\n for val in obj:\n objOut.append(clippedObj(val))\n\n # Some other object\n else:\n objOut = str(obj)\n if len(objOut) > maxElementSize:\n objOut = objOut[0:maxElementSize] + '...'\n\n return objOut"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nvalidate a python value against the json schema", "response": "def validate(value, **kwds):\n \"\"\" Validate a python value against json schema:\n validate(value, schemaPath)\n validate(value, schemaDict)\n\n value: python object to validate against the schema\n\n The json schema may be specified either as a path of the file containing\n the json schema or as a python dictionary using one of the\n following keywords as arguments:\n schemaPath: Path of file containing the json schema object.\n schemaDict: Python dictionary containing the json schema object\n\n Returns: nothing\n\n Raises:\n ValidationError when value fails json validation\n \"\"\"\n\n assert len(kwds.keys()) >= 1\n assert 'schemaPath' in kwds or 'schemaDict' in kwds\n\n schemaDict = None\n if 'schemaPath' in kwds:\n schemaPath = kwds.pop('schemaPath')\n schemaDict = loadJsonValueFromFile(schemaPath)\n elif 'schemaDict' in kwds:\n schemaDict = kwds.pop('schemaDict')\n\n try:\n validictory.validate(value, schemaDict, **kwds)\n except validictory.ValidationError as e:\n raise ValidationError(e)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading a json value from a file and converts it to a corresponding python object.", "response": "def loadJsonValueFromFile(inputFilePath):\n \"\"\" Loads a json value from a file and converts it to the corresponding python\n object.\n\n inputFilePath:\n Path of the json file;\n\n Returns:\n python value that represents the loaded json value\n\n \"\"\"\n with open(inputFilePath) as fileObj:\n value = json.load(fileObj)\n\n return value"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sortedJSONDumpS(obj):\n\n itemStrs = []\n\n if isinstance(obj, dict):\n items = obj.items()\n items.sort()\n for key, value in items:\n itemStrs.append('%s: %s' % (json.dumps(key), sortedJSONDumpS(value)))\n return '{%s}' % (', '.join(itemStrs))\n\n elif hasattr(obj, '__iter__'):\n for val in obj:\n itemStrs.append(sortedJSONDumpS(val))\n return '[%s]' % (', '.join(itemStrs))\n\n else:\n return json.dumps(obj)", "response": "Return a JSON representation of obj with sorted keys on any embedded dicts."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef tick(self):\n\n # Run activities whose time has come\n for act in self.__activities:\n if not act.iteratorHolder[0]:\n continue\n\n try:\n next(act.iteratorHolder[0])\n except StopIteration:\n act.cb()\n if act.repeating:\n act.iteratorHolder[0] = iter(xrange(act.period))\n else:\n act.iteratorHolder[0] = None\n\n return True", "response": "This method is used to tick the internal iterator of the activity objects."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncompares two python dictionaries at the top level and reports differences to stdout.", "response": "def dictDiffAndReport(da, db):\n \"\"\" Compares two python dictionaries at the top level and report differences,\n if any, to stdout\n\n da: first dictionary\n db: second dictionary\n\n Returns: The same value as returned by dictDiff() for the given args\n \"\"\"\n differences = dictDiff(da, db)\n\n if not differences:\n return differences\n\n if differences['inAButNotInB']:\n print \">>> inAButNotInB: %s\" % differences['inAButNotInB']\n\n if differences['inBButNotInA']:\n print \">>> inBButNotInA: %s\" % differences['inBButNotInA']\n\n for key in differences['differentValues']:\n print \">>> da[%s] != db[%s]\" % (key, key)\n print \"da[%s] = %r\" % (key, da[key])\n print \"db[%s] = %r\" % (key, db[key])\n\n return differences"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncompares two python dictionaries at the top level and returns differences dictionary", "response": "def dictDiff(da, db):\n \"\"\" Compares two python dictionaries at the top level and return differences\n\n da: first dictionary\n db: second dictionary\n\n Returns: None if dictionaries test equal; otherwise returns a\n dictionary as follows:\n {\n 'inAButNotInB':\n \n 'inBButNotInA':\n \n 'differentValues':\n \n }\n \"\"\"\n different = False\n\n resultDict = dict()\n\n resultDict['inAButNotInB'] = set(da) - set(db)\n if resultDict['inAButNotInB']:\n different = True\n\n resultDict['inBButNotInA'] = set(db) - set(da)\n if resultDict['inBButNotInA']:\n different = True\n\n resultDict['differentValues'] = []\n for key in (set(da) - resultDict['inAButNotInB']):\n comparisonResult = da[key] == db[key]\n if isinstance(comparisonResult, bool):\n isEqual = comparisonResult\n else:\n # This handles numpy arrays (but only at the top level)\n isEqual = comparisonResult.all()\n if not isEqual:\n resultDict['differentValues'].append(key)\n different = True\n\n assert (((resultDict['inAButNotInB'] or resultDict['inBButNotInA'] or\n resultDict['differentValues']) and different) or not different)\n\n return resultDict if different else None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _seed(self, seed=-1):\n if seed != -1:\n self.random = NupicRandom(seed)\n else:\n self.random = NupicRandom()", "response": "Initialize the random seed"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate a new and unique representation. Returns a numpy array containing the n - th element of the array.", "response": "def _newRep(self):\n \"\"\"Generate a new and unique representation. Returns a numpy array\n of shape (n,). \"\"\"\n maxAttempts = 1000\n\n for _ in xrange(maxAttempts):\n foundUnique = True\n population = numpy.arange(self.n, dtype=numpy.uint32)\n choices = numpy.arange(self.w, dtype=numpy.uint32)\n oneBits = sorted(self.random.sample(population, choices))\n sdr = numpy.zeros(self.n, dtype='uint8')\n sdr[oneBits] = 1\n for i in xrange(self.ncategories):\n if (sdr == self.sdrs[i]).all():\n foundUnique = False\n break\n if foundUnique:\n break;\n if not foundUnique:\n raise RuntimeError(\"Error, could not find unique pattern %d after \"\n \"%d attempts\" % (self.ncategories, maxAttempts))\n return sdr"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a numpy array of scalar values for the given input.", "response": "def getScalars(self, input):\n \"\"\" See method description in base.py \"\"\"\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n return numpy.array([0])\n\n index = self.categoryToIndex.get(input, None)\n if index is None:\n if self._learningEnabled:\n self._addCategory(input)\n index = self.ncategories - 1\n else:\n # if not found, we encode category 0\n index = 0\n\n return numpy.array([index])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef decode(self, encoded, parentFieldName=''):\n\n assert (encoded[0:self.n] <= 1.0).all()\n\n resultString = \"\"\n resultRanges = []\n\n overlaps = (self.sdrs * encoded[0:self.n]).sum(axis=1)\n\n if self.verbosity >= 2:\n print \"Overlaps for decoding:\"\n for i in xrange(0, self.ncategories):\n print \"%d %s\" % (overlaps[i], self.categories[i])\n\n matchingCategories = (overlaps > self.thresholdOverlap).nonzero()[0]\n\n for index in matchingCategories:\n if resultString != \"\":\n resultString += \" \"\n resultString += str(self.categories[index])\n resultRanges.append([int(index),int(index)])\n\n if parentFieldName != '':\n fieldName = \"%s.%s\" % (parentFieldName, self.name)\n else:\n fieldName = self.name\n return ({fieldName: (resultRanges, resultString)}, [fieldName])", "response": "This function decodes the encoded array into a single object."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the encoder result for the given buckets.", "response": "def getBucketInfo(self, buckets):\n \"\"\" See the function description in base.py\n \"\"\"\n\n if self.ncategories==0:\n return 0\n\n topDownMappingM = self._getTopDownMapping()\n\n categoryIndex = buckets[0]\n category = self.categories[categoryIndex]\n encoding = topDownMappingM.getRow(categoryIndex)\n\n return [EncoderResult(value=category, scalar=categoryIndex,\n encoding=encoding)]"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncomputes the top - down representation of the encoded array.", "response": "def topDownCompute(self, encoded):\n \"\"\" See the function description in base.py\n \"\"\"\n\n if self.ncategories==0:\n return 0\n\n topDownMappingM = self._getTopDownMapping()\n\n categoryIndex = topDownMappingM.rightVecProd(encoded).argmax()\n category = self.categories[categoryIndex]\n encoding = topDownMappingM.getRow(categoryIndex)\n\n return EncoderResult(value=category, scalar=categoryIndex, encoding=encoding)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a list of scalar names for each encoder in the archive.", "response": "def getScalarNames(self, parentFieldName=''):\n \"\"\" See method description in base.py \"\"\"\n\n names = []\n\n # This forms a name which is the concatenation of the parentFieldName\n # passed in and the encoder's own name.\n def _formFieldName(encoder):\n if parentFieldName == '':\n return encoder.name\n else:\n return '%s.%s' % (parentFieldName, encoder.name)\n\n # -------------------------------------------------------------------------\n # Get the scalar values for each sub-field\n if self.seasonEncoder is not None:\n names.append(_formFieldName(self.seasonEncoder))\n\n if self.dayOfWeekEncoder is not None:\n names.append(_formFieldName(self.dayOfWeekEncoder))\n\n if self.customDaysEncoder is not None:\n names.append(_formFieldName(self.customDaysEncoder))\n\n if self.weekendEncoder is not None:\n names.append(_formFieldName(self.weekendEncoder))\n\n if self.holidayEncoder is not None:\n names.append(_formFieldName(self.holidayEncoder))\n\n if self.timeOfDayEncoder is not None:\n names.append(_formFieldName(self.timeOfDayEncoder))\n\n return names"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getEncodedValues(self, input):\n\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n return numpy.array([None])\n\n assert isinstance(input, datetime.datetime)\n values = []\n\n # -------------------------------------------------------------------------\n # Get the scalar values for each sub-field\n timetuple = input.timetuple()\n timeOfDay = timetuple.tm_hour + float(timetuple.tm_min)/60.0\n\n if self.seasonEncoder is not None:\n dayOfYear = timetuple.tm_yday\n # input.timetuple() computes the day of year 1 based, so convert to 0 based\n values.append(dayOfYear-1)\n\n if self.dayOfWeekEncoder is not None:\n dayOfWeek = timetuple.tm_wday + timeOfDay / 24.0\n values.append(dayOfWeek)\n\n if self.weekendEncoder is not None:\n # saturday, sunday or friday evening\n if timetuple.tm_wday == 6 or timetuple.tm_wday == 5 \\\n or (timetuple.tm_wday == 4 and timeOfDay > 18):\n weekend = 1\n else:\n weekend = 0\n values.append(weekend)\n\n if self.customDaysEncoder is not None:\n if timetuple.tm_wday in self.customDays:\n customDay = 1\n else:\n customDay = 0\n values.append(customDay)\n if self.holidayEncoder is not None:\n # A \"continuous\" binary value. = 1 on the holiday itself and smooth ramp\n # 0->1 on the day before the holiday and 1->0 on the day after the holiday.\n # Currently the only holiday we know about is December 25\n # holidays is a list of holidays that occur on a fixed date every year\n if len(self.holidays) == 0:\n holidays = [(12, 25)]\n else:\n holidays = self.holidays\n val = 0\n for h in holidays:\n # hdate is midnight on the holiday\n if len(h) == 3:\n hdate = datetime.datetime(h[0], h[1], h[2], 0, 0, 0)\n else:\n hdate = datetime.datetime(timetuple.tm_year, h[0], h[1], 0, 0, 0)\n if input > hdate:\n diff = input - hdate\n if diff.days == 0:\n # return 1 on the holiday itself\n val = 1\n break\n elif diff.days == 1:\n # ramp smoothly from 1 -> 0 on the next day\n val = 1.0 - (float(diff.seconds) / 86400)\n break\n else:\n diff = hdate - input\n if diff.days == 0:\n # ramp smoothly from 0 -> 1 on the previous day\n val = 1.0 - (float(diff.seconds) / 86400)\n\n values.append(val)\n\n if self.timeOfDayEncoder is not None:\n values.append(timeOfDay)\n\n return values", "response": "Returns the values for the sub - field of the given date."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the indices of the sub - fields that are in the encoder chain.", "response": "def getBucketIndices(self, input):\n \"\"\" See method description in base.py \"\"\"\n\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n # Encoder each sub-field\n return [None] * len(self.encoders)\n\n else:\n assert isinstance(input, datetime.datetime)\n\n # Get the scalar values for each sub-field\n scalars = self.getScalars(input)\n\n # Encoder each sub-field\n result = []\n for i in xrange(len(self.encoders)):\n (name, encoder, offset) = self.encoders[i]\n result.extend(encoder.getBucketIndices(scalars[i]))\n return result"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nencode the input into a list of bytes.", "response": "def encodeIntoArray(self, input, output):\n \"\"\" See method description in base.py \"\"\"\n\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n output[0:] = 0\n else:\n if not isinstance(input, datetime.datetime):\n raise ValueError(\"Input is type %s, expected datetime. Value: %s\" % (\n type(input), str(input)))\n\n # Get the scalar values for each sub-field\n scalars = self.getScalars(input)\n # Encoder each sub-field\n for i in xrange(len(self.encoders)):\n (name, encoder, offset) = self.encoders[i]\n encoder.encodeIntoArray(scalars[i], output[offset:])"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the Spec for IdentityRegion.", "response": "def getSpec(cls):\n \"\"\"Return the Spec for IdentityRegion.\n \"\"\"\n spec = {\n \"description\":IdentityRegion.__doc__,\n \"singleNodeOnly\":True,\n \"inputs\":{\n \"in\":{\n \"description\":\"The input vector.\",\n \"dataType\":\"Real32\",\n \"count\":0,\n \"required\":True,\n \"regionLevel\":False,\n \"isDefaultInput\":True,\n \"requireSplitterMap\":False},\n },\n \"outputs\":{\n \"out\":{\n \"description\":\"A copy of the input vector.\",\n \"dataType\":\"Real32\",\n \"count\":0,\n \"regionLevel\":True,\n \"isDefaultOutput\":True},\n },\n\n \"parameters\":{\n \"dataWidth\":{\n \"description\":\"Size of inputs\",\n \"accessMode\":\"Read\",\n \"dataType\":\"UInt32\",\n \"count\":1,\n \"constraints\":\"\"},\n },\n }\n\n return spec"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _setRandomEncoderResolution(minResolution=0.001):\n encoder = (\n model_params.MODEL_PARAMS[\"modelParams\"][\"sensorParams\"][\"encoders\"][\"value\"]\n )\n\n if encoder[\"type\"] == \"RandomDistributedScalarEncoder\":\n rangePadding = abs(_INPUT_MAX - _INPUT_MIN) * 0.2\n minValue = _INPUT_MIN - rangePadding\n maxValue = _INPUT_MAX + rangePadding\n resolution = max(minResolution,\n (maxValue - minValue) / encoder.pop(\"numBuckets\")\n )\n encoder[\"resolution\"] = resolution", "response": "Sets the resolution for RandomDistributed encoder."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef addLabel(self, start, end, labelName):\n if len(self.saved_states) == 0:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for 'addLabel'. \"\n \"Model has no saved records.\")\n\n startID = self.saved_states[0].ROWID\n\n clippedStart = max(0, start - startID)\n clippedEnd = max(0, min( len( self.saved_states) , end - startID))\n\n if clippedEnd <= clippedStart:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for 'addLabel'.\",\n debugInfo={\n 'requestRange': {\n 'startRecordID': start,\n 'endRecordID': end\n },\n 'clippedRequestRange': {\n 'startRecordID': clippedStart,\n 'endRecordID': clippedEnd\n },\n 'validRange': {\n 'startRecordID': startID,\n 'endRecordID': self.saved_states[len(self.saved_states)-1].ROWID\n },\n 'numRecordsStored': len(self.saved_states)\n })\n\n # Add label to range [clippedStart, clippedEnd)\n for state in self.saved_states[clippedStart:clippedEnd]:\n if labelName not in state.anomalyLabel:\n state.anomalyLabel.append(labelName)\n state.setByUser = True\n self._addRecordToKNN(state)\n\n assert len(self.saved_categories) > 0\n\n # Recompute [end, ...)\n for state in self.saved_states[clippedEnd:]:\n self._updateState(state)", "response": "Adds a label to each record with record ROWID in range from start to end noninclusive of end."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nremove all records from the KNN that are in the specified range from start to end.", "response": "def removeLabels(self, start=None, end=None, labelFilter=None):\n \"\"\"\n Remove labels from each record with record ROWID in range from\n start to end, noninclusive of end. Removes all records if labelFilter is\n None, otherwise only removes the labels eqaul to labelFilter.\n\n This will recalculate all points from end to the last record stored in the\n internal cache of this classifier.\n \"\"\"\n\n if len(self.saved_states) == 0:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for \"\n \"'removeLabels'. Model has no saved records.\")\n\n startID = self.saved_states[0].ROWID\n\n clippedStart = 0 if start is None else max(0, start - startID)\n clippedEnd = len(self.saved_states) if end is None else \\\n max(0, min( len( self.saved_states) , end - startID))\n\n if clippedEnd <= clippedStart:\n raise HTMPredictionModelInvalidRangeError(\"Invalid supplied range for \"\n \"'removeLabels'.\", debugInfo={\n 'requestRange': {\n 'startRecordID': start,\n 'endRecordID': end\n },\n 'clippedRequestRange': {\n 'startRecordID': clippedStart,\n 'endRecordID': clippedEnd\n },\n 'validRange': {\n 'startRecordID': startID,\n 'endRecordID': self.saved_states[len(self.saved_states)-1].ROWID\n },\n 'numRecordsStored': len(self.saved_states)\n })\n\n # Remove records within the cache\n recordsToDelete = []\n for state in self.saved_states[clippedStart:clippedEnd]:\n if labelFilter is not None:\n if labelFilter in state.anomalyLabel:\n state.anomalyLabel.remove(labelFilter)\n else:\n state.anomalyLabel = []\n state.setByUser = False\n recordsToDelete.append(state)\n self._deleteRecordsFromKNN(recordsToDelete)\n\n # Remove records not in cache\n self._deleteRangeFromKNN(start, end)\n\n # Recompute [clippedEnd, ...)\n for state in self.saved_states[clippedEnd:]:\n self._updateState(state)\n\n return {'status': 'success'}"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _addRecordToKNN(self, record):\n classifier = self.htm_prediction_model._getAnomalyClassifier()\n knn = classifier.getSelf()._knn\n\n prototype_idx = classifier.getSelf().getParameter('categoryRecencyList')\n category = self._labelListToCategoryNumber(record.anomalyLabel)\n\n # If record is already in the classifier, overwrite its labeling\n if record.ROWID in prototype_idx:\n knn.prototypeSetCategory(record.ROWID, category)\n return\n\n # Learn this pattern in the knn\n pattern = self._getStateAnomalyVector(record)\n rowID = record.ROWID\n knn.learn(pattern, category, rowID=rowID)", "response": "This method will add the record to the KNN classifier."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _recomputeRecordFromKNN(self, record):\n inputs = {\n \"categoryIn\": [None],\n \"bottomUpIn\": self._getStateAnomalyVector(record),\n }\n\n outputs = {\"categoriesOut\": numpy.zeros((1,)),\n \"bestPrototypeIndices\":numpy.zeros((1,)),\n \"categoryProbabilitiesOut\":numpy.zeros((1,))}\n\n # Run inference only to capture state before learning\n classifier = self.htm_prediction_model._getAnomalyClassifier()\n knn = classifier.getSelf()._knn\n\n # Only use points before record to classify and after the wait period.\n classifier_indexes = \\\n numpy.array(classifier.getSelf().getParameter('categoryRecencyList'))\n valid_idx = numpy.where(\n (classifier_indexes >= self._autoDetectWaitRecords) &\n (classifier_indexes < record.ROWID)\n )[0].tolist()\n\n if len(valid_idx) == 0:\n return None\n\n classifier.setParameter('inferenceMode', True)\n classifier.setParameter('learningMode', False)\n classifier.getSelf().compute(inputs, outputs)\n classifier.setParameter('learningMode', True)\n\n classifier_distances = classifier.getSelf().getLatestDistances()\n valid_distances = classifier_distances[valid_idx]\n if valid_distances.min() <= self._classificationMaxDist:\n classifier_indexes_prev = classifier_indexes[valid_idx]\n rowID = classifier_indexes_prev[valid_distances.argmin()]\n indexID = numpy.where(classifier_indexes == rowID)[0][0]\n category = classifier.getSelf().getCategoryList()[indexID]\n return category\n return None", "response": "Recompute the record from KNN."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconstructs a _HTMClassificationRecord based on the current state of the current classifier and the current state of the current classifier.", "response": "def _constructClassificationRecord(self):\n \"\"\"\n Construct a _HTMClassificationRecord based on the current state of the\n htm_prediction_model of this classifier.\n\n ***This will look into the internals of the model and may depend on the\n SP, TM, and KNNClassifier***\n \"\"\"\n model = self.htm_prediction_model\n sp = model._getSPRegion()\n tm = model._getTPRegion()\n tpImp = tm.getSelf()._tfdr\n\n # Count the number of unpredicted columns\n activeColumns = sp.getOutputData(\"bottomUpOut\").nonzero()[0]\n score = numpy.in1d(activeColumns, self._prevPredictedColumns).sum()\n score = (self._activeColumnCount - score)/float(self._activeColumnCount)\n\n spSize = sp.getParameter('activeOutputCount')\n tpSize = tm.getParameter('cellsPerColumn') * tm.getParameter('columnCount')\n\n classificationVector = numpy.array([])\n\n if self._vectorType == 'tpc':\n # Classification Vector: [---TM Cells---]\n classificationVector = numpy.zeros(tpSize)\n activeCellMatrix = tpImp.getLearnActiveStateT().reshape(tpSize, 1)\n activeCellIdx = numpy.where(activeCellMatrix > 0)[0]\n if activeCellIdx.shape[0] > 0:\n classificationVector[numpy.array(activeCellIdx, dtype=numpy.uint16)] = 1\n elif self._vectorType == 'sp_tpe':\n # Classification Vecotr: [---SP---|---(TM-SP)----]\n classificationVector = numpy.zeros(spSize+spSize)\n if activeColumns.shape[0] > 0:\n classificationVector[activeColumns] = 1.0\n\n errorColumns = numpy.setdiff1d(self._prevPredictedColumns, activeColumns)\n if errorColumns.shape[0] > 0:\n errorColumnIndexes = ( numpy.array(errorColumns, dtype=numpy.uint16) +\n spSize )\n classificationVector[errorColumnIndexes] = 1.0\n else:\n raise TypeError(\"Classification vector type must be either 'tpc' or\"\n \" 'sp_tpe', current value is %s\" % (self._vectorType))\n\n # Store the state for next time step\n numPredictedCols = len(self._prevPredictedColumns)\n predictedColumns = tm.getOutputData(\"topDownOut\").nonzero()[0]\n self._prevPredictedColumns = copy.deepcopy(predictedColumns)\n\n if self._anomalyVectorLength is None:\n self._anomalyVectorLength = len(classificationVector)\n\n result = _CLAClassificationRecord(\n ROWID=int(model.getParameter('__numRunCalls') - 1), #__numRunCalls called\n #at beginning of model.run\n anomalyScore=score,\n anomalyVector=classificationVector.nonzero()[0].tolist(),\n anomalyLabel=[]\n )\n return result"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef compute(self):\n result = self._constructClassificationRecord()\n\n # Classify this point after waiting the classification delay\n if result.ROWID >= self._autoDetectWaitRecords:\n self._updateState(result)\n\n # Save new classification record and keep history as moving window\n self.saved_states.append(result)\n if len(self.saved_states) > self._history_length:\n self.saved_states.pop(0)\n\n return result", "response": "Compute an anomaly classifier and save it to the history."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef setAutoDetectWaitRecords(self, waitRecords):\n if not isinstance(waitRecords, int):\n raise HTMPredictionModelInvalidArgument(\"Invalid argument type \\'%s\\'. WaitRecord \"\n \"must be a number.\" % (type(waitRecords)))\n\n if len(self.saved_states) > 0 and waitRecords < self.saved_states[0].ROWID:\n raise HTMPredictionModelInvalidArgument(\"Invalid value. autoDetectWaitRecord value \"\n \"must be valid record within output stream. Current minimum ROWID in \"\n \"output stream is %d.\" % (self.saved_states[0].ROWID))\n\n self._autoDetectWaitRecords = waitRecords\n\n # Update all the states in the classifier's cache\n for state in self.saved_states:\n self._updateState(state)", "response": "Sets the autoDetectWaitRecords property of the classifier."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef setAutoDetectThreshold(self, threshold):\n if not (isinstance(threshold, float) or isinstance(threshold, int)):\n raise HTMPredictionModelInvalidArgument(\"Invalid argument type \\'%s\\'. threshold \"\n \"must be a number.\" % (type(threshold)))\n\n self._autoDetectThreshold = threshold\n\n # Update all the states in the classifier's cache\n for state in self.saved_states:\n self._updateState(state)", "response": "Sets the autoDetectThreshold of the classifier."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds the additional specs in three groups based on the selected implementation.", "response": "def _getAdditionalSpecs(spatialImp, kwargs={}):\n \"\"\"Build the additional specs in three groups (for the inspector)\n\n Use the type of the default argument to set the Spec type, defaulting\n to 'Byte' for None and complex types\n\n Determines the spatial parameters based on the selected implementation.\n It defaults to SpatialPooler.\n \"\"\"\n typeNames = {int: 'UInt32', float: 'Real32', str: 'Byte', bool: 'bool', tuple: 'tuple'}\n\n def getArgType(arg):\n t = typeNames.get(type(arg), 'Byte')\n count = 0 if t == 'Byte' else 1\n if t == 'tuple':\n t = typeNames.get(type(arg[0]), 'Byte')\n count = len(arg)\n if t == 'bool':\n t = 'UInt32'\n return (t, count)\n\n def getConstraints(arg):\n t = typeNames.get(type(arg), 'Byte')\n if t == 'Byte':\n return 'multiple'\n elif t == 'bool':\n return 'bool'\n else:\n return ''\n\n # Get arguments from spatial pooler constructors, figure out types of\n # variables and populate spatialSpec.\n SpatialClass = getSPClass(spatialImp)\n sArgTuples = _buildArgs(SpatialClass.__init__)\n spatialSpec = {}\n for argTuple in sArgTuples:\n d = dict(\n description=argTuple[1],\n accessMode='ReadWrite',\n dataType=getArgType(argTuple[2])[0],\n count=getArgType(argTuple[2])[1],\n constraints=getConstraints(argTuple[2]))\n spatialSpec[argTuple[0]] = d\n\n # Add special parameters that weren't handled automatically\n # Spatial parameters only!\n spatialSpec.update(dict(\n\n columnCount=dict(\n description='Total number of columns (coincidences).',\n accessMode='Read',\n dataType='UInt32',\n count=1,\n constraints=''),\n\n inputWidth=dict(\n description='Size of inputs to the SP.',\n accessMode='Read',\n dataType='UInt32',\n count=1,\n constraints=''),\n\n spInputNonZeros=dict(\n description='The indices of the non-zero inputs to the spatial pooler',\n accessMode='Read',\n dataType='UInt32',\n count=0,\n constraints=''),\n\n spOutputNonZeros=dict(\n description='The indices of the non-zero outputs from the spatial pooler',\n accessMode='Read',\n dataType='UInt32',\n count=0,\n constraints=''),\n\n spOverlapDistribution=dict(\n description=\"\"\"The overlaps between the active output coincidences\n and the input. The overlap amounts for each coincidence are sorted\n from highest to lowest. \"\"\",\n accessMode='Read',\n dataType='Real32',\n count=0,\n constraints=''),\n\n sparseCoincidenceMatrix=dict(\n description='The coincidences, as a SparseMatrix',\n accessMode='Read',\n dataType='Byte',\n count=0,\n constraints=''),\n\n denseOutput=dict(\n description='Score for each coincidence.',\n accessMode='Read',\n dataType='Real32',\n count=0,\n constraints=''),\n\n spLearningStatsStr=dict(\n description=\"\"\"String representation of dictionary containing a number\n of statistics related to learning.\"\"\",\n accessMode='Read',\n dataType='Byte',\n count=0,\n constraints='handle'),\n\n spatialImp=dict(\n description=\"\"\"Which spatial pooler implementation to use. Set to either\n 'py', or 'cpp'. The 'cpp' implementation is optimized for\n speed in C++.\"\"\",\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints='enum: py, cpp'),\n ))\n\n\n # The last group is for parameters that aren't specific to spatial pooler\n otherSpec = dict(\n learningMode=dict(\n description='1 if the node is learning (default 1).',\n accessMode='ReadWrite',\n dataType='UInt32',\n count=1,\n constraints='bool'),\n\n inferenceMode=dict(\n description='1 if the node is inferring (default 0).',\n accessMode='ReadWrite',\n dataType='UInt32',\n count=1,\n constraints='bool'),\n\n anomalyMode=dict(\n description='1 if an anomaly score is being computed',\n accessMode='ReadWrite',\n dataType='UInt32',\n count=1,\n constraints='bool'),\n\n topDownMode=dict(\n description='1 if the node should do top down compute on the next call '\n 'to compute into topDownOut (default 0).',\n accessMode='ReadWrite',\n dataType='UInt32',\n count=1,\n constraints='bool'),\n\n activeOutputCount=dict(\n description='Number of active elements in bottomUpOut output.',\n accessMode='Read',\n dataType='UInt32',\n count=1,\n constraints=''),\n\n logPathInput=dict(\n description='Optional name of input log file. If set, every input vector'\n ' will be logged to this file.',\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints=''),\n\n logPathOutput=dict(\n description='Optional name of output log file. If set, every output vector'\n ' will be logged to this file.',\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints=''),\n\n logPathOutputDense=dict(\n description='Optional name of output log file. If set, every output vector'\n ' will be logged to this file as a dense vector.',\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints=''),\n\n )\n\n return spatialSpec, otherSpec"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _initializeEphemeralMembers(self):\n\n for attrName in self._getEphemeralMembersBase():\n if attrName != \"_loaded\":\n if hasattr(self, attrName):\n if self._loaded:\n # print self.__class__.__name__, \"contains base class member '%s' \" \\\n # \"after loading.\" % attrName\n # TODO: Re-enable warning or turn into error in a future release.\n pass\n else:\n print self.__class__.__name__, \"contains base class member '%s'\" % \\\n attrName\n if not self._loaded:\n for attrName in self._getEphemeralMembersBase():\n if attrName != \"_loaded\":\n # if hasattr(self, attrName):\n # import pdb; pdb.set_trace()\n assert not hasattr(self, attrName)\n else:\n assert hasattr(self, attrName)\n\n # Profiling information\n self._profileObj = None\n self._iterations = 0\n\n # Let derived class initialize ephemerals\n self._initEphemerals()\n self._checkEphemeralMembers()", "response": "Initialize all ephemeral data members and give the derived class a chance to do the same by invoking the virtual member _initEphemerals method."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize the spatial pooler.", "response": "def initialize(self):\n \"\"\"\n Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.initialize`.\n \"\"\"\n # Zero out the spatial output in case it is requested\n self._spatialPoolerOutput = numpy.zeros(self.columnCount,\n dtype=GetNTAReal())\n\n # Zero out the rfInput in case it is requested\n self._spatialPoolerInput = numpy.zeros((1, self.inputWidth),\n dtype=GetNTAReal())\n\n # Allocate the spatial pooler\n self._allocateSpatialFDR(None)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nallocating the spatial pooler instance.", "response": "def _allocateSpatialFDR(self, rfInput):\n \"\"\"Allocate the spatial pooler instance.\"\"\"\n if self._sfdr:\n return\n\n # Retrieve the necessary extra arguments that were handled automatically\n autoArgs = dict((name, getattr(self, name))\n for name in self._spatialArgNames)\n\n # Instantiate the spatial pooler class.\n if ( (self.SpatialClass == CPPSpatialPooler) or\n (self.SpatialClass == PYSpatialPooler) ):\n\n autoArgs['columnDimensions'] = [self.columnCount]\n autoArgs['inputDimensions'] = [self.inputWidth]\n autoArgs['potentialRadius'] = self.inputWidth\n\n self._sfdr = self.SpatialClass(\n **autoArgs\n )"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute(self, inputs, outputs):\n\n # Uncomment this to find out who is generating divide by 0, or other numpy warnings\n # numpy.seterr(divide='raise', invalid='raise', over='raise')\n\n # Modify this line to turn on profiling for a given node. The results file\n # ('hotshot.stats') will be sensed and printed out by the vision framework's\n # RunInference.py script at the end of inference.\n # Also uncomment the hotshot import at the top of this file.\n if False and self.learningMode \\\n and self._iterations > 0 and self._iterations <= 10:\n\n import hotshot\n if self._iterations == 10:\n print \"\\n Collecting and sorting internal node profiling stats generated by hotshot...\"\n stats = hotshot.stats.load(\"hotshot.stats\")\n stats.strip_dirs()\n stats.sort_stats('time', 'calls')\n stats.print_stats()\n\n # The guts of the compute are contained in the _compute() call so that we\n # can profile it if requested.\n if self._profileObj is None:\n print \"\\n Preparing to capture profile using hotshot...\"\n if os.path.exists('hotshot.stats'):\n # There is an old hotshot stats profile left over, remove it.\n os.remove('hotshot.stats')\n self._profileObj = hotshot.Profile(\"hotshot.stats\", 1, 1)\n # filename, lineevents, linetimings\n self._profileObj.runcall(self._compute, *[inputs, outputs])\n else:\n self._compute(inputs, outputs)", "response": "Run one iteration profiling it if requested."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _compute(self, inputs, outputs):\n\n #if self.topDownMode and (not 'topDownIn' in inputs):\n # raise RuntimeError(\"The input topDownIn must be linked in if \"\n # \"topDownMode is True\")\n\n if self._sfdr is None:\n raise RuntimeError(\"Spatial pooler has not been initialized\")\n\n\n if not self.topDownMode:\n #\n # BOTTOM-UP compute\n #\n\n self._iterations += 1\n\n # Get our inputs into numpy arrays\n buInputVector = inputs['bottomUpIn']\n\n resetSignal = False\n if 'resetIn' in inputs:\n assert len(inputs['resetIn']) == 1\n resetSignal = inputs['resetIn'][0] != 0\n\n # Perform inference and/or learning\n rfOutput = self._doBottomUpCompute(\n rfInput = buInputVector.reshape((1,buInputVector.size)),\n resetSignal = resetSignal\n )\n\n outputs['bottomUpOut'][:] = rfOutput.flat\n\n else:\n #\n # TOP-DOWN inference\n #\n\n topDownIn = inputs.get('topDownIn',None)\n spatialTopDownOut, temporalTopDownOut = self._doTopDownInfer(topDownIn)\n outputs['spatialTopDownOut'][:] = spatialTopDownOut\n if temporalTopDownOut is not None:\n outputs['temporalTopDownOut'][:] = temporalTopDownOut\n\n\n # OBSOLETE\n outputs['anomalyScore'][:] = 0", "response": "Compute the SPRegion s top - down and anomaly score."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndo one iteration of inference and/or learning and return the result Parameters: -------------------------------------------- rfInput: Input vector. Shape is: (1, inputVectorLen). resetSignal: True if reset is asserted", "response": "def _doBottomUpCompute(self, rfInput, resetSignal):\n \"\"\"\n Do one iteration of inference and/or learning and return the result\n\n Parameters:\n --------------------------------------------\n rfInput: Input vector. Shape is: (1, inputVectorLen).\n resetSignal: True if reset is asserted\n\n \"\"\"\n\n # Conditional compute break\n self._conditionalBreak()\n\n # Save the rfInput for the spInputNonZeros parameter\n self._spatialPoolerInput = rfInput.reshape(-1)\n assert(rfInput.shape[0] == 1)\n\n # Run inference using the spatial pooler. We learn on the coincidences only\n # if we are in learning mode and trainingStep is set appropriately.\n\n # Run SFDR bottom-up compute and cache output in self._spatialPoolerOutput\n\n inputVector = numpy.array(rfInput[0]).astype('uint32')\n outputVector = numpy.zeros(self._sfdr.getNumColumns()).astype('uint32')\n\n self._sfdr.compute(inputVector, self.learningMode, outputVector)\n\n self._spatialPoolerOutput[:] = outputVector[:]\n\n # Direct logging of SP outputs if requested\n if self._fpLogSP:\n output = self._spatialPoolerOutput.reshape(-1)\n outputNZ = output.nonzero()[0]\n outStr = \" \".join([\"%d\" % int(token) for token in outputNZ])\n print >>self._fpLogSP, output.size, outStr\n\n # Direct logging of SP inputs\n if self._fpLogSPInput:\n output = rfInput.reshape(-1)\n outputNZ = output.nonzero()[0]\n outStr = \" \".join([\"%d\" % int(token) for token in outputNZ])\n print >>self._fpLogSPInput, output.size, outStr\n\n return self._spatialPoolerOutput"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a base Spec for the specified base class.", "response": "def getBaseSpec(cls):\n \"\"\"\n Doesn't include the spatial, temporal and other parameters\n\n :returns: (dict) The base Spec for SPRegion.\n \"\"\"\n spec = dict(\n description=SPRegion.__doc__,\n singleNodeOnly=True,\n inputs=dict(\n bottomUpIn=dict(\n description=\"\"\"The input vector.\"\"\",\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=False,\n isDefaultInput=True,\n requireSplitterMap=False),\n\n resetIn=dict(\n description=\"\"\"A boolean flag that indicates whether\n or not the input vector received in this compute cycle\n represents the start of a new temporal sequence.\"\"\",\n dataType='Real32',\n count=1,\n required=False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n topDownIn=dict(\n description=\"\"\"The top-down input signal, generated from\n feedback from upper levels\"\"\",\n dataType='Real32',\n count=0,\n required = False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n sequenceIdIn=dict(\n description=\"Sequence ID\",\n dataType='UInt64',\n count=1,\n required=False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n ),\n outputs=dict(\n bottomUpOut=dict(\n description=\"\"\"The output signal generated from the bottom-up inputs\n from lower levels.\"\"\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=True),\n\n topDownOut=dict(\n description=\"\"\"The top-down output signal, generated from\n feedback from upper levels\"\"\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n spatialTopDownOut = dict(\n description=\"\"\"The top-down output, generated only from the current\n SP output. This can be used to evaluate how well the\n SP is representing the inputs independent of the TM.\"\"\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n temporalTopDownOut = dict(\n description=\"\"\"The top-down output, generated only from the current\n TM output feedback down through the SP.\"\"\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n anomalyScore = dict(\n description=\"\"\"The score for how 'anomalous' (i.e. rare) this spatial\n input pattern is. Higher values are increasingly rare\"\"\",\n dataType='Real32',\n count=1,\n regionLevel=True,\n isDefaultOutput=False),\n ),\n\n parameters=dict(\n breakPdb=dict(\n description='Set to 1 to stop in the pdb debugger on the next compute',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='ReadWrite'),\n\n breakKomodo=dict(\n description='Set to 1 to stop in the Komodo debugger on the next compute',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='ReadWrite'),\n\n ),\n )\n\n return spec"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getSpec(cls):\n spec = cls.getBaseSpec()\n s, o = _getAdditionalSpecs(spatialImp=getDefaultSPImp())\n spec['parameters'].update(s)\n spec['parameters'].update(o)\n\n return spec", "response": "Returns the base spec for this region."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getParameter(self, parameterName, index=-1):\n\n if parameterName == 'activeOutputCount':\n return self.columnCount\n elif parameterName == 'spatialPoolerInput':\n return list(self._spatialPoolerInput.reshape(-1))\n elif parameterName == 'spatialPoolerOutput':\n return list(self._spatialPoolerOutput)\n elif parameterName == 'spNumActiveOutputs':\n return len(self._spatialPoolerOutput.nonzero()[0])\n elif parameterName == 'spOutputNonZeros':\n return [len(self._spatialPoolerOutput)] + \\\n list(self._spatialPoolerOutput.nonzero()[0])\n elif parameterName == 'spInputNonZeros':\n import pdb; pdb.set_trace()\n return [len(self._spatialPoolerInput)] + \\\n list(self._spatialPoolerInput.nonzero()[0])\n elif parameterName == 'spLearningStatsStr':\n try:\n return str(self._sfdr.getLearningStats())\n except:\n return str(dict())\n else:\n return PyRegion.getParameter(self, parameterName, index)", "response": "Returns the value of the parameter with the specified name."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef setParameter(self, parameterName, index, parameterValue):\n if parameterName in self._spatialArgNames:\n setattr(self._sfdr, parameterName, parameterValue)\n\n elif parameterName == \"logPathInput\":\n self.logPathInput = parameterValue\n # Close any existing log file\n if self._fpLogSPInput:\n self._fpLogSPInput.close()\n self._fpLogSPInput = None\n # Open a new log file\n if parameterValue:\n self._fpLogSPInput = open(self.logPathInput, 'w')\n\n elif parameterName == \"logPathOutput\":\n self.logPathOutput = parameterValue\n # Close any existing log file\n if self._fpLogSP:\n self._fpLogSP.close()\n self._fpLogSP = None\n # Open a new log file\n if parameterValue:\n self._fpLogSP = open(self.logPathOutput, 'w')\n\n elif parameterName == \"logPathOutputDense\":\n self.logPathOutputDense = parameterValue\n # Close any existing log file\n if self._fpLogSPDense:\n self._fpLogSPDense.close()\n self._fpLogSPDense = None\n # Open a new log file\n if parameterValue:\n self._fpLogSPDense = open(self.logPathOutputDense, 'w')\n\n elif hasattr(self, parameterName):\n setattr(self, parameterName, parameterValue)\n\n else:\n raise Exception('Unknown parameter: ' + parameterName)", "response": "Sets the value of a parameter in the Spec."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef writeToProto(self, proto):\n proto.spatialImp = self.spatialImp\n proto.columnCount = self.columnCount\n proto.inputWidth = self.inputWidth\n proto.learningMode = 1 if self.learningMode else 0\n proto.inferenceMode = 1 if self.inferenceMode else 0\n proto.anomalyMode = 1 if self.anomalyMode else 0\n proto.topDownMode = 1 if self.topDownMode else 0\n\n self._sfdr.write(proto.spatialPooler)", "response": "Writes the state of this capnproto object to the given proto object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef readFromProto(cls, proto):\n instance = cls(proto.columnCount, proto.inputWidth)\n\n instance.spatialImp = proto.spatialImp\n instance.learningMode = proto.learningMode\n instance.inferenceMode = proto.inferenceMode\n instance.anomalyMode = proto.anomalyMode\n instance.topDownMode = proto.topDownMode\n\n spatialImp = proto.spatialImp\n instance._sfdr = getSPClass(spatialImp).read(proto.spatialPooler)\n\n return instance", "response": "Reads a sequence of objects from a proto object."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninitializes all ephemerals used by this object.", "response": "def _initEphemerals(self):\n \"\"\"\n Initialize all ephemerals used by derived classes.\n \"\"\"\n\n if hasattr(self, '_sfdr') and self._sfdr:\n self._spatialPoolerOutput = numpy.zeros(self.columnCount,\n dtype=GetNTAReal())\n else:\n self._spatialPoolerOutput = None # Will be filled in initInNetwork\n\n # Direct logging support (faster than node watch)\n self._fpLogSPInput = None\n self._fpLogSP = None\n self._fpLogSPDense = None\n self.logPathInput = \"\"\n self.logPathOutput = \"\"\n self.logPathOutputDense = \"\""} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the number of arrays with the given name.", "response": "def getParameterArrayCount(self, name, index):\n \"\"\"\n Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.getParameterArrayCount`.\n\n TODO: as a temporary hack, getParameterArrayCount checks to see if there's a\n variable, private or not, with that name. If so, it returns the value of the\n variable.\n \"\"\"\n p = self.getParameter(name)\n if (not hasattr(p, '__len__')):\n raise Exception(\"Attempt to access parameter '%s' as an array but it is not an array\" % name)\n return len(p)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\noverrides the getParameterArray method to return the value of the named parameter.", "response": "def getParameterArray(self, name, index, a):\n \"\"\"\n Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.getParameterArray`.\n\n TODO: as a temporary hack, getParameterArray checks to see if there's a\n variable, private or not, with that name. If so, it returns the value of the\n variable.\n \"\"\"\n p = self.getParameter(name)\n if (not hasattr(p, '__len__')):\n raise Exception(\"Attempt to access parameter '%s' as an array but it is not an array\" % name)\n\n if len(p) > 0:\n a[:] = p[:]"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfigure out whether reset sequenceId both or neither are present in the data.", "response": "def _cacheSequenceInfoType(self):\n \"\"\"Figure out whether reset, sequenceId,\n both or neither are present in the data.\n Compute once instead of every time.\n\n Taken from filesource.py\"\"\"\n\n hasReset = self.resetFieldName is not None\n hasSequenceId = self.sequenceIdFieldName is not None\n\n if hasReset and not hasSequenceId:\n self._sequenceInfoType = self.SEQUENCEINFO_RESET_ONLY\n self._prevSequenceId = 0\n elif not hasReset and hasSequenceId:\n self._sequenceInfoType = self.SEQUENCEINFO_SEQUENCEID_ONLY\n self._prevSequenceId = None\n elif hasReset and hasSequenceId:\n self._sequenceInfoType = self.SEQUENCEINFO_BOTH\n else:\n self._sequenceInfoType = self.SEQUENCEINFO_NONE"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the class corresponding to the given temporalImp string \"\"\"", "response": "def _getTPClass(temporalImp):\n \"\"\" Return the class corresponding to the given temporalImp string\n \"\"\"\n\n if temporalImp == 'py':\n return backtracking_tm.BacktrackingTM\n elif temporalImp == 'cpp':\n return backtracking_tm_cpp.BacktrackingTMCPP\n elif temporalImp == 'tm_py':\n return backtracking_tm_shim.TMShim\n elif temporalImp == 'tm_cpp':\n return backtracking_tm_shim.TMCPPShim\n elif temporalImp == 'monitored_tm_py':\n return backtracking_tm_shim.MonitoredTMShim\n else:\n raise RuntimeError(\"Invalid temporalImp '%s'. Legal values are: 'py', \"\n \"'cpp', 'tm_py', 'monitored_tm_py'\" % (temporalImp))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _buildArgs(f, self=None, kwargs={}):\n # Get the name, description, and default value for each argument\n argTuples = getArgumentDescriptions(f)\n argTuples = argTuples[1:] # Remove 'self'\n\n # Get the names of the parameters to our own constructor and remove them\n # Check for _originial_init first, because if LockAttributesMixin is used,\n # __init__'s signature will be just (self, *args, **kw), but\n # _original_init is created with the original signature\n #init = getattr(self, '_original_init', self.__init__)\n init = TMRegion.__init__\n ourArgNames = [t[0] for t in getArgumentDescriptions(init)]\n # Also remove a few other names that aren't in our constructor but are\n # computed automatically (e.g. numberOfCols for the TM)\n ourArgNames += [\n 'numberOfCols', # TM\n ]\n for argTuple in argTuples[:]:\n if argTuple[0] in ourArgNames:\n argTuples.remove(argTuple)\n\n # Build the dictionary of arguments\n if self:\n for argTuple in argTuples:\n argName = argTuple[0]\n if argName in kwargs:\n # Argument was provided\n argValue = kwargs.pop(argName)\n else:\n # Argument was not provided; use the default value if there is one, and\n # raise an exception otherwise\n if len(argTuple) == 2:\n # No default value\n raise TypeError(\"Must provide '%s'\" % argName)\n argValue = argTuple[2]\n # Set as an instance variable if 'self' was passed in\n setattr(self, argName, argValue)\n\n return argTuples", "response": "Build the arguments for the function f."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbuilding the additional specs for the specified temporal memory.", "response": "def _getAdditionalSpecs(temporalImp, kwargs={}):\n \"\"\"Build the additional specs in three groups (for the inspector)\n\n Use the type of the default argument to set the Spec type, defaulting\n to 'Byte' for None and complex types\n\n Determines the spatial parameters based on the selected implementation.\n It defaults to TemporalMemory.\n Determines the temporal parameters based on the temporalImp\n \"\"\"\n typeNames = {int: 'UInt32', float: 'Real32', str: 'Byte', bool: 'bool', tuple: 'tuple'}\n\n def getArgType(arg):\n t = typeNames.get(type(arg), 'Byte')\n count = 0 if t == 'Byte' else 1\n if t == 'tuple':\n t = typeNames.get(type(arg[0]), 'Byte')\n count = len(arg)\n if t == 'bool':\n t = 'UInt32'\n return (t, count)\n\n def getConstraints(arg):\n t = typeNames.get(type(arg), 'Byte')\n if t == 'Byte':\n return 'multiple'\n elif t == 'bool':\n return 'bool'\n else:\n return ''\n\n # Build up parameters from temporal memory's constructor\n TemporalClass = _getTPClass(temporalImp)\n tArgTuples = _buildArgs(TemporalClass.__init__)\n temporalSpec = {}\n for argTuple in tArgTuples:\n d = dict(\n description=argTuple[1],\n accessMode='ReadWrite',\n dataType=getArgType(argTuple[2])[0],\n count=getArgType(argTuple[2])[1],\n constraints=getConstraints(argTuple[2]))\n temporalSpec[argTuple[0]] = d\n\n # Add temporal parameters that weren't handled automatically\n temporalSpec.update(dict(\n columnCount=dict(\n description='Total number of columns.',\n accessMode='Read',\n dataType='UInt32',\n count=1,\n constraints=''),\n\n cellsPerColumn=dict(\n description='Number of cells per column.',\n accessMode='Read',\n dataType='UInt32',\n count=1,\n constraints=''),\n\n inputWidth=dict(\n description='Number of inputs to the TM.',\n accessMode='Read',\n dataType='UInt32',\n count=1,\n constraints=''),\n\n predictedSegmentDecrement=dict(\n description='Predicted segment decrement',\n accessMode='Read',\n dataType='Real',\n count=1,\n constraints=''),\n\n orColumnOutputs=dict(\n description=\"\"\"OR together the cell outputs from each column to produce\n the temporal memory output. When this mode is enabled, the number of\n cells per column must also be specified and the output size of the region\n should be set the same as columnCount\"\"\",\n accessMode='Read',\n dataType='Bool',\n count=1,\n constraints='bool'),\n\n cellsSavePath=dict(\n description=\"\"\"Optional path to file in which large temporal memory cells\n data structure is to be saved.\"\"\",\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints=''),\n\n temporalImp=dict(\n description=\"\"\"Which temporal memory implementation to use. Set to either\n 'py' or 'cpp'. The 'cpp' implementation is optimized for speed in C++.\"\"\",\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints='enum: py, cpp'),\n\n ))\n\n # The last group is for parameters that aren't strictly spatial or temporal\n otherSpec = dict(\n learningMode=dict(\n description='True if the node is learning (default True).',\n accessMode='ReadWrite',\n dataType='Bool',\n count=1,\n defaultValue=True,\n constraints='bool'),\n\n inferenceMode=dict(\n description='True if the node is inferring (default False).',\n accessMode='ReadWrite',\n dataType='Bool',\n count=1,\n defaultValue=False,\n constraints='bool'),\n\n computePredictedActiveCellIndices=dict(\n description='True if active and predicted active indices should be computed',\n accessMode='Create',\n dataType='Bool',\n count=1,\n defaultValue=False,\n constraints='bool'),\n\n anomalyMode=dict(\n description='True if an anomaly score is being computed',\n accessMode='Create',\n dataType='Bool',\n count=1,\n defaultValue=False,\n constraints='bool'),\n\n topDownMode=dict(\n description='True if the node should do top down compute on the next call '\n 'to compute into topDownOut (default False).',\n accessMode='ReadWrite',\n dataType='Bool',\n count=1,\n defaultValue=False,\n constraints='bool'),\n\n activeOutputCount=dict(\n description='Number of active elements in bottomUpOut output.',\n accessMode='Read',\n dataType='UInt32',\n count=1,\n constraints=''),\n\n storeDenseOutput=dict(\n description=\"\"\"Whether to keep the dense column output (needed for\n denseOutput parameter).\"\"\",\n accessMode='ReadWrite',\n dataType='UInt32',\n count=1,\n constraints='bool'),\n\n logPathOutput=dict(\n description='Optional name of output log file. If set, every output vector'\n ' will be logged to this file as a sparse vector.',\n accessMode='ReadWrite',\n dataType='Byte',\n count=0,\n constraints=''),\n\n )\n\n return temporalSpec, otherSpec"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef initialize(self):\n # Allocate appropriate temporal memory object\n # Retrieve the necessary extra arguments that were handled automatically\n autoArgs = dict((name, getattr(self, name))\n for name in self._temporalArgNames)\n\n if self._tfdr is None:\n tpClass = _getTPClass(self.temporalImp)\n\n if self.temporalImp in ['py', 'cpp', 'r',\n 'tm_py', 'tm_cpp',\n 'monitored_tm_py',]:\n self._tfdr = tpClass(\n numberOfCols=self.columnCount,\n cellsPerColumn=self.cellsPerColumn,\n **autoArgs)\n else:\n raise RuntimeError(\"Invalid temporalImp\")", "response": "Initializes the internal state of the object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompute one iteration of TMRegion s compute", "response": "def _compute(self, inputs, outputs):\n \"\"\"\n Run one iteration of TMRegion's compute\n \"\"\"\n\n #if self.topDownMode and (not 'topDownIn' in inputs):\n # raise RuntimeError(\"The input topDownIn must be linked in if \"\n # \"topDownMode is True\")\n\n if self._tfdr is None:\n raise RuntimeError(\"TM has not been initialized\")\n\n # Conditional compute break\n self._conditionalBreak()\n\n self._iterations += 1\n\n # Get our inputs as numpy array\n buInputVector = inputs['bottomUpIn']\n\n # Handle reset signal\n resetSignal = False\n if 'resetIn' in inputs:\n assert len(inputs['resetIn']) == 1\n if inputs['resetIn'][0] != 0:\n self._tfdr.reset()\n self._sequencePos = 0 # Position within the current sequence\n\n if self.computePredictedActiveCellIndices:\n prevPredictedState = self._tfdr.getPredictedState().reshape(-1).astype('float32')\n\n if self.anomalyMode:\n prevPredictedColumns = self._tfdr.topDownCompute().copy().nonzero()[0]\n\n # Perform inference and/or learning\n tpOutput = self._tfdr.compute(buInputVector, self.learningMode, self.inferenceMode)\n self._sequencePos += 1\n\n # OR'ing together the cells in each column?\n if self.orColumnOutputs:\n tpOutput= tpOutput.reshape(self.columnCount,\n self.cellsPerColumn).max(axis=1)\n\n # Direct logging of non-zero TM outputs\n if self._fpLogTPOutput:\n output = tpOutput.reshape(-1)\n outputNZ = tpOutput.nonzero()[0]\n outStr = \" \".join([\"%d\" % int(token) for token in outputNZ])\n print >>self._fpLogTPOutput, output.size, outStr\n\n # Write the bottom up out to our node outputs\n outputs['bottomUpOut'][:] = tpOutput.flat\n\n if self.topDownMode:\n # Top-down compute\n outputs['topDownOut'][:] = self._tfdr.topDownCompute().copy()\n\n # Set output for use with anomaly classification region if in anomalyMode\n if self.anomalyMode:\n activeLearnCells = self._tfdr.getLearnActiveStateT()\n size = activeLearnCells.shape[0] * activeLearnCells.shape[1]\n outputs['lrnActiveStateT'][:] = activeLearnCells.reshape(size)\n\n activeColumns = buInputVector.nonzero()[0]\n outputs['anomalyScore'][:] = anomaly.computeRawAnomalyScore(\n activeColumns, prevPredictedColumns)\n\n if self.computePredictedActiveCellIndices:\n # Reshape so we are dealing with 1D arrays\n activeState = self._tfdr._getActiveState().reshape(-1).astype('float32')\n activeIndices = numpy.where(activeState != 0)[0]\n predictedIndices= numpy.where(prevPredictedState != 0)[0]\n predictedActiveIndices = numpy.intersect1d(activeIndices, predictedIndices)\n outputs[\"activeCells\"].fill(0)\n outputs[\"activeCells\"][activeIndices] = 1\n outputs[\"predictedActiveCells\"].fill(0)\n outputs[\"predictedActiveCells\"][predictedActiveIndices] = 1"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a base Spec for the specified base class.", "response": "def getBaseSpec(cls):\n \"\"\"\n Doesn't include the spatial, temporal and other parameters\n\n :returns: (dict) the base Spec for TMRegion.\n \"\"\"\n spec = dict(\n description=TMRegion.__doc__,\n singleNodeOnly=True,\n inputs=dict(\n bottomUpIn=dict(\n description=\"\"\"The input signal, conceptually organized as an\n image pyramid data structure, but internally\n organized as a flattened vector.\"\"\",\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=False,\n isDefaultInput=True,\n requireSplitterMap=False),\n\n resetIn=dict(\n description=\"\"\"Effectively a boolean flag that indicates whether\n or not the input vector received in this compute cycle\n represents the first training presentation in a\n new temporal sequence.\"\"\",\n dataType='Real32',\n count=1,\n required=False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n sequenceIdIn=dict(\n description=\"Sequence ID\",\n dataType='UInt64',\n count=1,\n required=False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n ),\n\n outputs=dict(\n bottomUpOut=dict(\n description=\"\"\"The output signal generated from the bottom-up inputs\n from lower levels.\"\"\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=True),\n\n topDownOut=dict(\n description=\"\"\"The top-down inputsignal, generated from\n feedback from upper levels\"\"\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n activeCells=dict(\n description=\"The cells that are active\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n predictedActiveCells=dict(\n description=\"The cells that are active and predicted\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n anomalyScore = dict(\n description=\"\"\"The score for how 'anomalous' (i.e. rare) the current\n sequence is. Higher values are increasingly rare\"\"\",\n dataType='Real32',\n count=1,\n regionLevel=True,\n isDefaultOutput=False),\n\n lrnActiveStateT = dict(\n description=\"\"\"Active cells during learn phase at time t. This is\n used for anomaly classification.\"\"\",\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n ),\n\n parameters=dict(\n breakPdb=dict(\n description='Set to 1 to stop in the pdb debugger on the next compute',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='ReadWrite'),\n\n breakKomodo=dict(\n description='Set to 1 to stop in the Komodo debugger on the next compute',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='ReadWrite'),\n\n ),\n commands = {}\n )\n\n return spec"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getSpec(cls):\n spec = cls.getBaseSpec()\n t, o = _getAdditionalSpecs(temporalImp=gDefaultTemporalImp)\n spec['parameters'].update(t)\n spec['parameters'].update(o)\n\n return spec", "response": "Returns the base spec for the current region."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getParameter(self, parameterName, index=-1):\n if parameterName in self._temporalArgNames:\n return getattr(self._tfdr, parameterName)\n else:\n return PyRegion.getParameter(self, parameterName, index)", "response": "Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.getParameter`.\n\n Get the value of a parameter. Most parameters are handled automatically by\n :class:`~nupic.bindings.regions.PyRegion.PyRegion`'s parameter get mechanism. The \n ones that need special treatment are explicitly handled here."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset the value of the named parameter in the current object.", "response": "def setParameter(self, parameterName, index, parameterValue):\n \"\"\"\n Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.setParameter`.\n \"\"\"\n if parameterName in self._temporalArgNames:\n setattr(self._tfdr, parameterName, parameterValue)\n\n elif parameterName == \"logPathOutput\":\n self.logPathOutput = parameterValue\n # Close any existing log file\n if self._fpLogTPOutput is not None:\n self._fpLogTPOutput.close()\n self._fpLogTPOutput = None\n\n # Open a new log file if requested\n if parameterValue:\n self._fpLogTPOutput = open(self.logPathOutput, 'w')\n\n elif hasattr(self, parameterName):\n setattr(self, parameterName, parameterValue)\n\n else:\n raise Exception('Unknown parameter: ' + parameterName)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef finishLearning(self):\n if self._tfdr is None:\n raise RuntimeError(\"Temporal memory has not been initialized\")\n\n if hasattr(self._tfdr, 'finishLearning'):\n self.resetSequenceStates()\n self._tfdr.finishLearning()", "response": "Perform an internal optimization step that speeds up inference for the sequence of potential inputs."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwriting the state of this capnp object to the specified TMRegionProto object.", "response": "def writeToProto(self, proto):\n \"\"\"\n Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.writeToProto`.\n\n Write state to proto object.\n\n :param proto: TMRegionProto capnproto object\n \"\"\"\n proto.temporalImp = self.temporalImp\n proto.columnCount = self.columnCount\n proto.inputWidth = self.inputWidth\n proto.cellsPerColumn = self.cellsPerColumn\n proto.learningMode = self.learningMode\n proto.inferenceMode = self.inferenceMode\n proto.anomalyMode = self.anomalyMode\n proto.topDownMode = self.topDownMode\n proto.computePredictedActiveCellIndices = (\n self.computePredictedActiveCellIndices)\n proto.orColumnOutputs = self.orColumnOutputs\n\n if self.temporalImp == \"py\":\n tmProto = proto.init(\"backtrackingTM\")\n elif self.temporalImp == \"cpp\":\n tmProto = proto.init(\"backtrackingTMCpp\")\n elif self.temporalImp == \"tm_py\":\n tmProto = proto.init(\"temporalMemory\")\n elif self.temporalImp == \"tm_cpp\":\n tmProto = proto.init(\"temporalMemory\")\n else:\n raise TypeError(\n \"Unsupported temporalImp for capnp serialization: {}\".format(\n self.temporalImp))\n\n self._tfdr.write(tmProto)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading a sequence of objects from a capnp. TMRegionProto object.", "response": "def readFromProto(cls, proto):\n \"\"\"\n Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.readFromProto`.\n\n Read state from proto object.\n\n :param proto: TMRegionProto capnproto object\n \"\"\"\n instance = cls(proto.columnCount, proto.inputWidth, proto.cellsPerColumn)\n\n instance.temporalImp = proto.temporalImp\n instance.learningMode = proto.learningMode\n instance.inferenceMode = proto.inferenceMode\n instance.anomalyMode = proto.anomalyMode\n instance.topDownMode = proto.topDownMode\n instance.computePredictedActiveCellIndices = (\n proto.computePredictedActiveCellIndices)\n instance.orColumnOutputs = proto.orColumnOutputs\n\n if instance.temporalImp == \"py\":\n tmProto = proto.backtrackingTM\n elif instance.temporalImp == \"cpp\":\n tmProto = proto.backtrackingTMCpp\n elif instance.temporalImp == \"tm_py\":\n tmProto = proto.temporalMemory\n elif instance.temporalImp == \"tm_cpp\":\n tmProto = proto.temporalMemory\n else:\n raise TypeError(\n \"Unsupported temporalImp for capnp serialization: {}\".format(\n instance.temporalImp))\n\n instance._tfdr = _getTPClass(proto.temporalImp).read(tmProto)\n\n return instance"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the number of output elements in the specified name.", "response": "def getOutputElementCount(self, name):\n \"\"\"\n Overrides :meth:`~nupic.bindings.regions.PyRegion.PyRegion.getOutputElementCount`.\n \"\"\"\n if name == 'bottomUpOut':\n return self.outputWidth\n elif name == 'topDownOut':\n return self.columnCount\n elif name == 'lrnActiveStateT':\n return self.outputWidth\n elif name == \"activeCells\":\n return self.outputWidth\n elif name == \"predictedActiveCells\":\n return self.outputWidth\n else:\n raise Exception(\"Invalid output name specified\")"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompute the raw anomaly score.", "response": "def computeRawAnomalyScore(activeColumns, prevPredictedColumns):\n \"\"\"Computes the raw anomaly score.\n\n The raw anomaly score is the fraction of active columns not predicted.\n\n :param activeColumns: array of active column indices\n :param prevPredictedColumns: array of columns indices predicted in prev step\n :returns: anomaly score 0..1 (float)\n \"\"\"\n nActiveColumns = len(activeColumns)\n if nActiveColumns > 0:\n # Test whether each element of a 1-D array is also present in a second\n # array. Sum to get the total # of columns that are active and were\n # predicted.\n score = numpy.in1d(activeColumns, prevPredictedColumns).sum()\n # Get the percent of active columns that were NOT predicted, that is\n # our anomaly score.\n score = (nActiveColumns - score) / float(nActiveColumns)\n else:\n # There are no active columns.\n score = 0.0\n\n return score"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the anomaly score as the percent of active columns not predicted in this step T.", "response": "def compute(self, activeColumns, predictedColumns,\n inputValue=None, timestamp=None):\n \"\"\"Compute the anomaly score as the percent of active columns not predicted.\n\n :param activeColumns: array of active column indices\n :param predictedColumns: array of columns indices predicted in this step\n (used for anomaly in step T+1)\n :param inputValue: (optional) value of current input to encoders\n (eg \"cat\" for category encoder)\n (used in anomaly-likelihood)\n :param timestamp: (optional) date timestamp when the sample occured\n (used in anomaly-likelihood)\n :returns: the computed anomaly score; float 0..1\n \"\"\"\n # Start by computing the raw anomaly score.\n anomalyScore = computeRawAnomalyScore(activeColumns, predictedColumns)\n\n # Compute final anomaly based on selected mode.\n if self._mode == Anomaly.MODE_PURE:\n score = anomalyScore\n elif self._mode == Anomaly.MODE_LIKELIHOOD:\n if inputValue is None:\n raise ValueError(\"Selected anomaly mode 'Anomaly.MODE_LIKELIHOOD' \"\n \"requires 'inputValue' as parameter to compute() method. \")\n\n probability = self._likelihood.anomalyProbability(\n inputValue, anomalyScore, timestamp)\n # low likelihood -> hi anomaly\n score = 1 - probability\n elif self._mode == Anomaly.MODE_WEIGHTED:\n probability = self._likelihood.anomalyProbability(\n inputValue, anomalyScore, timestamp)\n score = anomalyScore * (1 - probability)\n\n # Last, do moving-average if windowSize was specified.\n if self._movingAverage is not None:\n score = self._movingAverage.next(score)\n\n # apply binary discretization if required\n if self._binaryThreshold is not None:\n if score >= self._binaryThreshold:\n score = 1.0\n else:\n score = 0.0\n\n return score"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef addGraph(self, data, position=111, xlabel=None, ylabel=None):\n ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)\n ax.plot(data)\n plt.draw()", "response": "Adds a graph to the figure."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef addHistogram(self, data, position=111, xlabel=None, ylabel=None,\n bins=None):\n \"\"\" Adds a histogram to the plot's figure.\n\n @param data See matplotlib.Axes.hist documentation.\n @param position A 3-digit number. The first two digits define a 2D grid\n where subplots may be added. The final digit specifies the nth grid\n location for the added subplot\n @param xlabel text to be displayed on the x-axis\n @param ylabel text to be displayed on the y-axis\n \"\"\"\n ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)\n ax.hist(data, bins=bins, color=\"green\", alpha=0.8)\n plt.draw()", "response": "Adds a histogram to the figure."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadding an image to the figure.", "response": "def add2DArray(self, data, position=111, xlabel=None, ylabel=None, cmap=None,\n aspect=\"auto\", interpolation=\"nearest\", name=None):\n \"\"\" Adds an image to the plot's figure.\n\n @param data a 2D array. See matplotlib.Axes.imshow documentation.\n @param position A 3-digit number. The first two digits define a 2D grid\n where subplots may be added. The final digit specifies the nth grid\n location for the added subplot\n @param xlabel text to be displayed on the x-axis\n @param ylabel text to be displayed on the y-axis\n @param cmap color map used in the rendering\n @param aspect how aspect ratio is handled during resize\n @param interpolation interpolation method\n \"\"\"\n if cmap is None:\n # The default colormodel is an ugly blue-red model.\n cmap = cm.Greys\n\n ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)\n ax.imshow(data, cmap=cmap, aspect=aspect, interpolation=interpolation)\n\n if self._show:\n plt.draw()\n\n if name is not None:\n if not os.path.exists(\"log\"):\n os.mkdir(\"log\")\n plt.savefig(\"log/{name}.png\".format(name=name), bbox_inches=\"tight\",\n figsize=(8, 6), dpi=400)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a subplot to the figure at the specified position.", "response": "def _addBase(self, position, xlabel=None, ylabel=None):\n \"\"\" Adds a subplot to the plot's figure at specified position.\n\n @param position A 3-digit number. The first two digits define a 2D grid\n where subplots may be added. The final digit specifies the nth grid\n location for the added subplot\n @param xlabel text to be displayed on the x-axis\n @param ylabel text to be displayed on the y-axis\n @returns (matplotlib.Axes) Axes instance\n \"\"\"\n ax = self._fig.add_subplot(position)\n ax.set_xlabel(xlabel)\n ax.set_ylabel(ylabel)\n return ax"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _generateOverlapping(filename=\"overlap.csv\", numSequences=2, elementsPerSeq=3, \n numRepeats=10, hub=[0,1], hubOffset=1, resets=False):\n \n \"\"\" Generate a temporal dataset containing sequences that overlap one or more\n elements with other sequences. \n \n Parameters:\n ----------------------------------------------------\n filename: name of the file to produce, including extension. It will\n be created in a 'datasets' sub-directory within the \n directory containing this script. \n numSequences: how many sequences to generate\n elementsPerSeq: length of each sequence\n numRepeats: how many times to repeat each sequence in the output \n hub: sub-sequence to place within each other sequence \n hubOffset: where, within each sequence, to place the hub\n resets: if True, turn on reset at start of each sequence\n \"\"\"\n \n # Check for conflicts in arguments\n assert (hubOffset + len(hub) <= elementsPerSeq)\n \n # Create the output file\n scriptDir = os.path.dirname(__file__)\n pathname = os.path.join(scriptDir, 'datasets', filename)\n print \"Creating %s...\" % (pathname)\n fields = [('reset', 'int', 'R'), \n ('field1', 'string', ''), \n ('field2', 'float', '')] \n outFile = FileRecordStream(pathname, write=True, fields=fields)\n \n\n # Create the sequences with the hub in the middle\n sequences = []\n nextElemIdx = max(hub)+1\n \n for _ in range(numSequences):\n seq = []\n for j in range(hubOffset):\n seq.append(nextElemIdx)\n nextElemIdx += 1\n for j in hub:\n seq.append(j)\n j = hubOffset + len(hub)\n while j < elementsPerSeq:\n seq.append(nextElemIdx)\n nextElemIdx += 1\n j += 1\n sequences.append(seq)\n \n # Write out the sequences in random order\n seqIdxs = []\n for _ in range(numRepeats):\n seqIdxs += range(numSequences)\n random.shuffle(seqIdxs)\n \n for seqIdx in seqIdxs:\n reset = int(resets)\n seq = sequences[seqIdx]\n for (x) in seq:\n outFile.appendRecord([reset, str(x), x])\n reset = 0\n\n outFile.close()", "response": "Generates a temporal dataset containing sequences that overlap one or more sequences with another."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _generateFirstOrder0():\n\n\n # --------------------------------------------------------------------\n # Initial probabilities, 'a' and 'e' equally likely\n numCategories = 5\n initProb = numpy.zeros(numCategories)\n initProb[0] = 1.0\n \n\n # --------------------------------------------------------------------\n # 1st order transitions\n firstOrder = dict()\n firstOrder['0'] = numpy.array([0, 0.1, 0, 0, 0.9])\n firstOrder['1'] = numpy.array([0, 0, 0.75, 0.25, 0])\n firstOrder['2'] = numpy.array([1.0, 0, 0, 0, 0])\n firstOrder['3'] = numpy.array([1.0, 0, 0, 0, 0])\n firstOrder['4'] = numpy.array([0, 0, 0.5, 0.5, 0])\n \n # --------------------------------------------------------------------\n # 2nd order transitions don't apply\n secondOrder = None\n \n # Generate the category list\n categoryList = ['%d' % x for x in range(5)]\n return (initProb, firstOrder, secondOrder, 3, categoryList)", "response": "Generate the initial first order transition sequence and second order transition sequence."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates a set of records reflecting a set of probabilities.", "response": "def _generateFileFromProb(filename, numRecords, categoryList, initProb, \n firstOrderProb, secondOrderProb, seqLen, numNoise=0, resetsEvery=None):\n \"\"\" Generate a set of records reflecting a set of probabilities.\n \n Parameters:\n ----------------------------------------------------------------\n filename: name of .csv file to generate\n numRecords: number of records to generate\n categoryList: list of category names\n initProb: Initial probability for each category. This is a vector\n of length len(categoryList).\n firstOrderProb: A dictionary of the 1st order probabilities. The key\n is the 1st element of the sequence, the value is\n the probability of each 2nd element given the first. \n secondOrderProb: A dictionary of the 2nd order probabilities. The key\n is the first 2 elements of the sequence, the value is\n the probability of each possible 3rd element given the \n first two. If this is None, then the sequences will be\n first order only. \n seqLen: Desired length of each sequence. The 1st element will\n be generated using the initProb, the 2nd element by the\n firstOrder table, and the 3rd and all successive \n elements by the secondOrder table. None means infinite\n length. \n numNoise: Number of noise elements to place between each \n sequence. The noise elements are evenly distributed from \n all categories. \n resetsEvery: If not None, generate a reset every N records\n \n \n Here is an example of some parameters:\n \n categoryList: ['cat1', 'cat2', 'cat3']\n \n initProb: [0.7, 0.2, 0.1]\n \n firstOrderProb: {'[0]': [0.3, 0.3, 0.4],\n '[1]': [0.3, 0.3, 0.4],\n '[2]': [0.3, 0.3, 0.4]}\n \n secondOrderProb: {'[0,0]': [0.3, 0.3, 0.4],\n '[0,1]': [0.3, 0.3, 0.4],\n '[0,2]': [0.3, 0.3, 0.4],\n '[1,0]': [0.3, 0.3, 0.4],\n '[1,1]': [0.3, 0.3, 0.4],\n '[1,2]': [0.3, 0.3, 0.4],\n '[2,0]': [0.3, 0.3, 0.4],\n '[2,1]': [0.3, 0.3, 0.4],\n '[2,2]': [0.3, 0.3, 0.4]}\n \n \"\"\"\n \n # Create the file\n print \"Creating %s...\" % (filename)\n fields = [('reset', 'int', 'R'), \n ('field1', 'string', ''),\n ('field2', 'float', '')]\n\n scriptDir = os.path.dirname(__file__)\n pathname = os.path.join(scriptDir, 'datasets', filename)\n outFile = FileRecordStream(pathname, write=True, fields=fields)\n \n # --------------------------------------------------------------------\n # Convert the probabilitie tables into cumulative probabilities\n initCumProb = initProb.cumsum()\n \n firstOrderCumProb = dict()\n for (key,value) in firstOrderProb.iteritems():\n firstOrderCumProb[key] = value.cumsum()\n \n if secondOrderProb is not None:\n secondOrderCumProb = dict()\n for (key,value) in secondOrderProb.iteritems():\n secondOrderCumProb[key] = value.cumsum()\n else:\n secondOrderCumProb = None\n \n\n # --------------------------------------------------------------------\n # Write out the sequences\n elementsInSeq = []\n numElementsSinceReset = 0\n maxCatIdx = len(categoryList) - 1\n for _ in xrange(numRecords):\n\n # Generate a reset?\n if numElementsSinceReset == 0:\n reset = 1\n else:\n reset = 0\n \n # Pick the next element, based on how are we are into the 2nd order\n # sequence. \n rand = numpy.random.rand()\n \n # Generate 1st order sequences\n if secondOrderCumProb is None:\n if len(elementsInSeq) == 0:\n catIdx = numpy.searchsorted(initCumProb, rand)\n elif len(elementsInSeq) >= 1 and \\\n (seqLen is None or len(elementsInSeq) < seqLen-numNoise):\n catIdx = numpy.searchsorted(firstOrderCumProb[str(elementsInSeq[-1])], \n rand)\n else: # random \"noise\"\n catIdx = numpy.random.randint(len(categoryList))\n\n # Generate 2nd order sequences\n else:\n if len(elementsInSeq) == 0:\n catIdx = numpy.searchsorted(initCumProb, rand)\n elif len(elementsInSeq) == 1:\n catIdx = numpy.searchsorted(firstOrderCumProb[str(elementsInSeq)], rand)\n elif (len(elementsInSeq) >=2) and \\\n (seqLen is None or len(elementsInSeq) < seqLen-numNoise):\n catIdx = numpy.searchsorted(secondOrderCumProb[str(elementsInSeq[-2:])], rand)\n else: # random \"noise\"\n catIdx = numpy.random.randint(len(categoryList))\n \n # -------------------------------------------------------------------\n # Write out the record\n catIdx = min(maxCatIdx, catIdx)\n outFile.appendRecord([reset, categoryList[catIdx], catIdx]) \n #print categoryList[catIdx]\n \n # ------------------------------------------------------------\n # Increment counters\n elementsInSeq.append(catIdx)\n numElementsSinceReset += 1\n \n # Generate another reset?\n if resetsEvery is not None and numElementsSinceReset == resetsEvery:\n numElementsSinceReset = 0\n elementsInSeq = []\n \n # Start another 2nd order sequence?\n if seqLen is not None and (len(elementsInSeq) == seqLen+numNoise):\n elementsInSeq = []\n \n \n outFile.close()"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getVersion():\n with open(os.path.join(REPO_DIR, \"VERSION\"), \"r\") as versionFile:\n return versionFile.read().strip()", "response": "Get version from local file."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef nupicBindingsPrereleaseInstalled():\n try:\n nupicDistribution = pkg_resources.get_distribution(\"nupic.bindings\")\n if pkg_resources.parse_version(nupicDistribution.version).is_prerelease:\n # A pre-release dev version of nupic.bindings is installed.\n return True\n except pkg_resources.DistributionNotFound:\n pass # Silently ignore. The absence of nupic.bindings will be handled by\n # setuptools by default\n\n return False", "response": "Return True if a pre - release version of nupic. bindings is installed already."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading the requirements. txt file and parse into requirements for setup s install_requirements option.", "response": "def findRequirements():\n \"\"\"\n Read the requirements.txt file and parse into requirements for setup's\n install_requirements option.\n \"\"\"\n requirementsPath = os.path.join(REPO_DIR, \"requirements.txt\")\n requirements = parse_file(requirementsPath)\n\n if nupicBindingsPrereleaseInstalled():\n # User has a pre-release version of nupic.bindings installed, which is only\n # possible if the user installed and built nupic.bindings from source and\n # it is up to the user to decide when to update nupic.bindings. We'll\n # quietly remove the entry in requirements.txt so as to not conflate the\n # two.\n requirements = [req for req in requirements if \"nupic.bindings\" not in req]\n\n return requirements"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _handleDescriptionOption(cmdArgStr, outDir, usageStr, hsVersion,\n claDescriptionTemplateFile):\n \"\"\"\n Parses and validates the --description option args and executes the\n request\n\n Parameters:\n -----------------------------------------------------------------------\n cmdArgStr: JSON string compatible with _gExperimentDescriptionSchema\n outDir: where to place generated experiment files\n usageStr: program usage string\n hsVersion: which version of hypersearch permutations file to generate, can\n be 'v1' or 'v2'\n claDescriptionTemplateFile: Filename containing the template description\n retval: nothing\n\n\n \"\"\"\n # convert --description arg from JSON string to dict\n try:\n args = json.loads(cmdArgStr)\n except Exception, e:\n raise _InvalidCommandArgException(\n _makeUsageErrorStr(\n (\"JSON arg parsing failed for --description: %s\\n\" + \\\n \"ARG=<%s>\") % (str(e), cmdArgStr), usageStr))\n\n #print \"PARSED JSON ARGS=\\n%s\" % (json.dumps(args, indent=4))\n\n filesDescription = _generateExperiment(args, outDir, hsVersion=hsVersion,\n claDescriptionTemplateFile = claDescriptionTemplateFile)\n\n pprint.pprint(filesDescription)\n\n return", "response": "Parses and executes the command - line option args and executes the experiment"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _handleDescriptionFromFileOption(filename, outDir, usageStr, hsVersion,\n claDescriptionTemplateFile):\n \"\"\"\n Parses and validates the --descriptionFromFile option and executes the\n request\n\n Parameters:\n -----------------------------------------------------------------------\n filename: File from which we'll extract description JSON\n outDir: where to place generated experiment files\n usageStr: program usage string\n hsVersion: which version of hypersearch permutations file to generate, can\n be 'v1' or 'v2'\n claDescriptionTemplateFile: Filename containing the template description\n retval: nothing\n \"\"\"\n\n try:\n fileHandle = open(filename, 'r')\n JSONStringFromFile = fileHandle.read().splitlines()\n JSONStringFromFile = ''.join(JSONStringFromFile)\n\n except Exception, e:\n raise _InvalidCommandArgException(\n _makeUsageErrorStr(\n (\"File open failed for --descriptionFromFile: %s\\n\" + \\\n \"ARG=<%s>\") % (str(e), filename), usageStr))\n\n _handleDescriptionOption(JSONStringFromFile, outDir, usageStr,\n hsVersion=hsVersion,\n claDescriptionTemplateFile = claDescriptionTemplateFile)\n return", "response": "Parses and validates the description from a file and executes the command."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn True if x is a valid integer value.", "response": "def _isInt(x, precision = 0.0001):\n \"\"\"\n Return (isInt, intValue) for a given floating point number.\n\n Parameters:\n ----------------------------------------------------------------------\n x: floating point number to evaluate\n precision: desired precision\n retval: (isInt, intValue)\n isInt: True if x is close enough to an integer value\n intValue: x as an integer\n \"\"\"\n\n xInt = int(round(x))\n return (abs(x - xInt) < precision * x, xInt)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _indentLines(str, indentLevels = 1, indentFirstLine=True):\n\n indent = _ONE_INDENT * indentLevels\n\n lines = str.splitlines(True)\n result = ''\n\n if len(lines) > 0 and not indentFirstLine:\n first = 1\n result += lines[0]\n else:\n first = 0\n\n for line in lines[first:]:\n result += indent + line\n\n return result", "response": "Indent all lines in the given string with the given number of levels of indentation."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _generateMetricSpecString(inferenceElement, metric,\n params=None, field=None,\n returnLabel=False):\n \"\"\" Generates the string representation of a MetricSpec object, and returns\n the metric key associated with the metric.\n\n\n Parameters:\n -----------------------------------------------------------------------\n inferenceElement:\n An InferenceElement value that indicates which part of the inference this\n metric is computed on\n\n metric:\n The type of the metric being computed (e.g. aae, avg_error)\n\n params:\n A dictionary of parameters for the metric. The keys are the parameter names\n and the values should be the parameter values (e.g. window=200)\n\n field:\n The name of the field for which this metric is being computed\n\n returnLabel:\n If True, returns the label of the MetricSpec that was generated\n \"\"\"\n\n metricSpecArgs = dict(metric=metric,\n field=field,\n params=params,\n inferenceElement=inferenceElement)\n\n metricSpecAsString = \"MetricSpec(%s)\" % \\\n ', '.join(['%s=%r' % (item[0],item[1])\n for item in metricSpecArgs.iteritems()])\n\n if not returnLabel:\n return metricSpecAsString\n\n spec = MetricSpec(**metricSpecArgs)\n metricLabel = spec.getLabel()\n return metricSpecAsString, metricLabel", "response": "Generates a string representation of a MetricSpec object."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngenerate a file from a list of template file names and a dictionary of replacement pairs.", "response": "def _generateFileFromTemplates(templateFileNames, outputFilePath,\n replacementDict):\n \"\"\" Generates a file by applying token replacements to the given template\n file\n\n templateFileName:\n A list of template file names; these files are assumed to be in\n the same directory as the running experiment_generator.py script.\n ExpGenerator will perform the substitution and concanetate\n the files in the order they are specified\n\n outputFilePath: Absolute path of the output file\n\n replacementDict:\n A dictionary of token/replacement pairs\n \"\"\"\n\n # Find out where we're running from so we know where to find templates\n installPath = os.path.dirname(__file__)\n outputFile = open(outputFilePath, \"w\")\n outputLines = []\n inputLines = []\n\n firstFile = True\n for templateFileName in templateFileNames:\n # Separate lines from each file by two blank lines.\n if not firstFile:\n inputLines.extend([os.linesep]*2)\n firstFile = False\n\n inputFilePath = os.path.join(installPath, templateFileName)\n inputFile = open(inputFilePath)\n inputLines.extend(inputFile.readlines())\n inputFile.close()\n\n\n print \"Writing \", len(inputLines), \"lines...\"\n\n for line in inputLines:\n tempLine = line\n\n # Enumerate through each key in replacementDict and replace with value\n for k, v in replacementDict.iteritems():\n if v is None:\n v = \"None\"\n tempLine = re.sub(k, v, tempLine)\n outputFile.write(tempLine)\n outputFile.close()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating encoder choices for the given field.", "response": "def _generateEncoderChoicesV1(fieldInfo):\n \"\"\" Return a list of possible encoder parameter combinations for the given\n field and the default aggregation function to use. Each parameter combination\n is a dict defining the parameters for the encoder. Here is an example\n return value for the encoderChoicesList:\n\n [\n None,\n {'fieldname':'timestamp',\n 'name': 'timestamp_timeOfDay',\n 'type':'DateEncoder'\n 'dayOfWeek': (7,1)\n },\n {'fieldname':'timestamp',\n 'name': 'timestamp_timeOfDay',\n 'type':'DateEncoder'\n 'dayOfWeek': (7,3)\n },\n ],\n\n Parameters:\n --------------------------------------------------\n fieldInfo: item from the 'includedFields' section of the\n description JSON object\n\n retval: (encoderChoicesList, aggFunction)\n encoderChoicesList: a list of encoder choice lists for this field.\n Most fields will generate just 1 encoder choice list.\n DateTime fields can generate 2 or more encoder choice lists,\n one for dayOfWeek, one for timeOfDay, etc.\n aggFunction: name of aggregation function to use for this\n field type\n\n \"\"\"\n\n width = 7\n fieldName = fieldInfo['fieldName']\n fieldType = fieldInfo['fieldType']\n encoderChoicesList = []\n\n # Scalar?\n if fieldType in ['float', 'int']:\n aggFunction = 'mean'\n encoders = [None]\n for n in (13, 50, 150, 500):\n encoder = dict(type='ScalarSpaceEncoder', name=fieldName, fieldname=fieldName,\n n=n, w=width, clipInput=True,space=\"absolute\")\n if 'minValue' in fieldInfo:\n encoder['minval'] = fieldInfo['minValue']\n if 'maxValue' in fieldInfo:\n encoder['maxval'] = fieldInfo['maxValue']\n encoders.append(encoder)\n encoderChoicesList.append(encoders)\n\n # String?\n elif fieldType == 'string':\n aggFunction = 'first'\n encoders = [None]\n encoder = dict(type='SDRCategoryEncoder', name=fieldName,\n fieldname=fieldName, n=100, w=width)\n encoders.append(encoder)\n encoderChoicesList.append(encoders)\n\n\n # Datetime?\n elif fieldType == 'datetime':\n aggFunction = 'first'\n\n # First, the time of day representation\n encoders = [None]\n for radius in (1, 8):\n encoder = dict(type='DateEncoder', name='%s_timeOfDay' % (fieldName),\n fieldname=fieldName, timeOfDay=(width, radius))\n encoders.append(encoder)\n encoderChoicesList.append(encoders)\n\n # Now, the day of week representation\n encoders = [None]\n for radius in (1, 3):\n encoder = dict(type='DateEncoder', name='%s_dayOfWeek' % (fieldName),\n fieldname=fieldName, dayOfWeek=(width, radius))\n encoders.append(encoder)\n encoderChoicesList.append(encoders)\n\n else:\n raise RuntimeError(\"Unsupported field type '%s'\" % (fieldType))\n\n\n # Return results\n return (encoderChoicesList, aggFunction)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _generateEncoderStringsV1(includedFields):\n\n # ------------------------------------------------------------------------\n # First accumulate the possible choices for each encoder\n encoderChoicesList = []\n for fieldInfo in includedFields:\n\n fieldName = fieldInfo['fieldName']\n\n # Get the list of encoder choices for this field\n (choicesList, aggFunction) = _generateEncoderChoicesV1(fieldInfo)\n encoderChoicesList.extend(choicesList)\n\n\n # ------------------------------------------------------------------------\n # Generate the string containing the encoder specs and encoder schema. See\n # the function comments for an example of the encoderSpecsStr and\n # encoderSchemaStr\n #\n encoderSpecsList = []\n for encoderChoices in encoderChoicesList:\n # Use the last choice as the default in the base file because the 1st is\n # often None\n encoder = encoderChoices[-1]\n\n # Check for bad characters\n for c in _ILLEGAL_FIELDNAME_CHARACTERS:\n if encoder['name'].find(c) >= 0:\n raise _ExpGeneratorException(\"Illegal character in field: %r (%r)\" % (\n c, encoder['name']))\n\n encoderSpecsList.append(\"%s: \\n%s%s\" % (\n _quoteAndEscape(encoder['name']),\n 2*_ONE_INDENT,\n pprint.pformat(encoder, indent=2*_INDENT_STEP)))\n\n encoderSpecsStr = ',\\n '.join(encoderSpecsList)\n\n\n # ------------------------------------------------------------------------\n # Generate the string containing the permutation encoder choices. See the\n # function comments above for an example of the permEncoderChoicesStr\n\n permEncoderChoicesList = []\n for encoderChoices in encoderChoicesList:\n permEncoderChoicesList.append(\"%s: %s,\" % (\n _quoteAndEscape(encoderChoices[-1]['name']),\n pprint.pformat(encoderChoices, indent=2*_INDENT_STEP)))\n permEncoderChoicesStr = '\\n'.join(permEncoderChoicesList)\n permEncoderChoicesStr = _indentLines(permEncoderChoicesStr, 1,\n indentFirstLine=False)\n\n # Return results\n return (encoderSpecsStr, permEncoderChoicesStr)", "response": "Generates a string that contains the encoder related substitution variables for the base description file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating the string that defines the permutations to apply for a given encoder.", "response": "def _generatePermEncoderStr(options, encoderDict):\n \"\"\" Generate the string that defines the permutations to apply for a given\n encoder.\n\n Parameters:\n -----------------------------------------------------------------------\n options: experiment params\n encoderDict: the encoder dict, which gets placed into the description.py\n\n\n For example, if the encoderDict contains:\n 'consumption': {\n 'clipInput': True,\n 'fieldname': u'consumption',\n 'n': 100,\n 'name': u'consumption',\n 'type': 'AdaptiveScalarEncoder',\n 'w': 21},\n\n The return string will contain:\n \"PermuteEncoder(fieldName='consumption',\n encoderClass='AdaptiveScalarEncoder',\n w=21,\n n=PermuteInt(28, 521),\n clipInput=True)\"\n\n \"\"\"\n\n permStr = \"\"\n\n\n # If it's the encoder for the classifier input, then it's always present so\n # put it in as a dict in the permutations.py file instead of a\n # PermuteEncoder().\n if encoderDict.get('classifierOnly', False):\n permStr = \"dict(\"\n for key, value in encoderDict.items():\n if key == \"name\":\n continue\n\n if key == 'n' and encoderDict['type'] != 'SDRCategoryEncoder':\n permStr += \"n=PermuteInt(%d, %d), \" % (encoderDict[\"w\"] + 7,\n encoderDict[\"w\"] + 500)\n else:\n if issubclass(type(value), basestring):\n permStr += \"%s='%s', \" % (key, value)\n else:\n permStr += \"%s=%s, \" % (key, value)\n permStr += \")\"\n\n\n else:\n # Scalar encoders\n if encoderDict[\"type\"] in [\"ScalarSpaceEncoder\", \"AdaptiveScalarEncoder\",\n \"ScalarEncoder\", \"LogEncoder\"]:\n permStr = \"PermuteEncoder(\"\n for key, value in encoderDict.items():\n if key == \"fieldname\":\n key = \"fieldName\"\n elif key == \"type\":\n key = \"encoderClass\"\n elif key == \"name\":\n continue\n\n if key == \"n\":\n permStr += \"n=PermuteInt(%d, %d), \" % (encoderDict[\"w\"] + 1,\n encoderDict[\"w\"] + 500)\n elif key == \"runDelta\":\n if value and not \"space\" in encoderDict:\n permStr += \"space=PermuteChoices([%s,%s]), \" \\\n % (_quoteAndEscape(\"delta\"), _quoteAndEscape(\"absolute\"))\n encoderDict.pop(\"runDelta\")\n\n else:\n if issubclass(type(value), basestring):\n permStr += \"%s='%s', \" % (key, value)\n else:\n permStr += \"%s=%s, \" % (key, value)\n permStr += \")\"\n\n # Category encoder\n elif encoderDict[\"type\"] in [\"SDRCategoryEncoder\"]:\n permStr = \"PermuteEncoder(\"\n for key, value in encoderDict.items():\n if key == \"fieldname\":\n key = \"fieldName\"\n elif key == \"type\":\n key = \"encoderClass\"\n elif key == \"name\":\n continue\n\n if issubclass(type(value), basestring):\n permStr += \"%s='%s', \" % (key, value)\n else:\n permStr += \"%s=%s, \" % (key, value)\n permStr += \")\"\n\n\n # Datetime encoder\n elif encoderDict[\"type\"] in [\"DateEncoder\"]:\n permStr = \"PermuteEncoder(\"\n for key, value in encoderDict.items():\n if key == \"fieldname\":\n key = \"fieldName\"\n elif key == \"type\":\n continue\n elif key == \"name\":\n continue\n\n if key == \"timeOfDay\":\n permStr += \"encoderClass='%s.timeOfDay', \" % (encoderDict[\"type\"])\n permStr += \"radius=PermuteFloat(0.5, 12), \"\n permStr += \"w=%d, \" % (value[0])\n elif key == \"dayOfWeek\":\n permStr += \"encoderClass='%s.dayOfWeek', \" % (encoderDict[\"type\"])\n permStr += \"radius=PermuteFloat(1, 6), \"\n permStr += \"w=%d, \" % (value[0])\n elif key == \"weekend\":\n permStr += \"encoderClass='%s.weekend', \" % (encoderDict[\"type\"])\n permStr += \"radius=PermuteChoices([1]), \"\n permStr += \"w=%d, \" % (value)\n else:\n if issubclass(type(value), basestring):\n permStr += \"%s='%s', \" % (key, value)\n else:\n permStr += \"%s=%s, \" % (key, value)\n permStr += \")\"\n\n else:\n raise RuntimeError(\"Unsupported encoder type '%s'\" % \\\n (encoderDict[\"type\"]))\n\n return permStr"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngenerate and returns the encoder string for the encoder in the version 2 of the file.", "response": "def _generateEncoderStringsV2(includedFields, options):\n \"\"\" Generate and return the following encoder related substitution variables:\n\n encoderSpecsStr:\n For the base description file, this string defines the default\n encoding dicts for each encoder. For example:\n\n __gym_encoder = { 'fieldname': 'gym',\n 'n': 13,\n 'name': 'gym',\n 'type': 'SDRCategoryEncoder',\n 'w': 7},\n __address_encoder = { 'fieldname': 'address',\n 'n': 13,\n 'name': 'address',\n 'type': 'SDRCategoryEncoder',\n 'w': 7}\n\n\n permEncoderChoicesStr:\n For the permutations file, this defines the possible\n encoder dicts for each encoder. For example:\n\n '__gym_encoder' : PermuteEncoder('gym', 'SDRCategoryEncoder', w=7,\n n=100),\n\n '__address_encoder' : PermuteEncoder('address', 'SDRCategoryEncoder',\n w=7, n=100),\n\n '__timestamp_dayOfWeek_encoder' : PermuteEncoder('timestamp',\n 'DateEncoder.timeOfDay', w=7, radius=PermuteChoices([1, 8])),\n\n '__consumption_encoder': PermuteEncoder('consumption', 'AdaptiveScalarEncoder',\n w=7, n=PermuteInt(13, 500, 20), minval=0,\n maxval=PermuteInt(100, 300, 25)),\n\n\n\n Parameters:\n --------------------------------------------------\n includedFields: item from the 'includedFields' section of the\n description JSON object. This is a list of dicts, each\n dict defining the field name, type, and optional min\n and max values.\n\n retval: (encoderSpecsStr permEncoderChoicesStr)\n\n\n \"\"\"\n\n width = 21\n encoderDictsList = []\n\n\n # If this is a NontemporalClassification experiment, then the\n # the \"predicted\" field (the classification value) should be marked to ONLY\n # go to the classifier\n if options['inferenceType'] in [\"NontemporalClassification\",\n \"NontemporalMultiStep\",\n \"TemporalMultiStep\",\n \"MultiStep\"]:\n classifierOnlyField = options['inferenceArgs']['predictedField']\n else:\n classifierOnlyField = None\n\n\n # ==========================================================================\n # For each field, generate the default encoding dict and PermuteEncoder\n # constructor arguments\n for fieldInfo in includedFields:\n\n fieldName = fieldInfo['fieldName']\n fieldType = fieldInfo['fieldType']\n\n # ---------\n # Scalar?\n if fieldType in ['float', 'int']:\n # n=100 is reasonably hardcoded value for n when used by description.py\n # The swarming will use PermuteEncoder below, where n is variable and\n # depends on w\n runDelta = fieldInfo.get(\"runDelta\", False)\n if runDelta or \"space\" in fieldInfo:\n encoderDict = dict(type='ScalarSpaceEncoder', name=fieldName,\n fieldname=fieldName, n=100, w=width, clipInput=True)\n if runDelta:\n encoderDict[\"runDelta\"] = True\n else:\n encoderDict = dict(type='AdaptiveScalarEncoder', name=fieldName,\n fieldname=fieldName, n=100, w=width, clipInput=True)\n\n if 'minValue' in fieldInfo:\n encoderDict['minval'] = fieldInfo['minValue']\n if 'maxValue' in fieldInfo:\n encoderDict['maxval'] = fieldInfo['maxValue']\n\n # If both min and max were specified, use a non-adaptive encoder\n if ('minValue' in fieldInfo and 'maxValue' in fieldInfo) \\\n and (encoderDict['type'] == 'AdaptiveScalarEncoder'):\n encoderDict['type'] = 'ScalarEncoder'\n\n # Defaults may have been over-ridden by specifying an encoder type\n if 'encoderType' in fieldInfo:\n encoderDict['type'] = fieldInfo['encoderType']\n\n if 'space' in fieldInfo:\n encoderDict['space'] = fieldInfo['space']\n encoderDictsList.append(encoderDict)\n\n\n\n # ---------\n # String?\n elif fieldType == 'string':\n encoderDict = dict(type='SDRCategoryEncoder', name=fieldName,\n fieldname=fieldName, n=100+width, w=width)\n if 'encoderType' in fieldInfo:\n encoderDict['type'] = fieldInfo['encoderType']\n\n encoderDictsList.append(encoderDict)\n\n\n\n # ---------\n # Datetime?\n elif fieldType == 'datetime':\n\n # First, the time of day representation\n encoderDict = dict(type='DateEncoder', name='%s_timeOfDay' % (fieldName),\n fieldname=fieldName, timeOfDay=(width, 1))\n if 'encoderType' in fieldInfo:\n encoderDict['type'] = fieldInfo['encoderType']\n encoderDictsList.append(encoderDict)\n\n\n # Now, the day of week representation\n encoderDict = dict(type='DateEncoder', name='%s_dayOfWeek' % (fieldName),\n fieldname=fieldName, dayOfWeek=(width, 1))\n if 'encoderType' in fieldInfo:\n encoderDict['type'] = fieldInfo['encoderType']\n encoderDictsList.append(encoderDict)\n\n\n # Now, the day of week representation\n encoderDict = dict(type='DateEncoder', name='%s_weekend' % (fieldName),\n fieldname=fieldName, weekend=(width))\n if 'encoderType' in fieldInfo:\n encoderDict['type'] = fieldInfo['encoderType']\n encoderDictsList.append(encoderDict)\n\n\n\n\n else:\n raise RuntimeError(\"Unsupported field type '%s'\" % (fieldType))\n\n\n # -----------------------------------------------------------------------\n # If this was the predicted field, insert another encoder that sends it\n # to the classifier only\n if fieldName == classifierOnlyField:\n clEncoderDict = dict(encoderDict)\n clEncoderDict['classifierOnly'] = True\n clEncoderDict['name'] = '_classifierInput'\n encoderDictsList.append(clEncoderDict)\n\n # If the predicted field needs to be excluded, take it out of the encoder\n # lists\n if options[\"inferenceArgs\"][\"inputPredictedField\"] == \"no\":\n encoderDictsList.remove(encoderDict)\n\n # Remove any encoders not in fixedFields\n if options.get('fixedFields') is not None:\n tempList=[]\n for encoderDict in encoderDictsList:\n if encoderDict['name'] in options['fixedFields']:\n tempList.append(encoderDict)\n encoderDictsList = tempList\n\n # ==========================================================================\n # Now generate the encoderSpecsStr and permEncoderChoicesStr strings from\n # encoderDictsList and constructorStringList\n\n encoderSpecsList = []\n permEncoderChoicesList = []\n for encoderDict in encoderDictsList:\n\n if encoderDict['name'].find('\\\\') >= 0:\n raise _ExpGeneratorException(\"Illegal character in field: '\\\\'\")\n\n # Check for bad characters\n for c in _ILLEGAL_FIELDNAME_CHARACTERS:\n if encoderDict['name'].find(c) >= 0:\n raise _ExpGeneratorException(\"Illegal character %s in field %r\" %(c, encoderDict['name']))\n\n constructorStr = _generatePermEncoderStr(options, encoderDict)\n\n encoderKey = _quoteAndEscape(encoderDict['name'])\n encoderSpecsList.append(\"%s: %s%s\" % (\n encoderKey,\n 2*_ONE_INDENT,\n pprint.pformat(encoderDict, indent=2*_INDENT_STEP)))\n\n\n # Each permEncoderChoicesStr is of the form:\n # PermuteEncoder('gym', 'SDRCategoryEncoder',\n # w=7, n=100),\n permEncoderChoicesList.append(\"%s: %s,\" % (encoderKey, constructorStr))\n\n\n # Join into strings\n encoderSpecsStr = ',\\n '.join(encoderSpecsList)\n\n permEncoderChoicesStr = '\\n'.join(permEncoderChoicesList)\n permEncoderChoicesStr = _indentLines(permEncoderChoicesStr, 1,\n indentFirstLine=True)\n\n # Return results\n return (encoderSpecsStr, permEncoderChoicesStr)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nhandles legacy options for JAVA parameters.", "response": "def _handleJAVAParameters(options):\n \"\"\" Handle legacy options (TEMPORARY) \"\"\"\n\n # Find the correct InferenceType for the Model\n if 'inferenceType' not in options:\n prediction = options.get('prediction', {InferenceType.TemporalNextStep:\n {'optimize':True}})\n inferenceType = None\n for infType, value in prediction.iteritems():\n if value['optimize']:\n inferenceType = infType\n break\n\n if inferenceType == 'temporal':\n inferenceType = InferenceType.TemporalNextStep\n if inferenceType != InferenceType.TemporalNextStep:\n raise _ExpGeneratorException(\"Unsupported inference type %s\" % \\\n (inferenceType))\n options['inferenceType'] = inferenceType\n\n # Find the best value for the predicted field\n if 'predictionField' in options:\n if 'inferenceArgs' not in options:\n options['inferenceArgs'] = {'predictedField': options['predictionField']}\n elif 'predictedField' not in options['inferenceArgs']:\n options['inferenceArgs']['predictedField'] = options['predictionField']"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _getPropertyValue(schema, propertyName, options):\n\n if propertyName not in options:\n paramsSchema = schema['properties'][propertyName]\n if 'default' in paramsSchema:\n options[propertyName] = paramsSchema['default']\n else:\n options[propertyName] = None", "response": "Checks to see if property is specified in options."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _getExperimentDescriptionSchema():\n installPath = os.path.dirname(os.path.abspath(__file__))\n schemaFilePath = os.path.join(installPath, \"experimentDescriptionSchema.json\")\n return json.loads(open(schemaFilePath, 'r').read())", "response": "Returns the experiment description schema."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _generateExperiment(options, outputDirPath, hsVersion,\n claDescriptionTemplateFile):\n \"\"\" Executes the --description option, which includes:\n\n 1. Perform provider compatibility checks\n 2. Preprocess the training and testing datasets (filter, join providers)\n 3. If test dataset omitted, split the training dataset into training\n and testing datasets.\n 4. Gather statistics about the training and testing datasets.\n 5. Generate experiment scripts (description.py, permutaions.py)\n\n Parameters:\n --------------------------------------------------------------------------\n options: dictionary that matches the schema defined by the return value of\n _getExperimentDescriptionSchema(); NOTE: this arg may be modified\n by this function.\n\n outputDirPath: where to place generated files\n\n hsVersion: which version of hypersearch permutations file to generate, can\n be 'v1' or 'v2'\n claDescriptionTemplateFile: Filename containing the template description\n\n\n Returns: on success, returns a dictionary per _experimentResultsJSONSchema;\n raises exception on error\n\n Assumption1: input train and test files have identical field metadata\n \"\"\"\n\n _gExperimentDescriptionSchema = _getExperimentDescriptionSchema()\n\n # Validate JSON arg using JSON schema validator\n try:\n validictory.validate(options, _gExperimentDescriptionSchema)\n except Exception, e:\n raise _InvalidCommandArgException(\n (\"JSON arg validation failed for option --description: \" + \\\n \"%s\\nOPTION ARG=%s\") % (str(e), pprint.pformat(options)))\n\n # Validate the streamDef\n streamSchema = json.load(resource_stream(jsonschema.__name__,\n 'stream_def.json'))\n try:\n validictory.validate(options['streamDef'], streamSchema)\n except Exception, e:\n raise _InvalidCommandArgException(\n (\"JSON arg validation failed for streamDef \" + \\\n \"%s\\nOPTION ARG=%s\") % (str(e), json.dumps(options)))\n\n # -----------------------------------------------------------------------\n # Handle legacy parameters from JAVA API server\n # TODO: remove this!\n _handleJAVAParameters(options)\n\n # -----------------------------------------------------------------------\n # Get default values\n for propertyName in _gExperimentDescriptionSchema['properties']:\n _getPropertyValue(_gExperimentDescriptionSchema, propertyName, options)\n\n\n if options['inferenceArgs'] is not None:\n infArgs = _gExperimentDescriptionSchema['properties']['inferenceArgs']\n for schema in infArgs['type']:\n if isinstance(schema, dict):\n for propertyName in schema['properties']:\n _getPropertyValue(schema, propertyName, options['inferenceArgs'])\n\n if options['anomalyParams'] is not None:\n anomalyArgs = _gExperimentDescriptionSchema['properties']['anomalyParams']\n for schema in anomalyArgs['type']:\n if isinstance(schema, dict):\n for propertyName in schema['properties']:\n _getPropertyValue(schema, propertyName, options['anomalyParams'])\n\n\n # If the user specified nonTemporalClassification, make sure prediction\n # steps is 0\n predictionSteps = options['inferenceArgs'].get('predictionSteps', None)\n if options['inferenceType'] == InferenceType.NontemporalClassification:\n if predictionSteps is not None and predictionSteps != [0]:\n raise RuntimeError(\"When NontemporalClassification is used, prediction\"\n \" steps must be [0]\")\n\n # -------------------------------------------------------------------------\n # If the user asked for 0 steps of prediction, then make this a spatial\n # classification experiment\n if predictionSteps == [0] \\\n and options['inferenceType'] in ['NontemporalMultiStep',\n 'TemporalMultiStep',\n 'MultiStep']:\n options['inferenceType'] = InferenceType.NontemporalClassification\n\n\n # If NontemporalClassification was chosen as the inferenceType, then the\n # predicted field can NOT be used as an input\n if options[\"inferenceType\"] == InferenceType.NontemporalClassification:\n if options[\"inferenceArgs\"][\"inputPredictedField\"] == \"yes\" \\\n or options[\"inferenceArgs\"][\"inputPredictedField\"] == \"auto\":\n raise RuntimeError(\"When the inference type is NontemporalClassification\"\n \" inputPredictedField must be set to 'no'\")\n options[\"inferenceArgs\"][\"inputPredictedField\"] = \"no\"\n\n\n # -----------------------------------------------------------------------\n # Process the swarmSize setting, if provided\n swarmSize = options['swarmSize']\n\n if swarmSize is None:\n if options[\"inferenceArgs\"][\"inputPredictedField\"] is None:\n options[\"inferenceArgs\"][\"inputPredictedField\"] = \"auto\"\n\n elif swarmSize == 'small':\n if options['minParticlesPerSwarm'] is None:\n options['minParticlesPerSwarm'] = 3\n if options['iterationCount'] is None:\n options['iterationCount'] = 100\n if options['maxModels'] is None:\n options['maxModels'] = 1\n if options[\"inferenceArgs\"][\"inputPredictedField\"] is None:\n options[\"inferenceArgs\"][\"inputPredictedField\"] = \"yes\"\n\n elif swarmSize == 'medium':\n if options['minParticlesPerSwarm'] is None:\n options['minParticlesPerSwarm'] = 5\n if options['iterationCount'] is None:\n options['iterationCount'] = 4000\n if options['maxModels'] is None:\n options['maxModels'] = 200\n if options[\"inferenceArgs\"][\"inputPredictedField\"] is None:\n options[\"inferenceArgs\"][\"inputPredictedField\"] = \"auto\"\n\n elif swarmSize == 'large':\n if options['minParticlesPerSwarm'] is None:\n options['minParticlesPerSwarm'] = 15\n #options['killUselessSwarms'] = False\n #options['minFieldContribution'] = -1000\n #options['maxFieldBranching'] = 10\n #options['tryAll3FieldCombinations'] = True\n options['tryAll3FieldCombinationsWTimestamps'] = True\n if options[\"inferenceArgs\"][\"inputPredictedField\"] is None:\n options[\"inferenceArgs\"][\"inputPredictedField\"] = \"auto\"\n\n else:\n raise RuntimeError(\"Unsupported swarm size: %s\" % (swarmSize))\n\n\n\n # -----------------------------------------------------------------------\n # Get token replacements\n tokenReplacements = dict()\n\n #--------------------------------------------------------------------------\n # Generate the encoder related substitution strings\n includedFields = options['includedFields']\n if hsVersion == 'v1':\n (encoderSpecsStr, permEncoderChoicesStr) = \\\n _generateEncoderStringsV1(includedFields)\n elif hsVersion in ['v2', 'ensemble']:\n (encoderSpecsStr, permEncoderChoicesStr) = \\\n _generateEncoderStringsV2(includedFields, options)\n else:\n raise RuntimeError(\"Unsupported hsVersion of %s\" % (hsVersion))\n\n\n\n #--------------------------------------------------------------------------\n # Generate the string containing the sensor auto-reset dict.\n if options['resetPeriod'] is not None:\n sensorAutoResetStr = pprint.pformat(options['resetPeriod'],\n indent=2*_INDENT_STEP)\n else:\n sensorAutoResetStr = 'None'\n\n\n #--------------------------------------------------------------------------\n # Generate the string containing the aggregation settings.\n aggregationPeriod = {\n 'days': 0,\n 'hours': 0,\n 'microseconds': 0,\n 'milliseconds': 0,\n 'minutes': 0,\n 'months': 0,\n 'seconds': 0,\n 'weeks': 0,\n 'years': 0,\n }\n\n # Honor any overrides provided in the stream definition\n aggFunctionsDict = {}\n if 'aggregation' in options['streamDef']:\n for key in aggregationPeriod.keys():\n if key in options['streamDef']['aggregation']:\n aggregationPeriod[key] = options['streamDef']['aggregation'][key]\n if 'fields' in options['streamDef']['aggregation']:\n for (fieldName, func) in options['streamDef']['aggregation']['fields']:\n aggFunctionsDict[fieldName] = str(func)\n\n # Do we have any aggregation at all?\n hasAggregation = False\n for v in aggregationPeriod.values():\n if v != 0:\n hasAggregation = True\n break\n\n\n # Convert the aggFunctionsDict to a list\n aggFunctionList = aggFunctionsDict.items()\n aggregationInfo = dict(aggregationPeriod)\n aggregationInfo['fields'] = aggFunctionList\n\n # Form the aggregation strings\n aggregationInfoStr = \"%s\" % (pprint.pformat(aggregationInfo,\n indent=2*_INDENT_STEP))\n\n\n\n # -----------------------------------------------------------------------\n # Generate the string defining the dataset. This is basically the\n # streamDef, but referencing the aggregation we already pulled out into the\n # config dict (which enables permuting over it)\n datasetSpec = options['streamDef']\n if 'aggregation' in datasetSpec:\n datasetSpec.pop('aggregation')\n if hasAggregation:\n datasetSpec['aggregation'] = '$SUBSTITUTE'\n datasetSpecStr = pprint.pformat(datasetSpec, indent=2*_INDENT_STEP)\n datasetSpecStr = datasetSpecStr.replace(\n \"'$SUBSTITUTE'\", \"config['aggregationInfo']\")\n datasetSpecStr = _indentLines(datasetSpecStr, 2, indentFirstLine=False)\n\n\n # -----------------------------------------------------------------------\n # Was computeInterval specified with Multistep prediction? If so, this swarm\n # should permute over different aggregations\n computeInterval = options['computeInterval']\n if computeInterval is not None \\\n and options['inferenceType'] in ['NontemporalMultiStep',\n 'TemporalMultiStep',\n 'MultiStep']:\n\n # Compute the predictAheadTime based on the minAggregation (specified in\n # the stream definition) and the number of prediction steps\n predictionSteps = options['inferenceArgs'].get('predictionSteps', [1])\n if len(predictionSteps) > 1:\n raise _InvalidCommandArgException(\"Invalid predictionSteps: %s. \" \\\n \"When computeInterval is specified, there can only be one \" \\\n \"stepSize in predictionSteps.\" % predictionSteps)\n\n if max(aggregationInfo.values()) == 0:\n raise _InvalidCommandArgException(\"Missing or nil stream aggregation: \"\n \"When computeInterval is specified, then the stream aggregation \"\n \"interval must be non-zero.\")\n\n # Compute the predictAheadTime\n numSteps = predictionSteps[0]\n predictAheadTime = dict(aggregationPeriod)\n for key in predictAheadTime.iterkeys():\n predictAheadTime[key] *= numSteps\n predictAheadTimeStr = pprint.pformat(predictAheadTime,\n indent=2*_INDENT_STEP)\n\n # This tells us to plug in a wildcard string for the prediction steps that\n # we use in other parts of the description file (metrics, inferenceArgs,\n # etc.)\n options['dynamicPredictionSteps'] = True\n\n else:\n options['dynamicPredictionSteps'] = False\n predictAheadTimeStr = \"None\"\n\n\n\n # -----------------------------------------------------------------------\n # Save environment-common token substitutions\n\n tokenReplacements['\\$EXP_GENERATOR_PROGRAM_PATH'] = \\\n _quoteAndEscape(os.path.abspath(__file__))\n\n # If the \"uber\" metric 'MultiStep' was specified, then plug in TemporalMultiStep\n # by default\n inferenceType = options['inferenceType']\n if inferenceType == 'MultiStep':\n inferenceType = InferenceType.TemporalMultiStep\n tokenReplacements['\\$INFERENCE_TYPE'] = \"'%s'\" % inferenceType\n\n # Nontemporal classificaion uses only encoder and classifier\n if inferenceType == InferenceType.NontemporalClassification:\n tokenReplacements['\\$SP_ENABLE'] = \"False\"\n tokenReplacements['\\$TP_ENABLE'] = \"False\"\n else:\n tokenReplacements['\\$SP_ENABLE'] = \"True\"\n tokenReplacements['\\$TP_ENABLE'] = \"True\"\n tokenReplacements['\\$CLA_CLASSIFIER_IMPL'] = \"\"\n\n\n tokenReplacements['\\$ANOMALY_PARAMS'] = pprint.pformat(\n options['anomalyParams'], indent=2*_INDENT_STEP)\n\n tokenReplacements['\\$ENCODER_SPECS'] = encoderSpecsStr\n tokenReplacements['\\$SENSOR_AUTO_RESET'] = sensorAutoResetStr\n\n tokenReplacements['\\$AGGREGATION_INFO'] = aggregationInfoStr\n\n tokenReplacements['\\$DATASET_SPEC'] = datasetSpecStr\n if options['iterationCount'] is None:\n options['iterationCount'] = -1\n tokenReplacements['\\$ITERATION_COUNT'] \\\n = str(options['iterationCount'])\n\n tokenReplacements['\\$SP_POOL_PCT'] \\\n = str(options['spCoincInputPoolPct'])\n\n tokenReplacements['\\$HS_MIN_PARTICLES'] \\\n = str(options['minParticlesPerSwarm'])\n\n\n tokenReplacements['\\$SP_PERM_CONNECTED'] \\\n = str(options['spSynPermConnected'])\n\n tokenReplacements['\\$FIELD_PERMUTATION_LIMIT'] \\\n = str(options['fieldPermutationLimit'])\n\n tokenReplacements['\\$PERM_ENCODER_CHOICES'] \\\n = permEncoderChoicesStr\n\n predictionSteps = options['inferenceArgs'].get('predictionSteps', [1])\n predictionStepsStr = ','.join([str(x) for x in predictionSteps])\n tokenReplacements['\\$PREDICTION_STEPS'] = \"'%s'\" % (predictionStepsStr)\n\n tokenReplacements['\\$PREDICT_AHEAD_TIME'] = predictAheadTimeStr\n\n # Option permuting over SP synapse decrement value\n tokenReplacements['\\$PERM_SP_CHOICES'] = \"\"\n if options['spPermuteDecrement'] \\\n and options['inferenceType'] != 'NontemporalClassification':\n tokenReplacements['\\$PERM_SP_CHOICES'] = \\\n _ONE_INDENT +\"'synPermInactiveDec': PermuteFloat(0.0003, 0.1),\\n\"\n\n # The TM permutation parameters are not required for non-temporal networks\n if options['inferenceType'] in ['NontemporalMultiStep',\n 'NontemporalClassification']:\n tokenReplacements['\\$PERM_TP_CHOICES'] = \"\"\n else:\n tokenReplacements['\\$PERM_TP_CHOICES'] = \\\n \" 'activationThreshold': PermuteInt(12, 16),\\n\" \\\n + \" 'minThreshold': PermuteInt(9, 12),\\n\" \\\n + \" 'pamLength': PermuteInt(1, 5),\\n\"\n\n\n # If the inference type is just the generic 'MultiStep', then permute over\n # temporal/nonTemporal multistep\n if options['inferenceType'] == 'MultiStep':\n tokenReplacements['\\$PERM_INFERENCE_TYPE_CHOICES'] = \\\n \" 'inferenceType': PermuteChoices(['NontemporalMultiStep', \" \\\n + \"'TemporalMultiStep']),\"\n else:\n tokenReplacements['\\$PERM_INFERENCE_TYPE_CHOICES'] = \"\"\n\n\n # The Classifier permutation parameters are only required for\n # Multi-step inference types\n if options['inferenceType'] in ['NontemporalMultiStep', 'TemporalMultiStep',\n 'MultiStep', 'TemporalAnomaly',\n 'NontemporalClassification']:\n tokenReplacements['\\$PERM_CL_CHOICES'] = \\\n \" 'alpha': PermuteFloat(0.0001, 0.1),\\n\"\n\n else:\n tokenReplacements['\\$PERM_CL_CHOICES'] = \"\"\n\n\n # The Permutations alwaysIncludePredictedField setting.\n # * When the experiment description has 'inputPredictedField' set to 'no', we\n # simply do not put in an encoder for the predicted field.\n # * When 'inputPredictedField' is set to 'auto', we include an encoder for the\n # predicted field and swarming tries it out just like all the other fields.\n # * When 'inputPredictedField' is set to 'yes', we include this setting in\n # the permutations file which informs swarming to always use the\n # predicted field (the first swarm will be the predicted field only)\n tokenReplacements['\\$PERM_ALWAYS_INCLUDE_PREDICTED_FIELD'] = \\\n \"inputPredictedField = '%s'\" % \\\n (options[\"inferenceArgs\"][\"inputPredictedField\"])\n\n\n # The Permutations minFieldContribution setting\n if options.get('minFieldContribution', None) is not None:\n tokenReplacements['\\$PERM_MIN_FIELD_CONTRIBUTION'] = \\\n \"minFieldContribution = %d\" % (options['minFieldContribution'])\n else:\n tokenReplacements['\\$PERM_MIN_FIELD_CONTRIBUTION'] = \"\"\n\n # The Permutations killUselessSwarms setting\n if options.get('killUselessSwarms', None) is not None:\n tokenReplacements['\\$PERM_KILL_USELESS_SWARMS'] = \\\n \"killUselessSwarms = %r\" % (options['killUselessSwarms'])\n else:\n tokenReplacements['\\$PERM_KILL_USELESS_SWARMS'] = \"\"\n\n # The Permutations maxFieldBranching setting\n if options.get('maxFieldBranching', None) is not None:\n tokenReplacements['\\$PERM_MAX_FIELD_BRANCHING'] = \\\n \"maxFieldBranching = %r\" % (options['maxFieldBranching'])\n else:\n tokenReplacements['\\$PERM_MAX_FIELD_BRANCHING'] = \"\"\n\n # The Permutations tryAll3FieldCombinations setting\n if options.get('tryAll3FieldCombinations', None) is not None:\n tokenReplacements['\\$PERM_TRY_ALL_3_FIELD_COMBINATIONS'] = \\\n \"tryAll3FieldCombinations = %r\" % (options['tryAll3FieldCombinations'])\n else:\n tokenReplacements['\\$PERM_TRY_ALL_3_FIELD_COMBINATIONS'] = \"\"\n\n # The Permutations tryAll3FieldCombinationsWTimestamps setting\n if options.get('tryAll3FieldCombinationsWTimestamps', None) is not None:\n tokenReplacements['\\$PERM_TRY_ALL_3_FIELD_COMBINATIONS_W_TIMESTAMPS'] = \\\n \"tryAll3FieldCombinationsWTimestamps = %r\" % \\\n (options['tryAll3FieldCombinationsWTimestamps'])\n else:\n tokenReplacements['\\$PERM_TRY_ALL_3_FIELD_COMBINATIONS_W_TIMESTAMPS'] = \"\"\n\n\n # The Permutations fieldFields setting\n if options.get('fixedFields', None) is not None:\n tokenReplacements['\\$PERM_FIXED_FIELDS'] = \\\n \"fixedFields = %r\" % (options['fixedFields'])\n else:\n tokenReplacements['\\$PERM_FIXED_FIELDS'] = \"\"\n\n # The Permutations fastSwarmModelParams setting\n if options.get('fastSwarmModelParams', None) is not None:\n tokenReplacements['\\$PERM_FAST_SWARM_MODEL_PARAMS'] = \\\n \"fastSwarmModelParams = %r\" % (options['fastSwarmModelParams'])\n else:\n tokenReplacements['\\$PERM_FAST_SWARM_MODEL_PARAMS'] = \"\"\n\n\n # The Permutations maxModels setting\n if options.get('maxModels', None) is not None:\n tokenReplacements['\\$PERM_MAX_MODELS'] = \\\n \"maxModels = %r\" % (options['maxModels'])\n else:\n tokenReplacements['\\$PERM_MAX_MODELS'] = \"\"\n\n\n # --------------------------------------------------------------------------\n # The Aggregation choices have to be determined when we are permuting over\n # aggregations.\n if options['dynamicPredictionSteps']:\n debugAgg = True\n\n # First, we need to error check to insure that computeInterval is an integer\n # multiple of minAggregation (aggregationPeriod)\n quotient = aggregationDivide(computeInterval, aggregationPeriod)\n (isInt, multiple) = _isInt(quotient)\n if not isInt or multiple < 1:\n raise _InvalidCommandArgException(\"Invalid computeInterval: %s. \"\n \"computeInterval must be an integer multiple of the stream \"\n \"aggregation (%s).\" % (computeInterval, aggregationPeriod))\n\n\n # The valid aggregation choices are governed by the following constraint,\n # 1.) (minAggregation * N) * M = predictAheadTime\n # (minAggregation * N) * M = maxPredictionSteps * minAggregation\n # N * M = maxPredictionSteps\n #\n # 2.) computeInterval = K * aggregation\n # computeInterval = K * (minAggregation * N)\n #\n # where: aggregation = minAggregation * N\n # K, M and N are integers >= 1\n # N = aggregation / minAggregation\n # M = predictionSteps, for a particular aggregation\n # K = number of predictions within each compute interval\n #\n\n # Let's build up a a list of the possible N's that satisfy the\n # N * M = maxPredictionSteps constraint\n mTimesN = float(predictionSteps[0])\n possibleNs = []\n for n in xrange(1, int(mTimesN)+1):\n m = mTimesN / n\n mInt = int(round(m))\n if mInt < 1:\n break\n if abs(m - mInt) > 0.0001 * m:\n continue\n possibleNs.append(n)\n\n if debugAgg:\n print \"All integer factors of %d are: %s\" % (mTimesN, possibleNs)\n\n # Now go through and throw out any N's that don't satisfy the constraint:\n # computeInterval = K * (minAggregation * N)\n aggChoices = []\n for n in possibleNs:\n # Compute minAggregation * N\n agg = dict(aggregationPeriod)\n for key in agg.iterkeys():\n agg[key] *= n\n\n # Make sure computeInterval is an integer multiple of the aggregation\n # period\n quotient = aggregationDivide(computeInterval, agg)\n #print computeInterval, agg\n #print quotient\n #import sys; sys.exit()\n (isInt, multiple) = _isInt(quotient)\n if not isInt or multiple < 1:\n continue\n aggChoices.append(agg)\n\n # Only eveluate up to 5 different aggregations\n aggChoices = aggChoices[-5:]\n\n if debugAgg:\n print \"Aggregation choices that will be evaluted during swarming:\"\n for agg in aggChoices:\n print \" ==>\", agg\n print\n\n tokenReplacements['\\$PERM_AGGREGATION_CHOICES'] = (\n \"PermuteChoices(%s)\" % (\n pprint.pformat(aggChoices, indent=2*_INDENT_STEP)))\n\n else:\n tokenReplacements['\\$PERM_AGGREGATION_CHOICES'] = aggregationInfoStr\n\n\n\n # Generate the inferenceArgs replacement tokens\n _generateInferenceArgs(options, tokenReplacements)\n\n # Generate the metric replacement tokens\n _generateMetricsSubstitutions(options, tokenReplacements)\n\n\n # -----------------------------------------------------------------------\n # Generate Control dictionary\n environment = options['environment']\n if environment == OpfEnvironment.Nupic:\n tokenReplacements['\\$ENVIRONMENT'] = \"'%s'\"%OpfEnvironment.Nupic\n controlTemplate = \"nupicEnvironmentTemplate.tpl\"\n elif environment == OpfEnvironment.Experiment:\n tokenReplacements['\\$ENVIRONMENT'] = \"'%s'\"%OpfEnvironment.Experiment\n controlTemplate = \"opfExperimentTemplate.tpl\"\n else:\n raise _InvalidCommandArgException(\"Invalid environment type %s\"% environment)\n\n # -----------------------------------------------------------------------\n if outputDirPath is None:\n outputDirPath = tempfile.mkdtemp()\n if not os.path.exists(outputDirPath):\n os.makedirs(outputDirPath)\n\n print \"Generating experiment files in directory: %s...\" % (outputDirPath)\n descriptionPyPath = os.path.join(outputDirPath, \"description.py\")\n _generateFileFromTemplates([claDescriptionTemplateFile, controlTemplate],\n descriptionPyPath,\n tokenReplacements)\n\n permutationsPyPath = os.path.join(outputDirPath, \"permutations.py\")\n\n if hsVersion == 'v1':\n _generateFileFromTemplates(['permutationsTemplateV1.tpl'],permutationsPyPath,\n tokenReplacements)\n elif hsVersion == 'ensemble':\n _generateFileFromTemplates(['permutationsTemplateEnsemble.tpl'],permutationsPyPath,\n tokenReplacements)\n elif hsVersion == 'v2':\n _generateFileFromTemplates(['permutationsTemplateV2.tpl'],permutationsPyPath,\n tokenReplacements)\n else:\n raise(ValueError(\"This permutation version is not supported yet: %s\" %\n hsVersion))\n\n print \"done.\"", "response": "Generates the experiment for the given options."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate the token substitution for the metrics related fields.", "response": "def _generateMetricsSubstitutions(options, tokenReplacements):\n \"\"\"Generate the token substitution for metrics related fields.\n This includes:\n \\$METRICS\n \\$LOGGED_METRICS\n \\$PERM_OPTIMIZE_SETTING\n \"\"\"\n # -----------------------------------------------------------------------\n #\n options['loggedMetrics'] = [\".*\"]\n\n # -----------------------------------------------------------------------\n # Generate the required metrics\n metricList, optimizeMetricLabel = _generateMetricSpecs(options)\n\n metricListString = \",\\n\".join(metricList)\n metricListString = _indentLines(metricListString, 2, indentFirstLine=False)\n permOptimizeSettingStr = 'minimize = \"%s\"' % optimizeMetricLabel\n # -----------------------------------------------------------------------\n # Specify which metrics should be logged\n loggedMetricsListAsStr = \"[%s]\" % (\", \".join([\"'%s'\"% ptrn\n for ptrn in options['loggedMetrics']]))\n\n\n tokenReplacements['\\$LOGGED_METRICS'] \\\n = loggedMetricsListAsStr\n\n tokenReplacements['\\$METRICS'] = metricListString\n\n tokenReplacements['\\$PERM_OPTIMIZE_SETTING'] \\\n = permOptimizeSettingStr"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _generateMetricSpecs(options):\n inferenceType = options['inferenceType']\n inferenceArgs = options['inferenceArgs']\n predictionSteps = inferenceArgs['predictionSteps']\n metricWindow = options['metricWindow']\n if metricWindow is None:\n metricWindow = int(Configuration.get(\"nupic.opf.metricWindow\"))\n\n metricSpecStrings = []\n optimizeMetricLabel = \"\"\n\n # -----------------------------------------------------------------------\n # Generate the metrics specified by the expGenerator paramters\n metricSpecStrings.extend(_generateExtraMetricSpecs(options))\n\n # -----------------------------------------------------------------------\n\n optimizeMetricSpec = None\n # If using a dynamically computed prediction steps (i.e. when swarming\n # over aggregation is requested), then we will plug in the variable\n # predictionSteps in place of the statically provided predictionSteps\n # from the JSON description.\n if options['dynamicPredictionSteps']:\n assert len(predictionSteps) == 1\n predictionSteps = ['$REPLACE_ME']\n\n # -----------------------------------------------------------------------\n # Metrics for temporal prediction\n if inferenceType in (InferenceType.TemporalNextStep,\n InferenceType.TemporalAnomaly,\n InferenceType.TemporalMultiStep,\n InferenceType.NontemporalMultiStep,\n InferenceType.NontemporalClassification,\n 'MultiStep'):\n\n predictedFieldName, predictedFieldType = _getPredictedField(options)\n isCategory = _isCategory(predictedFieldType)\n metricNames = ('avg_err',) if isCategory else ('aae', 'altMAPE')\n trivialErrorMetric = 'avg_err' if isCategory else 'altMAPE'\n oneGramErrorMetric = 'avg_err' if isCategory else 'altMAPE'\n movingAverageBaselineName = 'moving_mode' if isCategory else 'moving_mean'\n\n # Multi-step metrics\n for metricName in metricNames:\n metricSpec, metricLabel = \\\n _generateMetricSpecString(field=predictedFieldName,\n inferenceElement=InferenceElement.multiStepBestPredictions,\n metric='multiStep',\n params={'errorMetric': metricName,\n 'window':metricWindow,\n 'steps': predictionSteps},\n returnLabel=True)\n metricSpecStrings.append(metricSpec)\n\n # If the custom error metric was specified, add that\n if options[\"customErrorMetric\"] is not None :\n metricParams = dict(options[\"customErrorMetric\"])\n metricParams['errorMetric'] = 'custom_error_metric'\n metricParams['steps'] = predictionSteps\n # If errorWindow is not specified, make it equal to the default window\n if not \"errorWindow\" in metricParams:\n metricParams[\"errorWindow\"] = metricWindow\n metricSpec, metricLabel =_generateMetricSpecString(field=predictedFieldName,\n inferenceElement=InferenceElement.multiStepPredictions,\n metric=\"multiStep\",\n params=metricParams,\n returnLabel=True)\n metricSpecStrings.append(metricSpec)\n\n # If this is the first specified step size, optimize for it. Be sure to\n # escape special characters since this is a regular expression\n optimizeMetricSpec = metricSpec\n metricLabel = metricLabel.replace('[', '\\\\[')\n metricLabel = metricLabel.replace(']', '\\\\]')\n optimizeMetricLabel = metricLabel\n\n if options[\"customErrorMetric\"] is not None :\n optimizeMetricLabel = \".*custom_error_metric.*\"\n\n # Add in the trivial metrics\n if options[\"runBaselines\"] \\\n and inferenceType != InferenceType.NontemporalClassification:\n for steps in predictionSteps:\n metricSpecStrings.append(\n _generateMetricSpecString(field=predictedFieldName,\n inferenceElement=InferenceElement.prediction,\n metric=\"trivial\",\n params={'window':metricWindow,\n \"errorMetric\":trivialErrorMetric,\n 'steps': steps})\n )\n\n ##Add in the One-Gram baseline error metric\n #metricSpecStrings.append(\n # _generateMetricSpecString(field=predictedFieldName,\n # inferenceElement=InferenceElement.encodings,\n # metric=\"two_gram\",\n # params={'window':metricWindow,\n # \"errorMetric\":oneGramErrorMetric,\n # 'predictionField':predictedFieldName,\n # 'steps': steps})\n # )\n #\n #Include the baseline moving mean/mode metric\n if isCategory:\n metricSpecStrings.append(\n _generateMetricSpecString(field=predictedFieldName,\n inferenceElement=InferenceElement.prediction,\n metric=movingAverageBaselineName,\n params={'window':metricWindow\n ,\"errorMetric\":\"avg_err\",\n \"mode_window\":200,\n \"steps\": steps})\n )\n else :\n metricSpecStrings.append(\n _generateMetricSpecString(field=predictedFieldName,\n inferenceElement=InferenceElement.prediction,\n metric=movingAverageBaselineName,\n params={'window':metricWindow\n ,\"errorMetric\":\"altMAPE\",\n \"mean_window\":200,\n \"steps\": steps})\n )\n\n\n\n\n # -----------------------------------------------------------------------\n # Metrics for classification\n elif inferenceType in (InferenceType.TemporalClassification):\n\n metricName = 'avg_err'\n trivialErrorMetric = 'avg_err'\n oneGramErrorMetric = 'avg_err'\n movingAverageBaselineName = 'moving_mode'\n\n optimizeMetricSpec, optimizeMetricLabel = \\\n _generateMetricSpecString(inferenceElement=InferenceElement.classification,\n metric=metricName,\n params={'window':metricWindow},\n returnLabel=True)\n\n metricSpecStrings.append(optimizeMetricSpec)\n\n if options[\"runBaselines\"]:\n # If temporal, generate the trivial predictor metric\n if inferenceType == InferenceType.TemporalClassification:\n metricSpecStrings.append(\n _generateMetricSpecString(inferenceElement=InferenceElement.classification,\n metric=\"trivial\",\n params={'window':metricWindow,\n \"errorMetric\":trivialErrorMetric})\n )\n metricSpecStrings.append(\n _generateMetricSpecString(inferenceElement=InferenceElement.classification,\n metric=\"two_gram\",\n params={'window':metricWindow,\n \"errorMetric\":oneGramErrorMetric})\n )\n metricSpecStrings.append(\n _generateMetricSpecString(inferenceElement=InferenceElement.classification,\n metric=movingAverageBaselineName,\n params={'window':metricWindow\n ,\"errorMetric\":\"avg_err\",\n \"mode_window\":200})\n )\n\n\n # Custom Error Metric\n if not options[\"customErrorMetric\"] == None :\n #If errorWindow is not specified, make it equal to the default window\n if not \"errorWindow\" in options[\"customErrorMetric\"]:\n options[\"customErrorMetric\"][\"errorWindow\"] = metricWindow\n optimizeMetricSpec = _generateMetricSpecString(\n inferenceElement=InferenceElement.classification,\n metric=\"custom\",\n params=options[\"customErrorMetric\"])\n optimizeMetricLabel = \".*custom_error_metric.*\"\n\n metricSpecStrings.append(optimizeMetricSpec)\n\n\n # -----------------------------------------------------------------------\n # If plug in the predictionSteps variable for any dynamically generated\n # prediction steps\n if options['dynamicPredictionSteps']:\n for i in range(len(metricSpecStrings)):\n metricSpecStrings[i] = metricSpecStrings[i].replace(\n \"'$REPLACE_ME'\", \"predictionSteps\")\n optimizeMetricLabel = optimizeMetricLabel.replace(\n \"'$REPLACE_ME'\", \".*\")\n return metricSpecStrings, optimizeMetricLabel", "response": "Generates the metric specs for a given InferenceType\n \u00bb"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _generateExtraMetricSpecs(options):\n _metricSpecSchema = {'properties': {}}\n\n results = []\n for metric in options['metrics']:\n\n for propertyName in _metricSpecSchema['properties'].keys():\n _getPropertyValue(_metricSpecSchema, propertyName, metric)\n\n\n specString, label = _generateMetricSpecString(\n field=metric['field'],\n metric=metric['metric'],\n params=metric['params'],\n inferenceElement=\\\n metric['inferenceElement'],\n returnLabel=True)\n if metric['logged']:\n options['loggedMetrics'].append(label)\n\n results.append(specString)\n\n return results", "response": "Generates the non - default metrics specified by the expGenerator params"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the predicted field and its datatype from the options dictionary", "response": "def _getPredictedField(options):\n \"\"\" Gets the predicted field and it's datatype from the options dictionary\n\n Returns: (predictedFieldName, predictedFieldType)\n \"\"\"\n if not options['inferenceArgs'] or \\\n not options['inferenceArgs']['predictedField']:\n return None, None\n\n predictedField = options['inferenceArgs']['predictedField']\n predictedFieldInfo = None\n includedFields = options['includedFields']\n\n for info in includedFields:\n if info['fieldName'] == predictedField:\n predictedFieldInfo = info\n break\n\n if predictedFieldInfo is None:\n raise ValueError(\n \"Predicted field '%s' does not exist in included fields.\" % predictedField\n )\n predictedFieldType = predictedFieldInfo['fieldType']\n\n return predictedField, predictedFieldType"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _generateInferenceArgs(options, tokenReplacements):\n inferenceType = options['inferenceType']\n optionInferenceArgs = options.get('inferenceArgs', None)\n resultInferenceArgs = {}\n predictedField = _getPredictedField(options)[0]\n\n if inferenceType in (InferenceType.TemporalNextStep,\n InferenceType.TemporalAnomaly):\n assert predictedField, \"Inference Type '%s' needs a predictedField \"\\\n \"specified in the inferenceArgs dictionary\"\\\n % inferenceType\n\n if optionInferenceArgs:\n # If we will be using a dynamically created predictionSteps, plug in that\n # variable name in place of the constant scalar value\n if options['dynamicPredictionSteps']:\n altOptionInferenceArgs = copy.deepcopy(optionInferenceArgs)\n altOptionInferenceArgs['predictionSteps'] = '$REPLACE_ME'\n resultInferenceArgs = pprint.pformat(altOptionInferenceArgs)\n resultInferenceArgs = resultInferenceArgs.replace(\"'$REPLACE_ME'\",\n '[predictionSteps]')\n else:\n resultInferenceArgs = pprint.pformat(optionInferenceArgs)\n\n tokenReplacements['\\$INFERENCE_ARGS'] = resultInferenceArgs\n\n tokenReplacements['\\$PREDICTION_FIELD'] = predictedField", "response": "Generates the token substitutions related to the predicted field\n and the supplemental arguments for prediction\n Generates the token substitutions related to the predicted field\n and the supplemental arguments for prediction\n "} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef expGenerator(args):\n\n # -----------------------------------------------------------------\n # Parse command line options\n #\n parser = OptionParser()\n parser.set_usage(\"%prog [options] --description='{json object with args}'\\n\" + \\\n \"%prog [options] --descriptionFromFile='{filename}'\\n\" + \\\n \"%prog [options] --showSchema\")\n\n parser.add_option(\"--description\", dest = \"description\",\n help = \"Tells ExpGenerator to generate an experiment description.py and \" \\\n \"permutations.py file using the given JSON formatted experiment \"\\\n \"description string.\")\n\n parser.add_option(\"--descriptionFromFile\", dest = 'descriptionFromFile',\n help = \"Tells ExpGenerator to open the given filename and use it's \" \\\n \"contents as the JSON formatted experiment description.\")\n\n parser.add_option(\"--claDescriptionTemplateFile\",\n dest = 'claDescriptionTemplateFile',\n default = 'claDescriptionTemplate.tpl',\n help = \"The file containing the template description file for \" \\\n \" ExpGenerator [default: %default]\")\n\n parser.add_option(\"--showSchema\",\n action=\"store_true\", dest=\"showSchema\",\n help=\"Prints the JSON schemas for the --description arg.\")\n\n parser.add_option(\"--version\", dest = 'version', default='v2',\n help = \"Generate the permutations file for this version of hypersearch.\"\n \" Possible choices are 'v1' and 'v2' [default: %default].\")\n\n parser.add_option(\"--outDir\",\n dest = \"outDir\", default=None,\n help = \"Where to generate experiment. If not specified, \" \\\n \"then a temp directory will be created\"\n )\n (options, remainingArgs) = parser.parse_args(args)\n\n #print(\"OPTIONS=%s\" % (str(options)))\n\n # -----------------------------------------------------------------\n # Check for unprocessed args\n #\n if len(remainingArgs) > 0:\n raise _InvalidCommandArgException(\n _makeUsageErrorStr(\"Unexpected command-line args: <%s>\" % \\\n (' '.join(remainingArgs),), parser.get_usage()))\n\n # -----------------------------------------------------------------\n # Check for use of mutually-exclusive options\n #\n activeOptions = filter(lambda x: getattr(options, x) != None,\n ('description', 'showSchema'))\n if len(activeOptions) > 1:\n raise _InvalidCommandArgException(\n _makeUsageErrorStr((\"The specified command options are \" + \\\n \"mutually-exclusive: %s\") % (activeOptions,),\n parser.get_usage()))\n\n\n\n # -----------------------------------------------------------------\n # Process requests\n #\n if options.showSchema:\n _handleShowSchemaOption()\n\n elif options.description:\n _handleDescriptionOption(options.description, options.outDir,\n parser.get_usage(), hsVersion=options.version,\n claDescriptionTemplateFile = options.claDescriptionTemplateFile)\n\n elif options.descriptionFromFile:\n _handleDescriptionFromFileOption(options.descriptionFromFile,\n options.outDir, parser.get_usage(), hsVersion=options.version,\n claDescriptionTemplateFile = options.claDescriptionTemplateFile)\n\n else:\n raise _InvalidCommandArgException(\n _makeUsageErrorStr(\"Error in validating command options. No option \"\n \"provided:\\n\", parser.get_usage()))", "response": "Generates an experiment description file and generates it with the given arguments."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse a textual datetime format and returns a Python datetime object.", "response": "def parseTimestamp(s):\n \"\"\"\n Parses a textual datetime format and return a Python datetime object.\n\n The supported format is: ``yyyy-mm-dd h:m:s.ms``\n\n The time component is optional.\n\n - hours are 00..23 (no AM/PM)\n - minutes are 00..59\n - seconds are 00..59\n - micro-seconds are 000000..999999\n\n :param s: (string) input time text\n :return: (datetime.datetime)\n \"\"\"\n s = s.strip()\n for pattern in DATETIME_FORMATS:\n try:\n return datetime.datetime.strptime(s, pattern)\n except ValueError:\n pass\n raise ValueError('The provided timestamp %s is malformed. The supported '\n 'formats are: [%s]' % (s, ', '.join(DATETIME_FORMATS)))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef parseBool(s):\n l = s.lower()\n if l in (\"true\", \"t\", \"1\"):\n return True\n if l in (\"false\", \"f\", \"0\"):\n return False\n raise Exception(\"Unable to convert string '%s' to a boolean value\" % s)", "response": "Convert string to boolean"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef escape(s):\n if s is None:\n return ''\n\n assert isinstance(s, basestring), \\\n \"expected %s but got %s; value=%s\" % (basestring, type(s), s)\n s = s.replace('\\\\', '\\\\\\\\')\n s = s.replace('\\n', '\\\\n')\n s = s.replace('\\t', '\\\\t')\n s = s.replace(',', '\\t')\n return s", "response": "Escape a string in a string\n archive."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse a string containing only 0 and 1 s and returns a Python list object.", "response": "def parseSdr(s):\n \"\"\"\n Parses a string containing only 0's and 1's and return a Python list object.\n\n :param s: (string) string to parse\n :returns: (list) SDR out\n \"\"\"\n assert isinstance(s, basestring)\n sdr = [int(c) for c in s if c in (\"0\", \"1\")]\n if len(sdr) != len(s):\n raise ValueError(\"The provided string %s is malformed. The string should \"\n \"have only 0's and 1's.\")\n\n return sdr"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse a string of space - separated numbers returning a Python list.", "response": "def parseStringList(s):\n \"\"\"\n Parse a string of space-separated numbers, returning a Python list.\n\n :param s: (string) to parse\n :returns: (list) binary SDR\n \"\"\"\n assert isinstance(s, basestring)\n return [int(i) for i in s.split()]"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntranslates an index into coordinates using the given coordinate system.", "response": "def coordinatesFromIndex(index, dimensions):\n \"\"\"\n Translate an index into coordinates, using the given coordinate system.\n\n Similar to ``numpy.unravel_index``.\n\n :param index: (int) The index of the point. The coordinates are expressed as a \n single index by using the dimensions as a mixed radix definition. For \n example, in dimensions 42x10, the point [1, 4] is index \n 1*420 + 4*10 = 460.\n\n :param dimensions (list of ints) The coordinate system.\n\n :returns: (list) of coordinates of length ``len(dimensions)``.\n \"\"\"\n coordinates = [0] * len(dimensions)\n\n shifted = index\n for i in xrange(len(dimensions) - 1, 0, -1):\n coordinates[i] = shifted % dimensions[i]\n shifted = shifted / dimensions[i]\n\n coordinates[0] = shifted\n\n return coordinates"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef indexFromCoordinates(coordinates, dimensions):\n index = 0\n for i, dimension in enumerate(dimensions):\n index *= dimension\n index += coordinates[i]\n\n return index", "response": "Translate coordinates into an index using the given coordinate system."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef neighborhood(centerIndex, radius, dimensions):\n centerPosition = coordinatesFromIndex(centerIndex, dimensions)\n\n intervals = []\n for i, dimension in enumerate(dimensions):\n left = max(0, centerPosition[i] - radius)\n right = min(dimension - 1, centerPosition[i] + radius)\n intervals.append(xrange(left, right + 1))\n\n coords = numpy.array(list(itertools.product(*intervals)))\n return numpy.ravel_multi_index(coords.T, dimensions)", "response": "Return the points in the neighborhood of a point."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef encodeIntoArray(self, inputData, output):\n (coordinate, radius) = inputData\n\n assert isinstance(radius, int), (\"Expected integer radius, got: {} ({})\"\n .format(radius, type(radius)))\n\n neighbors = self._neighbors(coordinate, radius)\n winners = self._topWCoordinates(neighbors, self.w)\n\n bitFn = lambda coordinate: self._bitForCoordinate(coordinate, self.n)\n indices = numpy.array([bitFn(w) for w in winners])\n\n output[:] = 0\n output[indices] = 1", "response": "Encodes the SDR in the input numpy array into the output numpy array."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _neighbors(coordinate, radius):\n ranges = (xrange(n-radius, n+radius+1) for n in coordinate.tolist())\n return numpy.array(list(itertools.product(*ranges)))", "response": "Returns coordinates around given coordinate within given radius."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the top W coordinates by order.", "response": "def _topWCoordinates(cls, coordinates, w):\n \"\"\"\n Returns the top W coordinates by order.\n\n @param coordinates (numpy.array) A 2D numpy array, where each element\n is a coordinate\n @param w (int) Number of top coordinates to return\n @return (numpy.array) A subset of `coordinates`, containing only the\n top ones by order\n \"\"\"\n orders = numpy.array([cls._orderForCoordinate(c)\n for c in coordinates.tolist()])\n indices = numpy.argsort(orders)[-w:]\n return coordinates[indices]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nhashing a coordinate to a 64 bit integer.", "response": "def _hashCoordinate(coordinate):\n \"\"\"Hash a coordinate to a 64 bit integer.\"\"\"\n coordinateStr = \",\".join(str(v) for v in coordinate)\n # Compute the hash and convert to 64 bit int.\n hash = int(int(hashlib.md5(coordinateStr).hexdigest(), 16) % (2 ** 64))\n return hash"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the order for a coordinate.", "response": "def _orderForCoordinate(cls, coordinate):\n \"\"\"\n Returns the order for a coordinate.\n\n @param coordinate (numpy.array) Coordinate\n @return (float) A value in the interval [0, 1), representing the\n order of the coordinate\n \"\"\"\n seed = cls._hashCoordinate(coordinate)\n rng = Random(seed)\n return rng.getReal64()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _bitForCoordinate(cls, coordinate, n):\n seed = cls._hashCoordinate(coordinate)\n rng = Random(seed)\n return rng.getUInt32(n)", "response": "Maps the coordinate to a bit in the SDR."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef binSearch(arr, val):\n i = bisect_left(arr, val)\n if i != len(arr) and arr[i] == val:\n return i\n return -1", "response": "Function for running binary search on a sorted list."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a new segment on a cell.", "response": "def createSegment(self, cell):\n \"\"\" \n Adds a new segment on a cell.\n\n :param cell: (int) Cell index\n :returns: (int) New segment index\n \"\"\"\n cellData = self._cells[cell]\n\n if len(self._freeFlatIdxs) > 0:\n flatIdx = self._freeFlatIdxs.pop()\n else:\n flatIdx = self._nextFlatIdx\n self._segmentForFlatIdx.append(None)\n self._nextFlatIdx += 1\n\n ordinal = self._nextSegmentOrdinal\n self._nextSegmentOrdinal += 1\n\n segment = Segment(cell, flatIdx, ordinal)\n cellData._segments.append(segment)\n self._segmentForFlatIdx[flatIdx] = segment\n\n return segment"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndestroy a segment. :param segment: (:class:`Segment`) representing the segment to be destroyed.", "response": "def destroySegment(self, segment):\n \"\"\"\n Destroys a segment.\n\n :param segment: (:class:`Segment`) representing the segment to be destroyed.\n \"\"\"\n # Remove the synapses from all data structures outside this Segment.\n for synapse in segment._synapses:\n self._removeSynapseFromPresynapticMap(synapse)\n self._numSynapses -= len(segment._synapses)\n\n # Remove the segment from the cell's list.\n segments = self._cells[segment.cell]._segments\n i = segments.index(segment)\n del segments[i]\n\n # Free the flatIdx and remove the final reference so the Segment can be\n # garbage-collected.\n self._freeFlatIdxs.append(segment.flatIdx)\n self._segmentForFlatIdx[segment.flatIdx] = None"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a new synapse on a segment.", "response": "def createSynapse(self, segment, presynapticCell, permanence):\n \"\"\" \n Creates a new synapse on a segment.\n\n :param segment: (:class:`Segment`) Segment object for synapse to be synapsed \n to.\n :param presynapticCell: (int) Source cell index.\n :param permanence: (float) Initial permanence of synapse.\n :returns: (:class:`Synapse`) created synapse\n \"\"\"\n idx = len(segment._synapses)\n synapse = Synapse(segment, presynapticCell, permanence,\n self._nextSynapseOrdinal)\n self._nextSynapseOrdinal += 1\n segment._synapses.add(synapse)\n\n self._synapsesForPresynapticCell[presynapticCell].add(synapse)\n\n self._numSynapses += 1\n\n return synapse"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef destroySynapse(self, synapse):\n\n self._numSynapses -= 1\n\n self._removeSynapseFromPresynapticMap(synapse)\n\n synapse.segment._synapses.remove(synapse)", "response": "Destroys a synapse.\n\n :param synapse: (:class:`Synapse`) synapse to destroy"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef computeActivity(self, activePresynapticCells, connectedPermanence):\n\n numActiveConnectedSynapsesForSegment = [0] * self._nextFlatIdx\n numActivePotentialSynapsesForSegment = [0] * self._nextFlatIdx\n\n threshold = connectedPermanence - EPSILON\n\n for cell in activePresynapticCells:\n for synapse in self._synapsesForPresynapticCell[cell]:\n flatIdx = synapse.segment.flatIdx\n numActivePotentialSynapsesForSegment[flatIdx] += 1\n if synapse.permanence > threshold:\n numActiveConnectedSynapsesForSegment[flatIdx] += 1\n\n return (numActiveConnectedSynapsesForSegment,\n numActivePotentialSynapsesForSegment)", "response": "Compute each segment s number of active synapses for a given input."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef numSegments(self, cell=None):\n if cell is not None:\n return len(self._cells[cell]._segments)\n\n return self._nextFlatIdx - len(self._freeFlatIdxs)", "response": "Returns the number of segments on a specific cell."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef segmentPositionSortKey(self, segment):\n return segment.cell + (segment._ordinal / float(self._nextSegmentOrdinal))", "response": "Returns a numeric key for sorting this segment. This can be used with the \n python built - in sorted function."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write(self, proto):\n protoCells = proto.init('cells', self.numCells)\n\n for i in xrange(self.numCells):\n segments = self._cells[i]._segments\n protoSegments = protoCells[i].init('segments', len(segments))\n\n for j, segment in enumerate(segments):\n synapses = segment._synapses\n protoSynapses = protoSegments[j].init('synapses', len(synapses))\n\n for k, synapse in enumerate(sorted(synapses, key=lambda s: s._ordinal)):\n protoSynapses[k].presynapticCell = synapse.presynapticCell\n protoSynapses[k].permanence = synapse.permanence", "response": "Writes serialized data to proto object."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads deserialized data from proto object", "response": "def read(cls, proto):\n \"\"\" \n Reads deserialized data from proto object\n\n :param proto: (DynamicStructBuilder) Proto object\n\n :returns: (:class:`Connections`) instance\n \"\"\"\n #pylint: disable=W0212\n protoCells = proto.cells\n connections = cls(len(protoCells))\n\n for cellIdx, protoCell in enumerate(protoCells):\n protoCell = protoCells[cellIdx]\n protoSegments = protoCell.segments\n connections._cells[cellIdx] = CellData()\n segments = connections._cells[cellIdx]._segments\n\n for segmentIdx, protoSegment in enumerate(protoSegments):\n segment = Segment(cellIdx, connections._nextFlatIdx,\n connections._nextSegmentOrdinal)\n\n segments.append(segment)\n connections._segmentForFlatIdx.append(segment)\n connections._nextFlatIdx += 1\n connections._nextSegmentOrdinal += 1\n\n synapses = segment._synapses\n protoSynapses = protoSegment.synapses\n\n for synapseIdx, protoSynapse in enumerate(protoSynapses):\n presynapticCell = protoSynapse.presynapticCell\n synapse = Synapse(segment, presynapticCell, protoSynapse.permanence,\n ordinal=connections._nextSynapseOrdinal)\n connections._nextSynapseOrdinal += 1\n synapses.add(synapse)\n connections._synapsesForPresynapticCell[presynapticCell].add(synapse)\n\n connections._numSynapses += 1\n\n #pylint: enable=W0212\n return connections"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve the requested property as a string.", "response": "def getString(cls, prop):\n \"\"\" Retrieve the requested property as a string. If property does not exist,\n then KeyError will be raised.\n\n :param prop: (string) name of the property\n :raises: KeyError\n :returns: (string) property value\n \"\"\"\n if cls._properties is None:\n cls._readStdConfigFiles()\n\n # Allow configuration properties to be overridden via environment variables\n envValue = os.environ.get(\"%s%s\" % (cls.envPropPrefix,\n prop.replace('.', '_')), None)\n if envValue is not None:\n return envValue\n\n return cls._properties[prop]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getBool(cls, prop):\n\n value = cls.getInt(prop)\n\n if value not in (0, 1):\n raise ValueError(\"Expected 0 or 1, but got %r in config property %s\" % (\n value, prop))\n\n return bool(value)", "response": "Retrieve the requested property and return it as a bool."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set(cls, prop, value):\n\n if cls._properties is None:\n cls._readStdConfigFiles()\n\n cls._properties[prop] = str(value)", "response": "Set the value of the given configuration property."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dict(cls):\n\n if cls._properties is None:\n cls._readStdConfigFiles()\n\n # Make a copy so we can update any current values obtained from environment\n # variables\n result = dict(cls._properties)\n keys = os.environ.keys()\n replaceKeys = filter(lambda x: x.startswith(cls.envPropPrefix),\n keys)\n for envKey in replaceKeys:\n key = envKey[len(cls.envPropPrefix):]\n key = key.replace('_', '.')\n result[key] = os.environ[envKey]\n\n return result", "response": "Returns a dict containing all of the configuration properties containing all of the configuration properties in the system."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef readConfigFile(cls, filename, path=None):\n properties = cls._readConfigFile(filename, path)\n\n # Create properties dict if necessary\n if cls._properties is None:\n cls._properties = dict()\n\n for name in properties:\n if 'value' in properties[name]:\n cls._properties[name] = properties[name]['value']", "response": "Parse the given XML file and store all properties it describes."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsearch the configuration path for the given filename.", "response": "def findConfigFile(cls, filename):\n \"\"\" Search the configuration path (specified via the NTA_CONF_PATH\n environment variable) for the given filename. If found, return the complete\n path to the file.\n\n :param filename: (string) name of file to locate\n \"\"\"\n\n paths = cls.getConfigPaths()\n for p in paths:\n testPath = os.path.join(p, filename)\n if os.path.isfile(testPath):\n return os.path.join(p, filename)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getConfigPaths(cls):\n configPaths = []\n if cls._configPaths is not None:\n return cls._configPaths\n\n else:\n if 'NTA_CONF_PATH' in os.environ:\n configVar = os.environ['NTA_CONF_PATH']\n # Return as a list of paths\n configPaths = configVar.split(os.pathsep)\n\n return configPaths", "response": "Return the list of paths to search for configuration files."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds noise to the input.", "response": "def addNoise(input, noise=0.1, doForeground=True, doBackground=True):\n \"\"\"\n Add noise to the given input.\n\n Parameters:\n -----------------------------------------------\n input: the input to add noise to\n noise: how much noise to add\n doForeground: If true, turn off some of the 1 bits in the input\n doBackground: If true, turn on some of the 0 bits in the input\n\n \"\"\"\n if doForeground and doBackground:\n return numpy.abs(input - (numpy.random.random(input.shape) < noise))\n else:\n if doForeground:\n return numpy.logical_and(input, numpy.random.random(input.shape) > noise)\n if doBackground:\n return numpy.logical_or(input, numpy.random.random(input.shape) < noise)\n return input"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates a coincidence matrix.", "response": "def generateCoincMatrix(nCoinc=10, length=500, activity=50):\n \"\"\"\n Generate a coincidence matrix. This is used to generate random inputs to the\n temporal learner and to compare the predicted output against.\n\n It generates a matrix of nCoinc rows, each row has length 'length' and has\n a total of 'activity' bits on.\n\n Parameters:\n -----------------------------------------------\n nCoinc: the number of rows to generate\n length: the length of each row\n activity: the number of ones to put into each row.\n\n \"\"\"\n\n coincMatrix0 = SM32(int(nCoinc), int(length))\n theOnes = numpy.array([1.0] * activity, dtype=numpy.float32)\n for rowIdx in xrange(nCoinc):\n coinc = numpy.array(random.sample(xrange(length),\n activity), dtype=numpy.uint32)\n coinc.sort()\n coincMatrix0.setRowFromSparse(rowIdx, coinc, theOnes)\n\n # This is the right code to use, it's faster, but it derails the unit\n # testing of the pooling for now.\n coincMatrix = SM32(int(nCoinc), int(length))\n coincMatrix.initializeWithFixedNNZR(activity)\n\n return coincMatrix0"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ngenerate a list of random sparse distributed vectors.", "response": "def generateVectors(numVectors=100, length=500, activity=50):\n \"\"\"\n Generate a list of random sparse distributed vectors. This is used to generate\n training vectors to the spatial or temporal learner and to compare the predicted\n output against.\n\n It generates a list of 'numVectors' elements, each element has length 'length'\n and has a total of 'activity' bits on.\n\n Parameters:\n -----------------------------------------------\n numVectors: the number of vectors to generate\n length: the length of each row\n activity: the number of ones to put into each row.\n\n \"\"\"\n\n vectors = []\n coinc = numpy.zeros(length, dtype='int32')\n indexList = range(length)\n\n for i in xrange(numVectors):\n coinc[:] = 0\n coinc[random.sample(indexList, activity)] = 1\n vectors.append(coinc.copy())\n\n return vectors"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef generateSimpleSequences(nCoinc=10, seqLength=[5,6,7], nSeq=100):\n\n coincList = range(nCoinc)\n seqList = []\n\n for i in xrange(nSeq):\n if max(seqLength) <= nCoinc:\n seqList.append(random.sample(coincList, random.choice(seqLength)))\n else:\n len = random.choice(seqLength)\n seq = []\n for x in xrange(len):\n seq.append(random.choice(coincList))\n seqList.append(seq)\n\n return seqList", "response": "Generates a set of simple sequences."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef generateHubSequences(nCoinc=10, hubs = [2,6], seqLength=[5,6,7], nSeq=100):\n\n\n coincList = range(nCoinc)\n for hub in hubs:\n coincList.remove(hub)\n\n seqList = []\n for i in xrange(nSeq):\n length = random.choice(seqLength)-1\n seq = random.sample(coincList,length)\n seq.insert(length//2, random.choice(hubs))\n seqList.append(seq)\n\n return seqList", "response": "Generates a list of hubs that are available in the middle of the hubs."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating a non overlapping coincidence matrix.", "response": "def generateSimpleCoincMatrix(nCoinc=10, length=500, activity=50):\n \"\"\"\n Generate a non overlapping coincidence matrix. This is used to generate random\n inputs to the temporal learner and to compare the predicted output against.\n\n It generates a matrix of nCoinc rows, each row has length 'length' and has\n a total of 'activity' bits on.\n\n Parameters:\n -----------------------------------------------\n nCoinc: the number of rows to generate\n length: the length of each row\n activity: the number of ones to put into each row.\n\n \"\"\"\n assert nCoinc*activity<=length, \"can't generate non-overlapping coincidences\"\n coincMatrix = SM32(0, length)\n coinc = numpy.zeros(length, dtype='int32')\n\n for i in xrange(nCoinc):\n coinc[:] = 0\n coinc[i*activity:(i+1)*activity] = 1\n coincMatrix.addRow(coinc)\n\n return coincMatrix"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef generateSequences(nPatterns=10, patternLen=500, patternActivity=50,\n hubs=[2,6], seqLength=[5,6,7],\n nSimpleSequences=50, nHubSequences=50):\n \"\"\"\n Generate a set of simple and hub sequences. A simple sequence contains\n a randomly chosen set of elements from 0 to 'nCoinc-1'. A hub sequence\n always contains a hub element in the middle of it.\n\n Parameters:\n -----------------------------------------------\n nPatterns: the number of patterns to use in the sequences.\n patternLen: The number of elements in each pattern\n patternActivity: The number of elements that should be active in\n each pattern\n hubs: which of the elements will be used as hubs.\n seqLength: a list of possible sequence lengths. The length of each\n sequence will be randomly chosen from here.\n nSimpleSequences: The number of simple sequences to generate\n nHubSequences: The number of hub sequences to generate\n\n retval: (seqList, patterns)\n seqList: a list of sequences. Each sequence is itself a list\n containing the input pattern indices for that sequence.\n patterns: the input patterns used in the seqList.\n \"\"\"\n\n # Create the input patterns\n patterns = generateCoincMatrix(nCoinc=nPatterns, length=patternLen,\n activity=patternActivity)\n\n # Create the raw sequences\n seqList = generateSimpleSequences(nCoinc=nPatterns, seqLength=seqLength,\n nSeq=nSimpleSequences) + \\\n generateHubSequences(nCoinc=nPatterns, hubs=hubs, seqLength=seqLength,\n nSeq=nHubSequences)\n\n # Return results\n return (seqList, patterns)", "response": "Generates a set of simple and hub sequences."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating the simulated output from a spatial pooler that's sitting on top of another spatial pooler / temporal memory pair. The average on-time of the outputs from the simulated TM is given by the l1Pooling argument. In this routine, L1 refers to the first spatial and temporal memory and L2 refers to the spatial pooler above that. Parameters: ----------------------------------------------- nL1Patterns: the number of patterns to use in the L1 sequences. l1Hubs: which of the elements will be used as hubs. l1SeqLength: a list of possible sequence lengths. The length of each sequence will be randomly chosen from here. nL1SimpleSequences: The number of simple sequences to generate for L1 nL1HubSequences: The number of hub sequences to generate for L1 l1Pooling: The number of time steps to pool over in the L1 temporal pooler perfectStability: If true, then the input patterns represented by the sequences generated will have perfect stability over l1Pooling time steps. This is the best case ideal input to a TM. In actual situations, with an actual SP providing input, the stability will always be less than this. spHystereisFactor: The hysteresisFactor to use in the L2 spatial pooler. Only used when perfectStability is False patternLen: The number of elements in each pattern output by L2 patternActivity: The number of elements that should be active in each pattern @retval: (seqList, patterns) seqList: a list of sequences output from L2. Each sequence is itself a list containing the input pattern indices for that sequence. patterns: the input patterns used in the L2 seqList.", "response": "def generateL2Sequences(nL1Patterns=10, l1Hubs=[2,6], l1SeqLength=[5,6,7],\n nL1SimpleSequences=50, nL1HubSequences=50,\n l1Pooling=4, perfectStability=False, spHysteresisFactor=1.0,\n patternLen=500, patternActivity=50):\n \"\"\"\n Generate the simulated output from a spatial pooler that's sitting\n on top of another spatial pooler / temporal memory pair. The average on-time\n of the outputs from the simulated TM is given by the l1Pooling argument.\n\n In this routine, L1 refers to the first spatial and temporal memory and L2\n refers to the spatial pooler above that.\n\n Parameters:\n -----------------------------------------------\n nL1Patterns: the number of patterns to use in the L1 sequences.\n l1Hubs: which of the elements will be used as hubs.\n l1SeqLength: a list of possible sequence lengths. The length of each\n sequence will be randomly chosen from here.\n nL1SimpleSequences: The number of simple sequences to generate for L1\n nL1HubSequences: The number of hub sequences to generate for L1\n l1Pooling: The number of time steps to pool over in the L1 temporal\n pooler\n perfectStability: If true, then the input patterns represented by the\n sequences generated will have perfect stability over\n l1Pooling time steps. This is the best case ideal input\n to a TM. In actual situations, with an actual SP\n providing input, the stability will always be less than\n this.\n spHystereisFactor: The hysteresisFactor to use in the L2 spatial pooler.\n Only used when perfectStability is False\n patternLen: The number of elements in each pattern output by L2\n patternActivity: The number of elements that should be active in\n each pattern\n\n @retval: (seqList, patterns)\n seqList: a list of sequences output from L2. Each sequence is\n itself a list containing the input pattern indices for that\n sequence.\n patterns: the input patterns used in the L2 seqList.\n \"\"\"\n\n # First, generate the L1 sequences\n l1SeqList = generateSimpleSequences(nCoinc=nL1Patterns, seqLength=l1SeqLength,\n nSeq=nL1SimpleSequences) + \\\n generateHubSequences(nCoinc=nL1Patterns, hubs=l1Hubs,\n seqLength=l1SeqLength, nSeq=nL1HubSequences)\n\n # Generate the L2 SP output from those\n spOutput = generateSlowSPOutput(seqListBelow = l1SeqList,\n poolingTimeBelow=l1Pooling, outputWidth=patternLen,\n activity=patternActivity, perfectStability=perfectStability,\n spHysteresisFactor=spHysteresisFactor)\n\n # Map the spOutput patterns into indices into a pattern matrix which we\n # generate now.\n outSeq = None\n outSeqList = []\n outPatterns = SM32(0, patternLen)\n for pattern in spOutput:\n # If we have a reset vector start a new sequence\n if pattern.sum() == 0:\n if outSeq is not None:\n outSeqList.append(outSeq)\n outSeq = []\n continue\n\n # See if this vector matches a pattern we've already seen before\n patternIdx = None\n if outPatterns.nRows() > 0:\n # Find most matching 1's.\n matches = outPatterns.rightVecSumAtNZ(pattern)\n outCoinc = matches.argmax().astype('uint32')\n # See if its number of 1's is the same in the pattern and in the\n # coincidence row. If so, it is an exact match\n numOnes = pattern.sum()\n if matches[outCoinc] == numOnes \\\n and outPatterns.getRow(int(outCoinc)).sum() == numOnes:\n patternIdx = outCoinc\n\n # If no match, add this pattern to our matrix\n if patternIdx is None:\n outPatterns.addRow(pattern)\n patternIdx = outPatterns.nRows() - 1\n\n # Store the pattern index into the sequence\n outSeq.append(patternIdx)\n\n # Put in last finished sequence\n if outSeq is not None:\n outSeqList.append(outSeq)\n\n # Return with the seqList and patterns matrix\n return (outSeqList, outPatterns)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef vectorsFromSeqList(seqList, patternMatrix):\n\n totalLen = 0\n for seq in seqList:\n totalLen += len(seq)\n\n vectors = numpy.zeros((totalLen, patternMatrix.shape[1]), dtype='bool')\n vecOffset = 0\n for seq in seqList:\n seq = numpy.array(seq, dtype='uint32')\n for idx,coinc in enumerate(seq):\n vectors[vecOffset] = patternMatrix.getRow(int(coinc))\n vecOffset += 1\n\n return vectors", "response": "Returns a list of vectors from a list of sequences of pattern indices and a pattern lookup table\n "} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sameTMParams(tp1, tp2):\n result = True\n for param in [\"numberOfCols\", \"cellsPerColumn\", \"initialPerm\", \"connectedPerm\",\n \"minThreshold\", \"newSynapseCount\", \"permanenceInc\", \"permanenceDec\",\n \"permanenceMax\", \"globalDecay\", \"activationThreshold\",\n \"doPooling\", \"segUpdateValidDuration\",\n \"burnIn\", \"pamLength\", \"maxAge\"]:\n if getattr(tp1, param) != getattr(tp2,param):\n print param,\"is different\"\n print getattr(tp1, param), \"vs\", getattr(tp2,param)\n result = False\n return result", "response": "Given two TM instances see if any parameters are different."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sameSynapse(syn, synapses):\n for s in synapses:\n if (s[0]==syn[0]) and (s[1]==syn[1]) and (abs(s[2]-syn[2]) <= 0.001):\n return True\n return False", "response": "Checks whether this synapse and a list of synapses exist in the list."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns True if seg1 and seg2 are identical ignoring order of synapses", "response": "def sameSegment(seg1, seg2):\n \"\"\"Return True if seg1 and seg2 are identical, ignoring order of synapses\"\"\"\n result = True\n\n # check sequence segment, total activations etc. In case any are floats,\n # check that they are within 0.001.\n for field in [1, 2, 3, 4, 5, 6]:\n if abs(seg1[0][field] - seg2[0][field]) > 0.001:\n result = False\n\n # Compare number of synapses\n if len(seg1[1:]) != len(seg2[1:]):\n result = False\n\n # Now compare synapses, ignoring order of synapses\n for syn in seg2[1:]:\n if syn[2] <= 0:\n print \"A synapse with zero permanence encountered\"\n result = False\n if result == True:\n for syn in seg1[1:]:\n if syn[2] <= 0:\n print \"A synapse with zero permanence encountered\"\n result = False\n res = sameSynapse(syn, seg2[1:])\n if res == False:\n result = False\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef tmDiff(tm1, tm2, verbosity = 0, relaxSegmentTests =True):\n\n # First check basic parameters. If we fail here, don't continue\n if sameTMParams(tm1, tm2) == False:\n print \"Two TM's have different parameters\"\n return False\n\n result = True\n\n # Compare states at t first, they usually diverge before the structure of the\n # cells starts diverging\n\n if (tm1.activeState['t'] != tm2.activeState['t']).any():\n print 'Active states diverge', numpy.where(tm1.activeState['t'] != tm2.activeState['t'])\n result = False\n\n if (tm1.predictedState['t'] - tm2.predictedState['t']).any():\n print 'Predicted states diverge', numpy.where(tm1.predictedState['t'] != tm2.predictedState['t'])\n result = False\n\n # TODO: check confidence at T (confT)\n\n # Now check some high level learned parameters.\n if tm1.getNumSegments() != tm2.getNumSegments():\n print \"Number of segments are different\", tm1.getNumSegments(), tm2.getNumSegments()\n result = False\n\n if tm1.getNumSynapses() != tm2.getNumSynapses():\n print \"Number of synapses are different\", tm1.getNumSynapses(), tm2.getNumSynapses()\n tm1.printCells()\n tm2.printCells()\n result = False\n\n # Check that each cell has the same number of segments and synapses\n for c in xrange(tm1.numberOfCols):\n for i in xrange(tm2.cellsPerColumn):\n if tm1.getNumSegmentsInCell(c, i) != tm2.getNumSegmentsInCell(c, i):\n print \"Num segments different in cell:\",c,i,\n print tm1.getNumSegmentsInCell(c, i), tm2.getNumSegmentsInCell(c, i)\n result = False\n\n # If the above tests pass, then check each segment and report differences\n # Note that segments in tm1 can be in a different order than tm2. Here we\n # make sure that, for each segment in tm1, there is an identical segment\n # in tm2.\n if result == True and not relaxSegmentTests:\n for c in xrange(tm1.numberOfCols):\n for i in xrange(tm2.cellsPerColumn):\n nSegs = tm1.getNumSegmentsInCell(c, i)\n for segIdx in xrange(nSegs):\n tm1seg = tm1.getSegmentOnCell(c, i, segIdx)\n\n # Loop through all segments in tm2seg and see if any of them match tm1seg\n res = False\n for tm2segIdx in xrange(nSegs):\n tm2seg = tm2.getSegmentOnCell(c, i, tm2segIdx)\n if sameSegment(tm1seg, tm2seg) == True:\n res = True\n break\n if res == False:\n print \"\\nSegments are different for cell:\",c,i\n if verbosity >= 1:\n print \"C++\"\n tm1.printCell(c, i)\n print \"Py\"\n tm2.printCell(c, i)\n result = False\n\n if result == True and (verbosity > 1):\n print \"TM's match\"\n\n return result", "response": "This function compares two TM instances and returns a list of difference between them."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn True if tm1 - tm2 is a difference between tm1 and tm2.", "response": "def tmDiff2(tm1, tm2, verbosity = 0, relaxSegmentTests =True,\n checkLearn = True, checkStates = True):\n \"\"\"\n Given two TM instances, list the difference between them and returns False\n if there is a difference. This function checks the major parameters. If this\n passes (and checkLearn is true) it checks the number of segments on each cell.\n If this passes, checks each synapse on each segment.\n When comparing C++ and Py, the segments are usually in different orders in the\n cells. tmDiff ignores segment order when comparing TM's.\n\n If checkLearn is True, will check learn states as well as all the segments\n\n If checkStates is True, will check the various state arrays\n\n \"\"\"\n\n # First check basic parameters. If we fail here, don't continue\n if sameTMParams(tm1, tm2) == False:\n print \"Two TM's have different parameters\"\n return False\n\n tm1Label = \"\" % tm1.__class__.__name__\n tm2Label = \"\" % tm2.__class__.__name__\n\n result = True\n\n if checkStates:\n # Compare states at t first, they usually diverge before the structure of the\n # cells starts diverging\n\n if (tm1.infActiveState['t'] != tm2.infActiveState['t']).any():\n print 'Active states diverged', numpy.where(tm1.infActiveState['t'] != tm2.infActiveState['t'])\n result = False\n\n if (tm1.infPredictedState['t'] - tm2.infPredictedState['t']).any():\n print 'Predicted states diverged', numpy.where(tm1.infPredictedState['t'] != tm2.infPredictedState['t'])\n result = False\n\n if checkLearn and (tm1.lrnActiveState['t'] - tm2.lrnActiveState['t']).any():\n print 'lrnActiveState[t] diverged', numpy.where(tm1.lrnActiveState['t'] != tm2.lrnActiveState['t'])\n result = False\n\n if checkLearn and (tm1.lrnPredictedState['t'] - tm2.lrnPredictedState['t']).any():\n print 'lrnPredictedState[t] diverged', numpy.where(tm1.lrnPredictedState['t'] != tm2.lrnPredictedState['t'])\n result = False\n\n if checkLearn and abs(tm1.getAvgLearnedSeqLength() - tm2.getAvgLearnedSeqLength()) > 0.01:\n print \"Average learned sequence lengths differ: \",\n print tm1.getAvgLearnedSeqLength(), \" vs \", tm2.getAvgLearnedSeqLength()\n result = False\n\n # TODO: check confidence at T (confT)\n\n # Now check some high level learned parameters.\n if tm1.getNumSegments() != tm2.getNumSegments():\n print \"Number of segments are different\", tm1.getNumSegments(), tm2.getNumSegments()\n result = False\n\n if tm1.getNumSynapses() != tm2.getNumSynapses():\n print \"Number of synapses are different\", tm1.getNumSynapses(), tm2.getNumSynapses()\n if verbosity >= 3:\n print \"%s: \" % tm1Label,\n tm1.printCells()\n print \"\\n%s : \" % tm2Label,\n tm2.printCells()\n #result = False\n\n # Check that each cell has the same number of segments and synapses\n for c in xrange(tm1.numberOfCols):\n for i in xrange(tm2.cellsPerColumn):\n if tm1.getNumSegmentsInCell(c, i) != tm2.getNumSegmentsInCell(c, i):\n print \"Num segments different in cell:\",c,i,\n print tm1.getNumSegmentsInCell(c, i), tm2.getNumSegmentsInCell(c, i)\n result = False\n\n # If the above tests pass, then check each segment and report differences\n # Note that segments in tm1 can be in a different order than tm2. Here we\n # make sure that, for each segment in tm1, there is an identical segment\n # in tm2.\n if result == True and not relaxSegmentTests and checkLearn:\n for c in xrange(tm1.numberOfCols):\n for i in xrange(tm2.cellsPerColumn):\n nSegs = tm1.getNumSegmentsInCell(c, i)\n for segIdx in xrange(nSegs):\n tm1seg = tm1.getSegmentOnCell(c, i, segIdx)\n\n # Loop through all segments in tm2seg and see if any of them match tm1seg\n res = False\n for tm2segIdx in xrange(nSegs):\n tm2seg = tm2.getSegmentOnCell(c, i, tm2segIdx)\n if sameSegment(tm1seg, tm2seg) == True:\n res = True\n break\n if res == False:\n print \"\\nSegments are different for cell:\",c,i\n result = False\n if verbosity >= 0:\n print \"%s : \" % tm1Label,\n tm1.printCell(c, i)\n print \"\\n%s : \" % tm2Label,\n tm2.printCell(c, i)\n\n if result == True and (verbosity > 1):\n print \"TM's match\"\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef spDiff(SP1,SP2):\n if(len(SP1._masterConnectedM)!=len(SP2._masterConnectedM)):\n print \"Connected synapse matrices are different sizes\"\n return False\n\n if(len(SP1._masterPotentialM)!=len(SP2._masterPotentialM)):\n print \"Potential synapse matrices are different sizes\"\n return False\n\n if(len(SP1._masterPermanenceM)!=len(SP2._masterPermanenceM)):\n print \"Permanence matrices are different sizes\"\n return False\n\n\n #iterate over cells\n for i in range(0,len(SP1._masterConnectedM)):\n #grab the Coincidence Matrices and compare them\n connected1 = SP1._masterConnectedM[i]\n connected2 = SP2._masterConnectedM[i]\n if(connected1!=connected2):\n print \"Connected Matrices for cell %d different\" % (i)\n return False\n #grab permanence Matrices and compare them\n permanences1 = SP1._masterPermanenceM[i];\n permanences2 = SP2._masterPermanenceM[i];\n if(permanences1!=permanences2):\n print \"Permanence Matrices for cell %d different\" % (i)\n return False\n #grab the potential connection Matrices and compare them\n potential1 = SP1._masterPotentialM[i];\n potential2 = SP2._masterPotentialM[i];\n if(potential1!=potential2):\n print \"Potential Matrices for cell %d different\" % (i)\n return False\n\n #Check firing boosts\n if(not numpy.array_equal(SP1._firingBoostFactors,SP2._firingBoostFactors)):\n print \"Firing boost factors are different between spatial poolers\"\n return False\n\n #Check duty cycles after inhibiton\n if(not numpy.array_equal(SP1._dutyCycleAfterInh,SP2._dutyCycleAfterInh)):\n print \"Duty cycles after inhibition are different between spatial poolers\"\n return False\n\n\n #Check duty cycles before inhibition\n if(not numpy.array_equal(SP1._dutyCycleBeforeInh,SP2._dutyCycleBeforeInh)):\n print \"Duty cycles before inhibition are different between spatial poolers\"\n return False\n\n\n print(\"Spatial Poolers are equivalent\")\n\n return True", "response": "This function compares two spatial pooler instances and returns True if the two spatial pooler instances are equivalent False otherwise."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef removeSeqStarts(vectors, resets, numSteps=1):\n\n # Do nothing if numSteps is 0\n if numSteps == 0:\n return vectors\n\n resetIndices = resets.nonzero()[0]\n removeRows = resetIndices\n for i in range(numSteps-1):\n removeRows = numpy.hstack((removeRows, resetIndices+i+1))\n\n return numpy.delete(vectors, removeRows, axis=0)", "response": "This function removes the first numSteps samples from the start of each sequence in the sequence."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _accumulateFrequencyCounts(values, freqCounts=None):\n\n # How big does our freqCounts vector need to be?\n values = numpy.array(values)\n numEntries = values.max() + 1\n if freqCounts is not None:\n numEntries = max(numEntries, freqCounts.size)\n\n # Where do we accumulate the results?\n if freqCounts is not None:\n if freqCounts.size != numEntries:\n newCounts = numpy.zeros(numEntries, dtype='int32')\n newCounts[0:freqCounts.size] = freqCounts\n else:\n newCounts = freqCounts\n else:\n newCounts = numpy.zeros(numEntries, dtype='int32')\n\n # Accumulate the new values\n for v in values:\n newCounts[v] += 1\n\n return newCounts", "response": "This function is used to accumulate a list of values into the frequency counts and return the updated frequency counts."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn 3 things for a vector.", "response": "def _listOfOnTimesInVec(vector):\n \"\"\"\n Returns 3 things for a vector:\n * the total on time\n * the number of runs\n * a list of the durations of each run.\n\n Parameters:\n -----------------------------------------------\n input stream: 11100000001100000000011111100000\n return value: (11, 3, [3, 2, 6])\n \"\"\"\n\n # init counters\n durations = []\n numOnTimes = 0\n totalOnTime = 0\n\n # Find where the nonzeros are\n nonzeros = numpy.array(vector).nonzero()[0]\n\n # Nothing to do if vector is empty\n if len(nonzeros) == 0:\n return (0, 0, [])\n\n # Special case of only 1 on bit\n if len(nonzeros) == 1:\n return (1, 1, [1])\n\n # Count the consecutive non-zeros\n prev = nonzeros[0]\n onTime = 1\n endIdx = nonzeros[-1]\n for idx in nonzeros[1:]:\n if idx != prev+1:\n totalOnTime += onTime\n numOnTimes += 1\n durations.append(onTime)\n onTime = 1\n else:\n onTime += 1\n prev = idx\n\n # Add in the last one\n totalOnTime += onTime\n numOnTimes += 1\n durations.append(onTime)\n\n return (totalOnTime, numOnTimes, durations)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef averageOnTimePerTimestep(vectors, numSamples=None):\n\n\n # Special case given a 1 dimensional vector: it represents a single column\n if vectors.ndim == 1:\n vectors.shape = (-1,1)\n numTimeSteps = len(vectors)\n numElements = len(vectors[0])\n\n # How many samples will we look at?\n if numSamples is not None:\n import pdb; pdb.set_trace() # Test this....\n countOn = numpy.random.randint(0, numElements, numSamples)\n vectors = vectors[:, countOn]\n\n # Fill in each non-zero of vectors with the on-time that that output was\n # on for.\n durations = numpy.zeros(vectors.shape, dtype='int32')\n for col in xrange(vectors.shape[1]):\n _fillInOnTimes(vectors[:,col], durations[:,col])\n\n # Compute the average on time for each time step\n sums = vectors.sum(axis=1)\n sums.clip(min=1, max=numpy.inf, out=sums)\n avgDurations = durations.sum(axis=1, dtype='float64') / sums\n avgOnTime = avgDurations.sum() / (avgDurations > 0).sum()\n\n # Generate the frequency counts for each duration\n freqCounts = _accumulateFrequencyCounts(avgDurations)\n return (avgOnTime, freqCounts)", "response": "Calculates the average on - time over all time steps and calculates the average on - time over all time steps."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the average on - time of all outputs.", "response": "def averageOnTime(vectors, numSamples=None):\n \"\"\"\n Returns the average on-time, averaged over all on-time runs.\n\n Parameters:\n -----------------------------------------------\n vectors: the vectors for which the onTime is calculated. Row 0\n contains the outputs from time step 0, row 1 from time step\n 1, etc.\n numSamples: the number of elements for which on-time is calculated.\n If not specified, then all elements are looked at.\n\n Returns: (scalar average on-time of all outputs,\n list containing frequency counts of each encountered on-time)\n\n\n \"\"\"\n\n # Special case given a 1 dimensional vector: it represents a single column\n if vectors.ndim == 1:\n vectors.shape = (-1,1)\n numTimeSteps = len(vectors)\n numElements = len(vectors[0])\n\n # How many samples will we look at?\n if numSamples is None:\n numSamples = numElements\n countOn = range(numElements)\n else:\n countOn = numpy.random.randint(0, numElements, numSamples)\n\n # Compute the on-times and accumulate the frequency counts of each on-time\n # encountered\n sumOfLengths = 0.0\n onTimeFreqCounts = None\n n = 0\n for i in countOn:\n (onTime, segments, durations) = _listOfOnTimesInVec(vectors[:,i])\n if onTime != 0.0:\n sumOfLengths += onTime\n n += segments\n onTimeFreqCounts = _accumulateFrequencyCounts(durations, onTimeFreqCounts)\n\n # Return the average on time of each element that was on.\n if n > 0:\n return (sumOfLengths/n, onTimeFreqCounts)\n else:\n return (0.0, onTimeFreqCounts)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plotOutputsOverTime(vectors, buVectors=None, title='On-times'):\n\n # Produce the plot\n import pylab\n pylab.ion()\n pylab.figure()\n imData = vectors.transpose()\n if buVectors is not None:\n assert(buVectors.shape == vectors.shape)\n imData = imData.copy()\n imData[buVectors.transpose().astype('bool')] = 2\n pylab.imshow(imData, aspect='auto', cmap=pylab.cm.gray_r,\n interpolation='nearest')\n\n pylab.title(title)", "response": "Generates a figure that shows each output of the sequence over time."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plotHistogram(freqCounts, title='On-Times Histogram', xLabel='On-Time'):\n\n import pylab\n pylab.ion()\n pylab.figure()\n pylab.bar(numpy.arange(len(freqCounts)) - 0.5, freqCounts)\n pylab.title(title)\n pylab.xlabel(xLabel)", "response": "Plots a histogram of the frequency counts of the on - times encountered by the user."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the stability of the active elements in a population over multiple time steps.", "response": "def populationStability(vectors, numSamples=None):\n \"\"\"\n Returns the stability for the population averaged over multiple time steps\n\n Parameters:\n -----------------------------------------------\n vectors: the vectors for which the stability is calculated\n numSamples the number of time steps where stability is counted\n\n At each time step, count the fraction of the active elements which are stable\n from the previous step\n Average all the fraction\n\n \"\"\"\n\n # ----------------------------------------------------------------------\n # Calculate the stability\n numVectors = len(vectors)\n\n if numSamples is None:\n numSamples = numVectors-1\n countOn = range(numVectors-1)\n else:\n countOn = numpy.random.randint(0, numVectors-1, numSamples)\n\n\n sigmap = 0.0\n for i in countOn:\n match = checkMatch(vectors[i], vectors[i+1], sparse=False)\n # Ignore reset vectors (all 0's)\n if match[1] != 0:\n sigmap += float(match[0])/match[1]\n\n return sigmap / numSamples"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the percent of the outputs that remain completely stable over N time steps.", "response": "def percentOutputsStableOverNTimeSteps(vectors, numSamples=None):\n \"\"\"\n Returns the percent of the outputs that remain completely stable over\n N time steps.\n\n Parameters:\n -----------------------------------------------\n vectors: the vectors for which the stability is calculated\n numSamples: the number of time steps where stability is counted\n\n For each window of numSamples, count how many outputs are active during\n the entire window.\n\n \"\"\"\n\n # ----------------------------------------------------------------------\n # Calculate the stability\n totalSamples = len(vectors)\n windowSize = numSamples\n\n # Process each window\n numWindows = 0\n pctStable = 0\n\n for wStart in range(0, totalSamples-windowSize+1):\n # Count how many elements are active for the entire time\n data = vectors[wStart:wStart+windowSize]\n outputSums = data.sum(axis=0)\n stableOutputs = (outputSums == windowSize).sum()\n\n # Accumulated\n samplePctStable = float(stableOutputs) / data[0].sum()\n print samplePctStable\n pctStable += samplePctStable\n numWindows += 1\n\n # Return percent average over all possible windows\n return float(pctStable) / numWindows"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the saturation for a continuous level. This breaks the level into multiple regions and computes the saturation level for each region. Parameters: -------------------------------------------- outputs: output of the level. If sparseForm is True, this is a list of the non-zeros. If sparseForm is False, it is the dense representation outputsShape: The shape of the outputs of the level (height, width) retval: (sat, innerSat): sat: list of the saturation levels of each non-empty region of the level (each 0 -> 1.0) innerSat: list of the saturation level of each non-empty region that is not near an edge (each 0 -> 1.0)", "response": "def computeSaturationLevels(outputs, outputsShape, sparseForm=False):\n \"\"\"\n Compute the saturation for a continuous level. This breaks the level into\n multiple regions and computes the saturation level for each region.\n\n Parameters:\n --------------------------------------------\n outputs: output of the level. If sparseForm is True, this is a list of\n the non-zeros. If sparseForm is False, it is the dense\n representation\n outputsShape: The shape of the outputs of the level (height, width)\n retval: (sat, innerSat):\n sat: list of the saturation levels of each non-empty\n region of the level (each 0 -> 1.0)\n innerSat: list of the saturation level of each non-empty region\n that is not near an edge (each 0 -> 1.0)\n\n \"\"\"\n\n # Get the outputs into a SparseBinaryMatrix\n if not sparseForm:\n outputs = outputs.reshape(outputsShape)\n spOut = SM32(outputs)\n else:\n if len(outputs) > 0:\n assert (outputs.max() < outputsShape[0] * outputsShape[1])\n spOut = SM32(1, outputsShape[0] * outputsShape[1])\n spOut.setRowFromSparse(0, outputs, [1]*len(outputs))\n spOut.reshape(outputsShape[0], outputsShape[1])\n\n # Get the activity in each local region using the nNonZerosPerBox method\n # This method takes a list of the end row indices and a list of the end\n # column indices.\n # We will use regions that are 15x15, which give us about a 1/225 (.4%) resolution\n # on saturation.\n regionSize = 15\n rows = xrange(regionSize+1, outputsShape[0]+1, regionSize)\n cols = xrange(regionSize+1, outputsShape[1]+1, regionSize)\n regionSums = spOut.nNonZerosPerBox(rows, cols)\n\n # Get all the nonzeros out - those are our saturation sums\n (locations, values) = regionSums.tolist()\n values /= float(regionSize * regionSize)\n sat = list(values)\n\n # Now, to compute which are the inner regions, we will only take the ones that\n # are surrounded by activity above, below, left and right\n innerSat = []\n locationSet = set(locations)\n for (location, value) in itertools.izip(locations, values):\n (row, col) = location\n if (row-1,col) in locationSet and (row, col-1) in locationSet \\\n and (row+1, col) in locationSet and (row, col+1) in locationSet:\n innerSat.append(value)\n\n\n return (sat, innerSat)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef checkMatch(input, prediction, sparse=True, verbosity=0):\n\n if sparse:\n activeElementsInInput = set(input)\n activeElementsInPrediction = set(prediction)\n\n else:\n activeElementsInInput = set(input.nonzero()[0])\n activeElementsInPrediction = set(prediction.nonzero()[0])\n\n totalActiveInPrediction = len(activeElementsInPrediction)\n totalActiveInInput = len(activeElementsInInput)\n\n foundInInput = len(activeElementsInPrediction.intersection(activeElementsInInput))\n missingFromInput = len(activeElementsInPrediction.difference(activeElementsInInput))\n missingFromPrediction = len(activeElementsInInput.difference(activeElementsInPrediction))\n\n if verbosity >= 1:\n print \"preds. found in input:\", foundInInput, \"out of\", totalActiveInPrediction,\n print \"; preds. missing from input:\", missingFromInput, \"out of\", \\\n totalActiveInPrediction,\n print \"; unexpected active in input:\", missingFromPrediction, \"out of\", \\\n totalActiveInInput\n\n return (foundInInput, totalActiveInInput, missingFromInput,\n totalActiveInPrediction)", "response": "Checks the input and prediction for the actual input and returns the result."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing the predictive ability of a temporal memory (TM). This routine returns a value which is the average number of time steps of prediction provided by the TM. It accepts as input the inputs, outputs, and resets provided to the TM as well as a 'minOverlapPct' used to evalulate whether or not a prediction is a good enough match to the actual input. The 'outputs' are the pooling outputs of the TM. This routine treats each output as a \"manifold\" that includes the active columns that should be present in the next N inputs. It then looks at each successive input and sees if it's active columns are within the manifold. For each output sample, it computes how many time steps it can go forward on the input before the input overlap with the manifold is less then 'minOverlapPct'. It returns the average number of time steps calculated for each output. Parameters: ----------------------------------------------- inputs: The inputs to the TM. Row 0 contains the inputs from time step 0, row 1 from time step 1, etc. resets: The reset input to the TM. Element 0 contains the reset from time step 0, element 1 from time step 1, etc. outputs: The pooling outputs from the TM. Row 0 contains the outputs from time step 0, row 1 from time step 1, etc. minOverlapPct: How much each input's columns must overlap with the pooling output's columns to be considered a valid prediction. retval: (Average number of time steps of prediction over all output samples, Average number of time steps of prediction when we aren't cut short by the end of the sequence, List containing frequency counts of each encountered prediction time)", "response": "def predictionExtent(inputs, resets, outputs, minOverlapPct=100.0):\n \"\"\"\n Computes the predictive ability of a temporal memory (TM). This routine returns\n a value which is the average number of time steps of prediction provided\n by the TM. It accepts as input the inputs, outputs, and resets provided to\n the TM as well as a 'minOverlapPct' used to evalulate whether or not a\n prediction is a good enough match to the actual input.\n\n The 'outputs' are the pooling outputs of the TM. This routine treats each output\n as a \"manifold\" that includes the active columns that should be present in the\n next N inputs. It then looks at each successive input and sees if it's active\n columns are within the manifold. For each output sample, it computes how\n many time steps it can go forward on the input before the input overlap with\n the manifold is less then 'minOverlapPct'. It returns the average number of\n time steps calculated for each output.\n\n Parameters:\n -----------------------------------------------\n inputs: The inputs to the TM. Row 0 contains the inputs from time\n step 0, row 1 from time step 1, etc.\n resets: The reset input to the TM. Element 0 contains the reset from\n time step 0, element 1 from time step 1, etc.\n outputs: The pooling outputs from the TM. Row 0 contains the outputs\n from time step 0, row 1 from time step 1, etc.\n minOverlapPct: How much each input's columns must overlap with the pooling\n output's columns to be considered a valid prediction.\n\n retval: (Average number of time steps of prediction over all output\n samples,\n Average number of time steps of prediction when we aren't\n cut short by the end of the sequence,\n List containing frequency counts of each encountered\n prediction time)\n\n \"\"\"\n\n # List of how many times we encountered each prediction amount. Element 0\n # is how many times we successfully predicted 0 steps in advance, element 1\n # is how many times we predicted 1 step in advance, etc.\n predCounts = None\n\n # Total steps of prediction over all samples\n predTotal = 0\n\n # Total number of samples\n nSamples = len(outputs)\n\n # Total steps of prediction for samples at the start of the sequence, or\n # for samples whose prediction runs aren't cut short by the end of the\n # sequence.\n predTotalNotLimited = 0\n nSamplesNotLimited = 0\n\n # Compute how many cells/column we have\n nCols = len(inputs[0])\n nCellsPerCol = len(outputs[0]) // nCols\n\n # Evalulate prediction for each output sample\n for idx in xrange(nSamples):\n\n # What are the active columns for this output?\n activeCols = outputs[idx].reshape(nCols, nCellsPerCol).max(axis=1)\n\n # How many steps of prediction do we have?\n steps = 0\n while (idx+steps+1 < nSamples) and (resets[idx+steps+1] == 0):\n overlap = numpy.logical_and(inputs[idx+steps+1], activeCols)\n overlapPct = 100.0 * float(overlap.sum()) / inputs[idx+steps+1].sum()\n if overlapPct >= minOverlapPct:\n steps += 1\n else:\n break\n\n # print \"idx:\", idx, \"steps:\", steps\n # Accumulate into our total\n predCounts = _accumulateFrequencyCounts([steps], predCounts)\n predTotal += steps\n\n # If this sample was not cut short by the end of the sequence, include\n # it into the \"NotLimited\" runs\n if resets[idx] or \\\n ((idx+steps+1 < nSamples) and (not resets[idx+steps+1])):\n predTotalNotLimited += steps\n nSamplesNotLimited += 1\n\n # Return results\n return (float(predTotal) / nSamples,\n float(predTotalNotLimited) / nSamplesNotLimited,\n predCounts)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getCentreAndSpreadOffsets(spaceShape,\n spreadShape,\n stepSize=1):\n \"\"\"\n Generates centre offsets and spread offsets for block-mode based training\n regimes - star, cross, block.\n\n Parameters:\n -----------------------------------------------\n spaceShape: The (height, width) of the 2-D space to explore. This\n sets the number of center-points.\n spreadShape: The shape (height, width) of the area around each center-point\n to explore.\n stepSize: The step size. How big each step is, in pixels. This controls\n *both* the spacing of the center-points within the block and the\n points we explore around each center-point\n retval: (centreOffsets, spreadOffsets)\n \"\"\"\n\n\n from nupic.math.cross import cross\n # =====================================================================\n # Init data structures\n # What is the range on the X and Y offsets of the center points?\n shape = spaceShape\n # If the shape is (1,1), special case of just 1 center point\n if shape[0] == 1 and shape[1] == 1:\n centerOffsets = [(0,0)]\n else:\n xMin = -1 * (shape[1] // 2)\n xMax = xMin + shape[1] - 1\n xPositions = range(stepSize * xMin, stepSize * xMax + 1, stepSize)\n\n yMin = -1 * (shape[0] // 2)\n yMax = yMin + shape[0] - 1\n yPositions = range(stepSize * yMin, stepSize * yMax + 1, stepSize)\n\n centerOffsets = list(cross(yPositions, xPositions))\n\n numCenterOffsets = len(centerOffsets)\n print \"centerOffsets:\", centerOffsets\n\n # What is the range on the X and Y offsets of the spread points?\n shape = spreadShape\n # If the shape is (1,1), special case of no spreading around each center\n # point\n if shape[0] == 1 and shape[1] == 1:\n spreadOffsets = [(0,0)]\n else:\n xMin = -1 * (shape[1] // 2)\n xMax = xMin + shape[1] - 1\n xPositions = range(stepSize * xMin, stepSize * xMax + 1, stepSize)\n\n yMin = -1 * (shape[0] // 2)\n yMax = yMin + shape[0] - 1\n yPositions = range(stepSize * yMin, stepSize * yMax + 1, stepSize)\n\n spreadOffsets = list(cross(yPositions, xPositions))\n\n # Put the (0,0) entry first\n spreadOffsets.remove((0,0))\n spreadOffsets.insert(0, (0,0))\n\n numSpreadOffsets = len(spreadOffsets)\n print \"spreadOffsets:\", spreadOffsets\n\n return centerOffsets, spreadOffsets", "response": "This function generates centre offsets and spread offsets for a single training block."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef makeCloneMap(columnsShape, outputCloningWidth, outputCloningHeight=-1):\n if outputCloningHeight < 0:\n outputCloningHeight = outputCloningWidth\n\n columnsHeight, columnsWidth = columnsShape\n\n numDistinctMasters = outputCloningWidth * outputCloningHeight\n\n a = numpy.empty((columnsHeight, columnsWidth), 'uint32')\n for row in xrange(columnsHeight):\n for col in xrange(columnsWidth):\n a[row, col] = (col % outputCloningWidth) + \\\n (row % outputCloningHeight) * outputCloningWidth\n\n return a, numDistinctMasters", "response": "Make a two - dimensional clone map mapping columns to clone master."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef numpyStr(array, format='%f', includeIndices=False, includeZeros=True):\n\n shape = array.shape\n assert (len(shape) <= 2)\n items = ['[']\n if len(shape) == 1:\n if includeIndices:\n format = '%d:' + format\n if includeZeros:\n rowItems = [format % (c,x) for (c,x) in enumerate(array)]\n else:\n rowItems = [format % (c,x) for (c,x) in enumerate(array) if x != 0]\n else:\n rowItems = [format % (x) for x in array]\n items.extend(rowItems)\n\n else:\n (rows, cols) = shape\n if includeIndices:\n format = '%d,%d:' + format\n\n for r in xrange(rows):\n if includeIndices:\n rowItems = [format % (r,c,x) for c,x in enumerate(array[r])]\n else:\n rowItems = [format % (x) for x in array[r]]\n if r > 0:\n items.append('')\n\n items.append('[')\n items.extend(rowItems)\n if r < rows-1:\n items.append(']\\n')\n else:\n items.append(']')\n\n\n items.append(']')\n return ' '.join(items)", "response": "Pretty print a numpy array using the given format string for each value."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate a random sample from the discrete probability distribution and returns its value and the log of the probability of sampling that value.", "response": "def sample(self, rgen):\n \"\"\"Generates a random sample from the discrete probability distribution\n and returns its value and the log of the probability of sampling that value.\n \"\"\"\n rf = rgen.uniform(0, self.sum)\n index = bisect.bisect(self.cdf, rf)\n return self.keys[index], numpy.log(self.pmf[index])"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlogging probability of distribution.", "response": "def logProbability(self, distn):\n \"\"\"Form of distribution must be an array of counts in order of self.keys.\"\"\"\n x = numpy.asarray(distn)\n n = x.sum()\n return (logFactorial(n) - numpy.sum([logFactorial(k) for k in x]) +\n numpy.sum(x * numpy.log(self.dist.pmf)))"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sample(self, rgen):\n x = rgen.poisson(self.lambdaParameter)\n return x, self.logDensity(x)", "response": "Generates a random sample from the Poisson probability distribution and\n returns its value and the log of the probability of sampling that value."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlink sensor region to other region so that it can pass it data.", "response": "def createDataOutLink(network, sensorRegionName, regionName):\n \"\"\"Link sensor region to other region so that it can pass it data.\"\"\"\n network.link(sensorRegionName, regionName, \"UniformLink\", \"\",\n srcOutput=\"dataOut\", destInput=\"bottomUpIn\")"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a feed - forward link between 2 regions.", "response": "def createFeedForwardLink(network, regionName1, regionName2):\n \"\"\"Create a feed-forward link between 2 regions: regionName1 -> regionName2\"\"\"\n network.link(regionName1, regionName2, \"UniformLink\", \"\",\n srcOutput=\"bottomUpOut\", destInput=\"bottomUpIn\")"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a reset link from a sensor region.", "response": "def createResetLink(network, sensorRegionName, regionName):\n \"\"\"Create a reset link from a sensor region: sensorRegionName -> regionName\"\"\"\n network.link(sensorRegionName, regionName, \"UniformLink\", \"\",\n srcOutput=\"resetOut\", destInput=\"resetIn\")"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating required links from a sensor region to a classifier region.", "response": "def createSensorToClassifierLinks(network, sensorRegionName,\n classifierRegionName):\n \"\"\"Create required links from a sensor region to a classifier region.\"\"\"\n network.link(sensorRegionName, classifierRegionName, \"UniformLink\", \"\",\n srcOutput=\"bucketIdxOut\", destInput=\"bucketIdxIn\")\n network.link(sensorRegionName, classifierRegionName, \"UniformLink\", \"\",\n srcOutput=\"actValueOut\", destInput=\"actValueIn\")\n network.link(sensorRegionName, classifierRegionName, \"UniformLink\", \"\",\n srcOutput=\"categoryOut\", destInput=\"categoryIn\")"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef createNetwork(dataSource):\n with open(_PARAMS_PATH, \"r\") as f:\n modelParams = yaml.safe_load(f)[\"modelParams\"]\n\n # Create a network that will hold the regions.\n network = Network()\n\n # Add a sensor region.\n network.addRegion(\"sensor\", \"py.RecordSensor\", '{}')\n\n # Set the encoder and data source of the sensor region.\n sensorRegion = network.regions[\"sensor\"].getSelf()\n sensorRegion.encoder = createEncoder(modelParams[\"sensorParams\"][\"encoders\"])\n sensorRegion.dataSource = dataSource\n\n # Make sure the SP input width matches the sensor region output width.\n modelParams[\"spParams\"][\"inputWidth\"] = sensorRegion.encoder.getWidth()\n\n # Add SP and TM regions.\n network.addRegion(\"SP\", \"py.SPRegion\", json.dumps(modelParams[\"spParams\"]))\n network.addRegion(\"TM\", \"py.TMRegion\", json.dumps(modelParams[\"tmParams\"]))\n\n # Add a classifier region.\n clName = \"py.%s\" % modelParams[\"clParams\"].pop(\"regionName\")\n network.addRegion(\"classifier\", clName, json.dumps(modelParams[\"clParams\"]))\n\n # Add all links\n createSensorToClassifierLinks(network, \"sensor\", \"classifier\")\n createDataOutLink(network, \"sensor\", \"SP\")\n createFeedForwardLink(network, \"SP\", \"TM\")\n createFeedForwardLink(network, \"TM\", \"classifier\")\n # Reset links are optional, since the sensor region does not send resets.\n createResetLink(network, \"sensor\", \"SP\")\n createResetLink(network, \"sensor\", \"TM\")\n\n # Make sure all objects are initialized.\n network.initialize()\n\n return network", "response": "Create and initialize a network."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getPredictionResults(network, clRegionName):\n classifierRegion = network.regions[clRegionName]\n actualValues = classifierRegion.getOutputData(\"actualValues\")\n probabilities = classifierRegion.getOutputData(\"probabilities\")\n steps = classifierRegion.getSelf().stepsList\n N = classifierRegion.getSelf().maxCategoryCount\n results = {step: {} for step in steps}\n for i in range(len(steps)):\n # stepProbabilities are probabilities for this prediction step only.\n stepProbabilities = probabilities[i * N:(i + 1) * N - 1]\n mostLikelyCategoryIdx = stepProbabilities.argmax()\n predictedValue = actualValues[mostLikelyCategoryIdx]\n predictionConfidence = stepProbabilities[mostLikelyCategoryIdx]\n results[steps[i]][\"predictedValue\"] = predictedValue\n results[steps[i]][\"predictionConfidence\"] = predictionConfidence\n return results", "response": "Get prediction results for all prediction steps."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nruns the Hot Gym example.", "response": "def runHotgym(numRecords):\n \"\"\"Run the Hot Gym example.\"\"\"\n\n # Create a data source for the network.\n dataSource = FileRecordStream(streamID=_INPUT_FILE_PATH)\n numRecords = min(numRecords, dataSource.getDataRowCount())\n network = createNetwork(dataSource)\n\n # Set predicted field\n network.regions[\"sensor\"].setParameter(\"predictedField\", \"consumption\")\n\n # Enable learning for all regions.\n network.regions[\"SP\"].setParameter(\"learningMode\", 1)\n network.regions[\"TM\"].setParameter(\"learningMode\", 1)\n network.regions[\"classifier\"].setParameter(\"learningMode\", 1)\n\n # Enable inference for all regions.\n network.regions[\"SP\"].setParameter(\"inferenceMode\", 1)\n network.regions[\"TM\"].setParameter(\"inferenceMode\", 1)\n network.regions[\"classifier\"].setParameter(\"inferenceMode\", 1)\n\n results = []\n N = 1 # Run the network, N iterations at a time.\n for iteration in range(0, numRecords, N):\n network.run(N)\n\n predictionResults = getPredictionResults(network, \"classifier\")\n oneStep = predictionResults[1][\"predictedValue\"]\n oneStepConfidence = predictionResults[1][\"predictionConfidence\"]\n fiveStep = predictionResults[5][\"predictedValue\"]\n fiveStepConfidence = predictionResults[5][\"predictionConfidence\"]\n\n result = (oneStep, oneStepConfidence * 100,\n fiveStep, fiveStepConfidence * 100)\n print \"1-step: {:16} ({:4.4}%)\\t 5-step: {:16} ({:4.4}%)\".format(*result)\n results.append(result)\n\n return results"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _loadDummyModelParameters(self, params):\n\n for key, value in params.iteritems():\n if type(value) == list:\n index = self.modelIndex % len(params[key])\n self._params[key] = params[key][index]\n else:\n self._params[key] = params[key]", "response": "Loads all the parameters for this dummy model."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _computModelDelay(self):\n\n # 'delay' and 'sleepModelRange' are mutually exclusive\n if self._params['delay'] is not None \\\n and self._params['sleepModelRange'] is not None:\n raise RuntimeError(\"Only one of 'delay' or \"\n \"'sleepModelRange' may be specified\")\n\n # Get the sleepModel range\n if self._sleepModelRange is not None:\n range, delay = self._sleepModelRange.split(':')\n delay = float(delay)\n range = map(int, range.split(','))\n modelIDs = self._jobsDAO.jobGetModelIDs(self._jobID)\n modelIDs.sort()\n\n range[1] = min(range[1], len(modelIDs))\n\n # If the model is in range, add the delay\n if self._modelID in modelIDs[range[0]:range[1]]:\n self._delay = delay\n else:\n self._delay = self._params['delay']", "response": "Compute the amount of time to delay the run of this model."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _getMetrics(self):\n metric = None\n if self.metrics is not None:\n metric = self.metrics(self._currentRecordIndex+1)\n elif self.metricValue is not None:\n metric = self.metricValue\n else:\n raise RuntimeError('No metrics or metric value specified for dummy model')\n\n return {self._optimizeKeyPattern:metric}", "response": "Private function that can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses. It can be overridden by subclasses."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run(self):\n\n self._logger.debug(\"Starting Dummy Model: modelID=%s;\" % (self._modelID))\n\n # =========================================================================\n # Initialize periodic activities (e.g., for model result updates)\n # =========================================================================\n periodic = self._initPeriodicActivities()\n\n self._optimizedMetricLabel = self._optimizeKeyPattern\n self._reportMetricLabels = [self._optimizeKeyPattern]\n\n # =========================================================================\n # Create our top-level loop-control iterator\n # =========================================================================\n if self._iterations >= 0:\n iterTracker = iter(xrange(self._iterations))\n else:\n iterTracker = iter(itertools.count())\n\n # =========================================================================\n # This gets set in the unit tests. It tells the worker to sys exit\n # the first N models. This is how we generate orphaned models\n doSysExit = False\n if self._sysExitModelRange is not None:\n modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)\n modelIDs = [x[0] for x in modelAndCounters]\n modelIDs.sort()\n (beg,end) = self._sysExitModelRange\n if self._modelID in modelIDs[int(beg):int(end)]:\n doSysExit = True\n\n if self._delayModelRange is not None:\n modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)\n modelIDs = [x[0] for x in modelAndCounters]\n modelIDs.sort()\n (beg,end) = self._delayModelRange\n if self._modelID in modelIDs[int(beg):int(end)]:\n time.sleep(10)\n\n # DEBUG!!!! infinite wait if we have 50 models\n #if len(modelIDs) >= 50:\n # jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['cancel'])[0]\n # while not jobCancel:\n # time.sleep(1)\n # jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['cancel'])[0]\n\n if self._errModelRange is not None:\n modelAndCounters = self._jobsDAO.modelsGetUpdateCounters(self._jobID)\n modelIDs = [x[0] for x in modelAndCounters]\n modelIDs.sort()\n (beg,end) = self._errModelRange\n if self._modelID in modelIDs[int(beg):int(end)]:\n raise RuntimeError(\"Exiting with error due to errModelRange parameter\")\n\n # =========================================================================\n # Delay, if necessary\n if self._delay is not None:\n time.sleep(self._delay)\n\n # =========================================================================\n # Run it!\n # =========================================================================\n self._currentRecordIndex = 0\n while True:\n\n # =========================================================================\n # Check if the model should be stopped\n # =========================================================================\n\n # If killed by a terminator, stop running\n if self._isKilled:\n break\n\n # If job stops or hypersearch ends, stop running\n if self._isCanceled:\n break\n\n # If model is mature, stop running ONLY IF we are not the best model\n # for the job. Otherwise, keep running so we can keep returning\n # predictions to the user\n if self._isMature:\n if not self._isBestModel:\n self._cmpReason = self._jobsDAO.CMPL_REASON_STOPPED\n break\n else:\n self._cmpReason = self._jobsDAO.CMPL_REASON_EOF\n\n # =========================================================================\n # Get the the next record, and \"write it\"\n # =========================================================================\n try:\n self._currentRecordIndex = next(iterTracker)\n except StopIteration:\n break\n\n # \"Write\" a dummy output value. This is used to test that the batched\n # writing works properly\n\n self._writePrediction(ModelResult(None, None, None, None))\n\n periodic.tick()\n\n # =========================================================================\n # Compute wait times. See if model should exit\n # =========================================================================\n\n if self.__shouldSysExit(self._currentRecordIndex):\n sys.exit(1)\n\n # Simulate computation time\n if self._busyWaitTime is not None:\n time.sleep(self._busyWaitTime)\n self.__computeWaitTime()\n\n # Asked to abort after so many iterations?\n if doSysExit:\n sys.exit(1)\n\n # Asked to raise a jobFailException?\n if self._jobFailErr:\n raise utils.JobFailException(\"E10000\",\n \"dummyModel's jobFailErr was True.\")\n\n # =========================================================================\n # Handle final operations\n # =========================================================================\n if self._doFinalize:\n if not self._makeCheckpoint:\n self._model = None\n\n # Delay finalization operation\n if self._finalDelay is not None:\n time.sleep(self._finalDelay)\n\n self._finalize()\n\n self._logger.info(\"Finished: modelID=%r \"% (self._modelID))\n\n return (self._cmpReason, None)", "response": "Runs the OPF task against the given Model instance."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates the PredictionLogger object which is an interface to write model results to a permanent storage location.", "response": "def _createPredictionLogger(self):\n \"\"\"\n Creates the model's PredictionLogger object, which is an interface to write\n model results to a permanent storage location\n \"\"\"\n\n class DummyLogger:\n def writeRecord(self, record): pass\n def writeRecords(self, records, progressCB): pass\n def close(self): pass\n\n self._predictionLogger = DummyLogger()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck to see if the model should exit based on the exitAfter dummy parameter", "response": "def __shouldSysExit(self, iteration):\n \"\"\"\n Checks to see if the model should exit based on the exitAfter dummy\n parameter\n \"\"\"\n\n if self._exitAfter is None \\\n or iteration < self._exitAfter:\n return False\n\n results = self._jobsDAO.modelsGetFieldsForJob(self._jobID, ['params'])\n\n modelIDs = [e[0] for e in results]\n modelNums = [json.loads(e[1][0])['structuredParams']['__model_num'] for e in results]\n\n sameModelNumbers = filter(lambda x: x[1] == self.modelIndex,\n zip(modelIDs, modelNums))\n\n firstModelID = min(zip(*sameModelNumbers)[0])\n\n return firstModelID == self._modelID"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getDescription(self):\n\n description = {'name':self.name, 'fields':[f.name for f in self.fields], \\\n 'numRecords by field':[f.numRecords for f in self.fields]}\n\n return description", "response": "Returns a description of the dataset"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef setSeed(self, seed):\n\n rand.seed(seed)\n np.random.seed(seed)", "response": "Sets the random seed and numpy seed"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a single field to the dataset.", "response": "def addField(self, name, fieldParams, encoderParams):\n \"\"\"Add a single field to the dataset.\n Parameters:\n -------------------------------------------------------------------\n name: The user-specified name of the field\n fieldSpec: A list of one or more dictionaries specifying parameters\n to be used for dataClass initialization. Each dict must\n contain the key 'type' that specifies a distribution for\n the values in this field\n encoderParams: Parameters for the field encoder\n \"\"\"\n\n assert fieldParams is not None and'type' in fieldParams\n\n dataClassName = fieldParams.pop('type')\n try:\n dataClass=eval(dataClassName)(fieldParams)\n\n except TypeError, e:\n print (\"#### Error in constructing %s class object. Possibly missing \"\n \"some required constructor parameters. Parameters \"\n \"that were provided are: %s\" % (dataClass, fieldParams))\n raise\n\n encoderParams['dataClass']=dataClass\n encoderParams['dataClassName']=dataClassName\n\n fieldIndex = self.defineField(name, encoderParams)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef addMultipleFields(self, fieldsInfo):\n assert all(x in field for x in ['name', 'fieldSpec', 'encoderParams'] for field \\\n in fieldsInfo)\n\n for spec in fieldsInfo:\n self.addField(spec.pop('name'), spec.pop('fieldSpec'), spec.pop('encoderParams'))", "response": "Adds multiple fields to the dataset."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef defineField(self, name, encoderParams=None):\n self.fields.append(_field(name, encoderParams))\n\n return len(self.fields)-1", "response": "Define a new field in the encoder."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef setFlag(self, index, flag):\n assert len(self.fields)>index\n self.fields[index].flag=flag", "response": "Sets flag for the field at index."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef generateRecord(self, record):\n assert(len(record)==len(self.fields))\n if record is not None:\n for x in range(len(self.fields)):\n self.fields[x].addValue(record[x])\n\n else:\n for field in self.fields:\n field.addValue(field.dataClass.getNext())", "response": "Generates a record. Each value is stored in its respective field."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating multiple records. Refer to definition for generateRecord", "response": "def generateRecords(self, records):\n \"\"\"Generate multiple records. Refer to definition for generateRecord\"\"\"\n\n if self.verbosity>0: print 'Generating', len(records), 'records...'\n for record in records:\n self.generateRecord(record)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the nth record", "response": "def getRecord(self, n=None):\n \"\"\"Returns the nth record\"\"\"\n\n if n is None:\n assert len(self.fields)>0\n n = self.fields[0].numRecords-1\n\n assert (all(field.numRecords>n for field in self.fields))\n\n record = [field.values[n] for field in self.fields]\n\n return record"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getAllRecords(self):\n values=[]\n numRecords = self.fields[0].numRecords\n assert (all(field.numRecords==numRecords for field in self.fields))\n\n for x in range(numRecords):\n values.append(self.getRecord(x))\n\n return values", "response": "Returns all the records"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nencodes a record as a sparse distributed representation.", "response": "def encodeRecord(self, record, toBeAdded=True):\n \"\"\"Encode a record as a sparse distributed representation\n Parameters:\n --------------------------------------------------------------------\n record: Record to be encoded\n toBeAdded: Whether the encodings corresponding to the record are added to\n the corresponding fields\n \"\"\"\n encoding=[self.fields[i].encodeValue(record[i], toBeAdded) for i in \\\n xrange(len(self.fields))]\n\n return encoding"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef encodeAllRecords(self, records=None, toBeAdded=True):\n if records is None:\n records = self.getAllRecords()\n if self.verbosity>0: print 'Encoding', len(records), 'records.'\n encodings = [self.encodeRecord(record, toBeAdded) for record in records]\n\n return encodings", "response": "Encodes a list of records."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a value to the field i.", "response": "def addValueToField(self, i, value=None):\n \"\"\"Add 'value' to the field i.\n Parameters:\n --------------------------------------------------------------------\n value: value to be added\n i: value is added to field i\n \"\"\"\n\n assert(len(self.fields)>i)\n if value is None:\n value = self.fields[i].dataClass.getNext()\n self.fields[i].addValue(value)\n return value\n\n else: self.fields[i].addValue(value)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds values to the field i.", "response": "def addValuesToField(self, i, numValues):\n \"\"\"Add values to the field i.\"\"\"\n\n assert(len(self.fields)>i)\n values = [self.addValueToField(i) for n in range(numValues)]\n return values"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the sdr for the value at column i", "response": "def getSDRforValue(self, i, j):\n \"\"\"Returns the sdr for jth value at column i\"\"\"\n assert len(self.fields)>i\n assert self.fields[i].numRecords>j\n encoding = self.fields[i].encodings[j]\n\n return encoding"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the nth encoding with the predictedField zeroed out", "response": "def getZeroedOutEncoding(self, n):\n \"\"\"Returns the nth encoding with the predictedField zeroed out\"\"\"\n\n assert all(field.numRecords>n for field in self.fields)\n\n encoding = np.concatenate([field.encoder.encode(SENTINEL_VALUE_FOR_MISSING_DATA)\\\n if field.isPredictedField else field.encodings[n] for field in self.fields])\n\n return encoding"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getTotaln(self):\n\n n = sum([field.n for field in self.fields])\n return n", "response": "Returns the cumulative n for all the fields in the dataset"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getTotalw(self):\n\n w = sum([field.w for field in self.fields])\n return w", "response": "Returns the cumulative w for all the fields in the dataset"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the nth encoding", "response": "def getEncoding(self, n):\n \"\"\"Returns the nth encoding\"\"\"\n\n assert (all(field.numEncodings>n for field in self.fields))\n encoding = np.concatenate([field.encodings[n] for field in self.fields])\n\n return encoding"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn all the encodings for all the records", "response": "def getAllEncodings(self):\n \"\"\"Returns encodings for all the records\"\"\"\n\n numEncodings=self.fields[0].numEncodings\n assert (all(field.numEncodings==numEncodings for field in self.fields))\n encodings = [self.getEncoding(index) for index in range(numEncodings)]\n\n return encodings"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef saveRecords(self, path='myOutput'):\n numRecords = self.fields[0].numRecords\n assert (all(field.numRecords==numRecords for field in self.fields))\n\n import csv\n with open(path+'.csv', 'wb') as f:\n writer = csv.writer(f)\n writer.writerow(self.getAllFieldNames())\n writer.writerow(self.getAllDataTypes())\n writer.writerow(self.getAllFlags())\n writer.writerows(self.getAllRecords())\n if self.verbosity>0:\n print '******', numRecords,'records exported in numenta format to file:',\\\n path,'******\\n'", "response": "Export all the records into a csv file in numenta format."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef removeAllRecords(self):\n\n for field in self.fields:\n field.encodings, field.values=[], []\n field.numRecords, field.numEncodings= (0, 0)", "response": "Deletes all the records in the dataset"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef encodeValue(self, value, toBeAdded=True):\n\n encodedValue = np.array(self.encoder.encode(value), dtype=realDType)\n\n if toBeAdded:\n self.encodings.append(encodedValue)\n self.numEncodings+=1\n\n return encodedValue", "response": "Encode a value using the encoder parameters of the Field"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting up the dataTypes and initialize encoders", "response": "def _setTypes(self, encoderSpec):\n \"\"\"Set up the dataTypes and initialize encoders\"\"\"\n\n if self.encoderType is None:\n if self.dataType in ['int','float']:\n self.encoderType='adaptiveScalar'\n elif self.dataType=='string':\n self.encoderType='category'\n elif self.dataType in ['date', 'datetime']:\n self.encoderType='date'\n\n if self.dataType is None:\n if self.encoderType in ['scalar','adaptiveScalar']:\n self.dataType='float'\n elif self.encoderType in ['category', 'enumeration']:\n self.dataType='string'\n elif self.encoderType in ['date', 'datetime']:\n self.dataType='datetime'"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _initializeEncoders(self, encoderSpec):\n\n #Initializing scalar encoder\n if self.encoderType in ['adaptiveScalar', 'scalar']:\n if 'minval' in encoderSpec:\n self.minval = encoderSpec.pop('minval')\n else: self.minval=None\n if 'maxval' in encoderSpec:\n self.maxval = encoderSpec.pop('maxval')\n else: self.maxval = None\n self.encoder=adaptive_scalar.AdaptiveScalarEncoder(name='AdaptiveScalarEncoder', \\\n w=self.w, n=self.n, minval=self.minval, maxval=self.maxval, periodic=False, forced=True)\n\n #Initializing category encoder\n elif self.encoderType=='category':\n self.encoder=sdr_category.SDRCategoryEncoder(name='categoryEncoder', \\\n w=self.w, n=self.n)\n\n #Initializing date encoder\n elif self.encoderType in ['date', 'datetime']:\n self.encoder=date.DateEncoder(name='dateEncoder')\n else:\n raise RuntimeError('Error in constructing class object. Either encoder type'\n 'or dataType must be specified')", "response": "Initializes the encoders for the current encoder object based on the encoderSpec."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a numpy array of scalar values for the given input.", "response": "def getScalars(self, input):\n \"\"\" See method description in base.py \"\"\"\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n return numpy.array([None])\n else:\n return numpy.array([self.categoryToIndex.get(input, 0)])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getBucketIndices(self, input):\n\n # Get the bucket index from the underlying scalar encoder\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n return [None]\n else:\n return self.encoder.getBucketIndices(self.categoryToIndex.get(input, 0))", "response": "Get the bucket indices for the given input."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsees the function description in base.py", "response": "def decode(self, encoded, parentFieldName=''):\n \"\"\" See the function description in base.py\n \"\"\"\n\n # Get the scalar values from the underlying scalar encoder\n (fieldsDict, fieldNames) = self.encoder.decode(encoded)\n if len(fieldsDict) == 0:\n return (fieldsDict, fieldNames)\n\n # Expect only 1 field\n assert(len(fieldsDict) == 1)\n\n # Get the list of categories the scalar values correspond to and\n # generate the description from the category name(s).\n (inRanges, inDesc) = fieldsDict.values()[0]\n outRanges = []\n desc = \"\"\n for (minV, maxV) in inRanges:\n minV = int(round(minV))\n maxV = int(round(maxV))\n outRanges.append((minV, maxV))\n while minV <= maxV:\n if len(desc) > 0:\n desc += \", \"\n desc += self.indexToCategory[minV]\n minV += 1\n\n # Return result\n if parentFieldName != '':\n fieldName = \"%s.%s\" % (parentFieldName, self.name)\n else:\n fieldName = self.name\n return ({fieldName: (outRanges, desc)}, [fieldName])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef closenessScores(self, expValues, actValues, fractional=True,):\n\n expValue = expValues[0]\n actValue = actValues[0]\n\n if expValue == actValue:\n closeness = 1.0\n else:\n closeness = 0.0\n\n if not fractional:\n closeness = 1.0 - closeness\n\n return numpy.array([closeness])", "response": "Returns the closeness scores of the given expValues and actValues."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a list of all bucket values in the encoder.", "response": "def getBucketValues(self):\n \"\"\" See the function description in base.py \"\"\"\n\n if self._bucketValues is None:\n numBuckets = len(self.encoder.getBucketValues())\n self._bucketValues = []\n for bucketIndex in range(numBuckets):\n self._bucketValues.append(self.getBucketInfo([bucketIndex])[0].value)\n\n return self._bucketValues"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns encoderResult for the given buckets.", "response": "def getBucketInfo(self, buckets):\n \"\"\" See the function description in base.py\n \"\"\"\n\n # For the category encoder, the bucket index is the category index\n bucketInfo = self.encoder.getBucketInfo(buckets)[0]\n\n categoryIndex = int(round(bucketInfo.value))\n category = self.indexToCategory[categoryIndex]\n\n return [EncoderResult(value=category, scalar=categoryIndex,\n encoding=bucketInfo.encoding)]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef topDownCompute(self, encoded):\n\n encoderResult = self.encoder.topDownCompute(encoded)[0]\n value = encoderResult.value\n categoryIndex = int(round(value))\n category = self.indexToCategory[categoryIndex]\n\n return EncoderResult(value=category, scalar=categoryIndex,\n encoding=encoderResult.encoding)", "response": "Compute the encoder for the given encoded string."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef loadExperiment(path):\n if not os.path.isdir(path):\n path = os.path.dirname(path)\n descriptionPyModule = loadExperimentDescriptionScriptFromDir(path)\n expIface = getExperimentDescriptionInterfaceFromModule(descriptionPyModule)\n return expIface.getModelDescription(), expIface.getModelControl()", "response": "Loads the experiment description file from the path."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload the experiment description python script from the given experiment directory.", "response": "def loadExperimentDescriptionScriptFromDir(experimentDir):\n \"\"\" Loads the experiment description python script from the given experiment\n directory.\n\n :param experimentDir: (string) experiment directory path\n\n :returns: module of the loaded experiment description scripts\n \"\"\"\n descriptionScriptPath = os.path.join(experimentDir, \"description.py\")\n module = _loadDescriptionFile(descriptionScriptPath)\n return module"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getExperimentDescriptionInterfaceFromModule(module):\n result = module.descriptionInterface\n assert isinstance(result, exp_description_api.DescriptionIface), \\\n \"expected DescriptionIface-based instance, but got %s\" % type(result)\n\n return result", "response": "Returns the description interface from the given module."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nload a description. py file and returns it as a module.", "response": "def _loadDescriptionFile(descriptionPyPath):\n \"\"\"Loads a description file and returns it as a module.\n\n descriptionPyPath: path of description.py file to load\n \"\"\"\n global g_descriptionImportCount\n\n if not os.path.isfile(descriptionPyPath):\n raise RuntimeError((\"Experiment description file %s does not exist or \" + \\\n \"is not a file\") % (descriptionPyPath,))\n\n mod = imp.load_source(\"pf_description%d\" % g_descriptionImportCount,\n descriptionPyPath)\n g_descriptionImportCount += 1\n\n if not hasattr(mod, \"descriptionInterface\"):\n raise RuntimeError(\"Experiment description file %s does not define %s\" % \\\n (descriptionPyPath, \"descriptionInterface\"))\n\n if not isinstance(mod.descriptionInterface, exp_description_api.DescriptionIface):\n raise RuntimeError((\"Experiment description file %s defines %s but it \" + \\\n \"is not DescriptionIface-based\") % \\\n (descriptionPyPath, name))\n\n return mod"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update(self, modelID, modelParams, modelParamsHash, metricResult,\n completed, completionReason, matured, numRecords):\n \"\"\" Insert a new entry or update an existing one. If this is an update\n of an existing entry, then modelParams will be None\n\n Parameters:\n --------------------------------------------------------------------\n modelID: globally unique modelID of this model\n modelParams: params dict for this model, or None if this is just an update\n of a model that it already previously reported on.\n\n See the comments for the createModels() method for\n a description of this dict.\n\n modelParamsHash: hash of the modelParams dict, generated by the worker\n that put it into the model database.\n metricResult: value on the optimizeMetric for this model.\n May be None if we have no results yet.\n completed: True if the model has completed evaluation, False if it\n is still running (and these are online results)\n completionReason: One of the ClientJobsDAO.CMPL_REASON_XXX equates\n matured: True if this model has matured\n numRecords: Number of records that have been processed so far by this\n model.\n\n retval: Canonicalized result on the optimize metric\n \"\"\"\n # The modelParamsHash must always be provided - it can change after a\n # model is inserted into the models table if it got detected as an\n # orphan\n assert (modelParamsHash is not None)\n\n # We consider a model metricResult as \"final\" if it has completed or\n # matured. By default, assume anything that has completed has matured\n if completed:\n matured = True\n\n # Get the canonicalized optimize metric results. For this metric, lower\n # is always better\n if metricResult is not None and matured and \\\n completionReason in [ClientJobsDAO.CMPL_REASON_EOF,\n ClientJobsDAO.CMPL_REASON_STOPPED]:\n # Canonicalize the error score so that lower is better\n if self._hsObj._maximize:\n errScore = -1 * metricResult\n else:\n errScore = metricResult\n\n if errScore < self._bestResult:\n self._bestResult = errScore\n self._bestModelID = modelID\n self._hsObj.logger.info(\"New best model after %d evaluations: errScore \"\n \"%g on model %s\" % (len(self._allResults), self._bestResult,\n self._bestModelID))\n\n else:\n errScore = numpy.inf\n\n # If this model completed with an unacceptable completion reason, set the\n # errScore to infinite and essentially make this model invisible to\n # further queries\n if completed and completionReason in [ClientJobsDAO.CMPL_REASON_ORPHAN]:\n errScore = numpy.inf\n hidden = True\n else:\n hidden = False\n\n # Update our set of erred models and completed models. These are used\n # to determine if we should abort the search because of too many errors\n if completed:\n self._completedModels.add(modelID)\n self._numCompletedModels = len(self._completedModels)\n if completionReason == ClientJobsDAO.CMPL_REASON_ERROR:\n self._errModels.add(modelID)\n self._numErrModels = len(self._errModels)\n\n # Are we creating a new entry?\n wasHidden = False\n if modelID not in self._modelIDToIdx:\n assert (modelParams is not None)\n entry = dict(modelID=modelID, modelParams=modelParams,\n modelParamsHash=modelParamsHash,\n errScore=errScore, completed=completed,\n matured=matured, numRecords=numRecords, hidden=hidden)\n self._allResults.append(entry)\n entryIdx = len(self._allResults) - 1\n self._modelIDToIdx[modelID] = entryIdx\n\n self._paramsHashToIndexes[modelParamsHash] = entryIdx\n\n swarmId = modelParams['particleState']['swarmId']\n if not hidden:\n # Update the list of particles in each swarm\n if swarmId in self._swarmIdToIndexes:\n self._swarmIdToIndexes[swarmId].append(entryIdx)\n else:\n self._swarmIdToIndexes[swarmId] = [entryIdx]\n\n # Update number of particles at each generation in this swarm\n genIdx = modelParams['particleState']['genIdx']\n numPsEntry = self._swarmNumParticlesPerGeneration.get(swarmId, [0])\n while genIdx >= len(numPsEntry):\n numPsEntry.append(0)\n numPsEntry[genIdx] += 1\n self._swarmNumParticlesPerGeneration[swarmId] = numPsEntry\n\n # Replacing an existing one\n else:\n entryIdx = self._modelIDToIdx.get(modelID, None)\n assert (entryIdx is not None)\n entry = self._allResults[entryIdx]\n wasHidden = entry['hidden']\n\n # If the paramsHash changed, note that. This can happen for orphaned\n # models\n if entry['modelParamsHash'] != modelParamsHash:\n\n self._paramsHashToIndexes.pop(entry['modelParamsHash'])\n self._paramsHashToIndexes[modelParamsHash] = entryIdx\n entry['modelParamsHash'] = modelParamsHash\n\n # Get the model params, swarmId, and genIdx\n modelParams = entry['modelParams']\n swarmId = modelParams['particleState']['swarmId']\n genIdx = modelParams['particleState']['genIdx']\n\n # If this particle just became hidden, remove it from our swarm counts\n if hidden and not wasHidden:\n assert (entryIdx in self._swarmIdToIndexes[swarmId])\n self._swarmIdToIndexes[swarmId].remove(entryIdx)\n self._swarmNumParticlesPerGeneration[swarmId][genIdx] -= 1\n\n # Update the entry for the latest info\n entry['errScore'] = errScore\n entry['completed'] = completed\n entry['matured'] = matured\n entry['numRecords'] = numRecords\n entry['hidden'] = hidden\n\n # Update the particle best errScore\n particleId = modelParams['particleState']['id']\n genIdx = modelParams['particleState']['genIdx']\n if matured and not hidden:\n (oldResult, pos) = self._particleBest.get(particleId, (numpy.inf, None))\n if errScore < oldResult:\n pos = Particle.getPositionFromState(modelParams['particleState'])\n self._particleBest[particleId] = (errScore, pos)\n\n # Update the particle latest generation index\n prevGenIdx = self._particleLatestGenIdx.get(particleId, -1)\n if not hidden and genIdx > prevGenIdx:\n self._particleLatestGenIdx[particleId] = genIdx\n elif hidden and not wasHidden and genIdx == prevGenIdx:\n self._particleLatestGenIdx[particleId] = genIdx-1\n\n # Update the swarm best score\n if not hidden:\n swarmId = modelParams['particleState']['swarmId']\n if not swarmId in self._swarmBestOverall:\n self._swarmBestOverall[swarmId] = []\n\n bestScores = self._swarmBestOverall[swarmId]\n while genIdx >= len(bestScores):\n bestScores.append((None, numpy.inf))\n if errScore < bestScores[genIdx][1]:\n bestScores[genIdx] = (modelID, errScore)\n\n # Update the self._modifiedSwarmGens flags to support the\n # getMaturedSwarmGenerations() call.\n if not hidden:\n key = (swarmId, genIdx)\n if not key in self._maturedSwarmGens:\n self._modifiedSwarmGens.add(key)\n\n return errScore", "response": "Insert a new entry or update an existing entry."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getModelIDFromParamsHash(self, paramsHash):\n entryIdx = self. _paramsHashToIndexes.get(paramsHash, None)\n if entryIdx is not None:\n return self._allResults[entryIdx]['modelID']\n else:\n return None", "response": "Returns the modelID of the model with the given paramsHash or None if not found."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef numModels(self, swarmId=None, includeHidden=False):\n # Count all models\n if includeHidden:\n if swarmId is None:\n return len(self._allResults)\n\n else:\n return len(self._swarmIdToIndexes.get(swarmId, []))\n # Only count non-hidden models\n else:\n if swarmId is None:\n entries = self._allResults\n else:\n entries = [self._allResults[entryIdx]\n for entryIdx in self._swarmIdToIndexes.get(swarmId,[])]\n\n return len([entry for entry in entries if not entry['hidden']])", "response": "Return the total number of models in a specific swarm."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the model ID of the model with the best result so far and the best score on the optimize metric.", "response": "def bestModelIdAndErrScore(self, swarmId=None, genIdx=None):\n \"\"\"Return the model ID of the model with the best result so far and\n it's score on the optimize metric. If swarm is None, then it returns\n the global best, otherwise it returns the best for the given swarm\n for all generatons up to and including genIdx.\n\n Parameters:\n ---------------------------------------------------------------------\n swarmId: A string representation of the sorted list of encoders in this\n swarm. For example '__address_encoder.__gym_encoder'\n genIdx: consider the best in all generations up to and including this\n generation if not None.\n retval: (modelID, result)\n \"\"\"\n if swarmId is None:\n return (self._bestModelID, self._bestResult)\n\n else:\n if swarmId not in self._swarmBestOverall:\n return (None, numpy.inf)\n\n\n # Get the best score, considering the appropriate generations\n genScores = self._swarmBestOverall[swarmId]\n bestModelId = None\n bestScore = numpy.inf\n\n for (i, (modelId, errScore)) in enumerate(genScores):\n if genIdx is not None and i > genIdx:\n break\n if errScore < bestScore:\n bestScore = errScore\n bestModelId = modelId\n\n return (bestModelId, bestScore)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn particle info for a specific modelId.", "response": "def getParticleInfo(self, modelId):\n \"\"\"Return particle info for a specific modelId.\n\n Parameters:\n ---------------------------------------------------------------------\n modelId: which model Id\n\n retval: (particleState, modelId, errScore, completed, matured)\n \"\"\"\n entry = self._allResults[self._modelIDToIdx[modelId]]\n return (entry['modelParams']['particleState'], modelId, entry['errScore'],\n entry['completed'], entry['matured'])"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a list of particleStates modelIds errScores completed and matured states for all the models in this swarm.", "response": "def getParticleInfos(self, swarmId=None, genIdx=None, completed=None,\n matured=None, lastDescendent=False):\n \"\"\"Return a list of particleStates for all particles we know about in\n the given swarm, their model Ids, and metric results.\n\n Parameters:\n ---------------------------------------------------------------------\n swarmId: A string representation of the sorted list of encoders in this\n swarm. For example '__address_encoder.__gym_encoder'\n\n genIdx: If not None, only return particles at this specific generation\n index.\n\n completed: If not None, only return particles of the given state (either\n completed if 'completed' is True, or running if 'completed'\n is false\n\n matured: If not None, only return particles of the given state (either\n matured if 'matured' is True, or not matured if 'matured'\n is false. Note that any model which has completed is also\n considered matured.\n\n lastDescendent: If True, only return particles that are the last descendent,\n that is, the highest generation index for a given particle Id\n\n retval: (particleStates, modelIds, errScores, completed, matured)\n particleStates: list of particleStates\n modelIds: list of modelIds\n errScores: list of errScores, numpy.inf is plugged in\n if we don't have a result yet\n completed: list of completed booleans\n matured: list of matured booleans\n \"\"\"\n # The indexes of all the models in this swarm. This list excludes hidden\n # (orphaned) models.\n if swarmId is not None:\n entryIdxs = self._swarmIdToIndexes.get(swarmId, [])\n else:\n entryIdxs = range(len(self._allResults))\n if len(entryIdxs) == 0:\n return ([], [], [], [], [])\n\n # Get the particles of interest\n particleStates = []\n modelIds = []\n errScores = []\n completedFlags = []\n maturedFlags = []\n for idx in entryIdxs:\n entry = self._allResults[idx]\n\n # If this entry is hidden (i.e. it was an orphaned model), it should\n # not be in this list\n if swarmId is not None:\n assert (not entry['hidden'])\n\n # Get info on this model\n modelParams = entry['modelParams']\n isCompleted = entry['completed']\n isMatured = entry['matured']\n particleState = modelParams['particleState']\n particleGenIdx = particleState['genIdx']\n particleId = particleState['id']\n\n if genIdx is not None and particleGenIdx != genIdx:\n continue\n\n if completed is not None and (completed != isCompleted):\n continue\n\n if matured is not None and (matured != isMatured):\n continue\n\n if lastDescendent \\\n and (self._particleLatestGenIdx[particleId] != particleGenIdx):\n continue\n\n # Incorporate into return values\n particleStates.append(particleState)\n modelIds.append(entry['modelID'])\n errScores.append(entry['errScore'])\n completedFlags.append(isCompleted)\n maturedFlags.append(isMatured)\n\n\n return (particleStates, modelIds, errScores, completedFlags, maturedFlags)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getOrphanParticleInfos(self, swarmId, genIdx):\n\n entryIdxs = range(len(self._allResults))\n if len(entryIdxs) == 0:\n return ([], [], [], [], [])\n\n # Get the particles of interest\n particleStates = []\n modelIds = []\n errScores = []\n completedFlags = []\n maturedFlags = []\n for idx in entryIdxs:\n\n # Get info on this model\n entry = self._allResults[idx]\n if not entry['hidden']:\n continue\n\n modelParams = entry['modelParams']\n if modelParams['particleState']['swarmId'] != swarmId:\n continue\n\n isCompleted = entry['completed']\n isMatured = entry['matured']\n particleState = modelParams['particleState']\n particleGenIdx = particleState['genIdx']\n particleId = particleState['id']\n\n if genIdx is not None and particleGenIdx != genIdx:\n continue\n\n # Incorporate into return values\n particleStates.append(particleState)\n modelIds.append(entry['modelID'])\n errScores.append(entry['errScore'])\n completedFlags.append(isCompleted)\n maturedFlags.append(isMatured)\n\n return (particleStates, modelIds, errScores, completedFlags, maturedFlags)", "response": "Returns a list of particleStates modelIds errScores completed flags and matured flags for all particles in the given swarm generation."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the index of the first non - full generation in the given swarm.", "response": "def firstNonFullGeneration(self, swarmId, minNumParticles):\n \"\"\" Return the generation index of the first generation in the given\n swarm that does not have numParticles particles in it, either still in the\n running state or completed. This does not include orphaned particles.\n\n Parameters:\n ---------------------------------------------------------------------\n swarmId: A string representation of the sorted list of encoders in this\n swarm. For example '__address_encoder.__gym_encoder'\n minNumParticles: minium number of partices required for a full\n generation.\n\n retval: generation index, or None if no particles at all.\n \"\"\"\n\n if not swarmId in self._swarmNumParticlesPerGeneration:\n return None\n\n numPsPerGen = self._swarmNumParticlesPerGeneration[swarmId]\n\n numPsPerGen = numpy.array(numPsPerGen)\n firstNonFull = numpy.where(numPsPerGen < minNumParticles)[0]\n if len(firstNonFull) == 0:\n return len(numPsPerGen)\n else:\n return firstNonFull[0]"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a dict of the errors obtained on models that were run with a PermuteChoice variable.", "response": "def getResultsPerChoice(self, swarmId, maxGenIdx, varName):\n \"\"\" Return a dict of the errors obtained on models that were run with\n each value from a PermuteChoice variable.\n\n For example, if a PermuteChoice variable has the following choices:\n ['a', 'b', 'c']\n\n The dict will have 3 elements. The keys are the stringified choiceVars,\n and each value is tuple containing (choiceVar, errors) where choiceVar is\n the original form of the choiceVar (before stringification) and errors is\n the list of errors received from models that used the specific choice:\n retval:\n ['a':('a', [0.1, 0.2, 0.3]), 'b':('b', [0.5, 0.1, 0.6]), 'c':('c', [])]\n\n\n Parameters:\n ---------------------------------------------------------------------\n swarmId: swarm Id of the swarm to retrieve info from\n maxGenIdx: max generation index to consider from other models, ignored\n if None\n varName: which variable to retrieve\n\n retval: list of the errors obtained from each choice.\n \"\"\"\n results = dict()\n # Get all the completed particles in this swarm\n (allParticles, _, resultErrs, _, _) = self.getParticleInfos(swarmId,\n genIdx=None, matured=True)\n\n for particleState, resultErr in itertools.izip(allParticles, resultErrs):\n # Consider this generation?\n if maxGenIdx is not None:\n if particleState['genIdx'] > maxGenIdx:\n continue\n\n # Ignore unless this model completed successfully\n if resultErr == numpy.inf:\n continue\n\n position = Particle.getPositionFromState(particleState)\n varPosition = position[varName]\n varPositionStr = str(varPosition)\n if varPositionStr in results:\n results[varPositionStr][1].append(resultErr)\n else:\n results[varPositionStr] = (varPosition, [resultErr])\n\n return results"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _getStreamDef(self, modelDescription):\n #--------------------------------------------------------------------------\n # Generate the string containing the aggregation settings.\n aggregationPeriod = {\n 'days': 0,\n 'hours': 0,\n 'microseconds': 0,\n 'milliseconds': 0,\n 'minutes': 0,\n 'months': 0,\n 'seconds': 0,\n 'weeks': 0,\n 'years': 0,\n }\n\n # Honor any overrides provided in the stream definition\n aggFunctionsDict = {}\n if 'aggregation' in modelDescription['streamDef']:\n for key in aggregationPeriod.keys():\n if key in modelDescription['streamDef']['aggregation']:\n aggregationPeriod[key] = modelDescription['streamDef']['aggregation'][key]\n if 'fields' in modelDescription['streamDef']['aggregation']:\n for (fieldName, func) in modelDescription['streamDef']['aggregation']['fields']:\n aggFunctionsDict[fieldName] = str(func)\n\n # Do we have any aggregation at all?\n hasAggregation = False\n for v in aggregationPeriod.values():\n if v != 0:\n hasAggregation = True\n break\n\n # Convert the aggFunctionsDict to a list\n aggFunctionList = aggFunctionsDict.items()\n aggregationInfo = dict(aggregationPeriod)\n aggregationInfo['fields'] = aggFunctionList\n\n streamDef = copy.deepcopy(modelDescription['streamDef'])\n streamDef['aggregation'] = copy.deepcopy(aggregationInfo)\n return streamDef", "response": "Generate the stream definition based on the model description"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef close(self):\n if self._tempDir is not None and os.path.isdir(self._tempDir):\n self.logger.debug(\"Removing temporary directory %r\", self._tempDir)\n shutil.rmtree(self._tempDir)\n self._tempDir = None\n\n return", "response": "Deletes temporary system objects and files."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _readPermutationsFile(self, filename, modelDescription):\n # Open and execute the permutations file\n vars = {}\n\n permFile = execfile(filename, globals(), vars)\n\n\n # Read in misc info.\n self._reportKeys = vars.get('report', [])\n self._filterFunc = vars.get('permutationFilter', None)\n self._dummyModelParamsFunc = vars.get('dummyModelParams', None)\n self._predictedField = None # default\n self._predictedFieldEncoder = None # default\n self._fixedFields = None # default\n\n # The fastSwarm variable, if present, contains the params from a best\n # model from a previous swarm. If present, use info from that to seed\n # a fast swarm\n self._fastSwarmModelParams = vars.get('fastSwarmModelParams', None)\n if self._fastSwarmModelParams is not None:\n encoders = self._fastSwarmModelParams['structuredParams']['modelParams']\\\n ['sensorParams']['encoders']\n self._fixedFields = []\n for fieldName in encoders:\n if encoders[fieldName] is not None:\n self._fixedFields.append(fieldName)\n\n if 'fixedFields' in vars:\n self._fixedFields = vars['fixedFields']\n\n # Get min number of particles per swarm from either permutations file or\n # config.\n self._minParticlesPerSwarm = vars.get('minParticlesPerSwarm')\n if self._minParticlesPerSwarm == None:\n self._minParticlesPerSwarm = Configuration.get(\n 'nupic.hypersearch.minParticlesPerSwarm')\n self._minParticlesPerSwarm = int(self._minParticlesPerSwarm)\n\n # Enable logic to kill off speculative swarms when an earlier sprint\n # has found that it contains poorly performing field combination?\n self._killUselessSwarms = vars.get('killUselessSwarms', True)\n\n # The caller can request that the predicted field ALWAYS be included (\"yes\")\n # or optionally include (\"auto\"). The setting of \"no\" is N/A and ignored\n # because in that case the encoder for the predicted field will not even\n # be present in the permutations file.\n # When set to \"yes\", this will force the first sprint to try the predicted\n # field only (the legacy mode of swarming).\n # When set to \"auto\", the first sprint tries all possible fields (one at a\n # time) in the first sprint.\n self._inputPredictedField = vars.get(\"inputPredictedField\", \"yes\")\n\n # Try all possible 3-field combinations? Normally, we start with the best\n # 2-field combination as a base. When this flag is set though, we try\n # all possible 3-field combinations which takes longer but can find a\n # better model.\n self._tryAll3FieldCombinations = vars.get('tryAll3FieldCombinations', False)\n\n # Always include timestamp fields in the 3-field swarms?\n # This is a less compute intensive version of tryAll3FieldCombinations.\n # Instead of trying ALL possible 3 field combinations, it just insures\n # that the timestamp fields (dayOfWeek, timeOfDay, weekend) are never left\n # out when generating the 3-field swarms.\n self._tryAll3FieldCombinationsWTimestamps = vars.get(\n 'tryAll3FieldCombinationsWTimestamps', False)\n\n # Allow the permutations file to override minFieldContribution. This would\n # be set to a negative number for large swarms so that you don't disqualify\n # a field in an early sprint just because it did poorly there. Sometimes,\n # a field that did poorly in an early sprint could help accuracy when\n # added in a later sprint\n minFieldContribution = vars.get('minFieldContribution', None)\n if minFieldContribution is not None:\n self._minFieldContribution = minFieldContribution\n\n # Allow the permutations file to override maxBranching.\n maxBranching = vars.get('maxFieldBranching', None)\n if maxBranching is not None:\n self._maxBranching = maxBranching\n\n # Read in the optimization info.\n if 'maximize' in vars:\n self._optimizeKey = vars['maximize']\n self._maximize = True\n elif 'minimize' in vars:\n self._optimizeKey = vars['minimize']\n self._maximize = False\n else:\n raise RuntimeError(\"Permutations file '%s' does not include a maximize\"\n \" or minimize metric.\")\n\n # The permutations file is the new location for maxModels. The old location,\n # in the jobParams is deprecated.\n maxModels = vars.get('maxModels')\n if maxModels is not None:\n if self._maxModels is None:\n self._maxModels = maxModels\n else:\n raise RuntimeError('It is an error to specify maxModels both in the job'\n ' params AND in the permutations file.')\n\n\n # Figure out if what kind of search this is:\n #\n # If it's a temporal prediction search:\n # the first sprint has 1 swarm, with just the predicted field\n # elif it's a spatial prediction search:\n # the first sprint has N swarms, each with predicted field + one\n # other field.\n # elif it's a classification search:\n # the first sprint has N swarms, each with 1 field\n inferenceType = modelDescription['modelParams']['inferenceType']\n if not InferenceType.validate(inferenceType):\n raise ValueError(\"Invalid inference type %s\" %inferenceType)\n\n if inferenceType in [InferenceType.TemporalMultiStep,\n InferenceType.NontemporalMultiStep]:\n # If it does not have a separate encoder for the predicted field that\n # goes to the classifier, it is a legacy multi-step network\n classifierOnlyEncoder = None\n for encoder in modelDescription[\"modelParams\"][\"sensorParams\"]\\\n [\"encoders\"].values():\n if encoder.get(\"classifierOnly\", False) \\\n and encoder[\"fieldname\"] == vars.get('predictedField', None):\n classifierOnlyEncoder = encoder\n break\n\n if classifierOnlyEncoder is None or self._inputPredictedField==\"yes\":\n # If we don't have a separate encoder for the classifier (legacy\n # MultiStep) or the caller explicitly wants to include the predicted\n # field, then use the legacy temporal search methodology.\n self._searchType = HsSearchType.legacyTemporal\n else:\n self._searchType = HsSearchType.temporal\n\n\n elif inferenceType in [InferenceType.TemporalNextStep,\n InferenceType.TemporalAnomaly]:\n self._searchType = HsSearchType.legacyTemporal\n\n elif inferenceType in (InferenceType.TemporalClassification,\n InferenceType.NontemporalClassification):\n self._searchType = HsSearchType.classification\n\n else:\n raise RuntimeError(\"Unsupported inference type: %s\" % inferenceType)\n\n # Get the predicted field. Note that even classification experiments\n # have a \"predicted\" field - which is the field that contains the\n # classification value.\n self._predictedField = vars.get('predictedField', None)\n if self._predictedField is None:\n raise RuntimeError(\"Permutations file '%s' does not have the required\"\n \" 'predictedField' variable\" % filename)\n\n # Read in and validate the permutations dict\n if 'permutations' not in vars:\n raise RuntimeError(\"Permutations file '%s' does not define permutations\" % filename)\n\n if not isinstance(vars['permutations'], dict):\n raise RuntimeError(\"Permutations file '%s' defines a permutations variable \"\n \"but it is not a dict\")\n\n self._encoderNames = []\n self._permutations = vars['permutations']\n self._flattenedPermutations = dict()\n def _flattenPermutations(value, keys):\n if ':' in keys[-1]:\n raise RuntimeError(\"The permutation variable '%s' contains a ':' \"\n \"character, which is not allowed.\")\n flatKey = _flattenKeys(keys)\n if isinstance(value, PermuteEncoder):\n self._encoderNames.append(flatKey)\n\n # If this is the encoder for the predicted field, save its name.\n if value.fieldName == self._predictedField:\n self._predictedFieldEncoder = flatKey\n\n # Store the flattened representations of the variables within the\n # encoder.\n for encKey, encValue in value.kwArgs.iteritems():\n if isinstance(encValue, PermuteVariable):\n self._flattenedPermutations['%s:%s' % (flatKey, encKey)] = encValue\n elif isinstance(value, PermuteVariable):\n self._flattenedPermutations[flatKey] = value\n\n\n else:\n if isinstance(value, PermuteVariable):\n self._flattenedPermutations[key] = value\n rApply(self._permutations, _flattenPermutations)", "response": "Reads the permutations file and initializes the internal state of the object."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks if any orphaned models are in the results DB and if so updates them accordingly.", "response": "def _checkForOrphanedModels (self):\n \"\"\"If there are any models that haven't been updated in a while, consider\n them dead, and mark them as hidden in our resultsDB. We also change the\n paramsHash and particleHash of orphaned models so that we can\n re-generate that particle and/or model again if we desire.\n\n Parameters:\n ----------------------------------------------------------------------\n retval:\n\n \"\"\"\n\n self.logger.debug(\"Checking for orphaned models older than %s\" % \\\n (self._modelOrphanIntervalSecs))\n\n while True:\n orphanedModelId = self._cjDAO.modelAdoptNextOrphan(self._jobID,\n self._modelOrphanIntervalSecs)\n if orphanedModelId is None:\n return\n\n self.logger.info(\"Removing orphaned model: %d\" % (orphanedModelId))\n\n # Change the model hash and params hash as stored in the models table so\n # that we can insert a new model with the same paramsHash\n for attempt in range(100):\n paramsHash = hashlib.md5(\"OrphanParams.%d.%d\" % (orphanedModelId,\n attempt)).digest()\n particleHash = hashlib.md5(\"OrphanParticle.%d.%d\" % (orphanedModelId,\n attempt)).digest()\n try:\n self._cjDAO.modelSetFields(orphanedModelId,\n dict(engParamsHash=paramsHash,\n engParticleHash=particleHash))\n success = True\n except:\n success = False\n if success:\n break\n if not success:\n raise RuntimeError(\"Unexpected failure to change paramsHash and \"\n \"particleHash of orphaned model\")\n\n # Mark this model as complete, with reason \"orphaned\"\n self._cjDAO.modelSetCompleted(modelID=orphanedModelId,\n completionReason=ClientJobsDAO.CMPL_REASON_ORPHAN,\n completionMsg=\"Orphaned\")\n\n # Update our results DB immediately, rather than wait for the worker\n # to inform us. This insures that the getParticleInfos() calls we make\n # below don't include this particle. Setting the metricResult to None\n # sets it to worst case\n self._resultsDB.update(modelID=orphanedModelId,\n modelParams=None,\n modelParamsHash=paramsHash,\n metricResult=None,\n completed = True,\n completionReason = ClientJobsDAO.CMPL_REASON_ORPHAN,\n matured = True,\n numRecords = 0)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the state of the hypersearch with the current state of the current set of hypersearch swarms.", "response": "def _hsStatePeriodicUpdate(self, exhaustedSwarmId=None):\n \"\"\"\n Periodically, check to see if we should remove a certain field combination\n from evaluation (because it is doing so poorly) or move on to the next\n sprint (add in more fields).\n\n This method is called from _getCandidateParticleAndSwarm(), which is called\n right before we try and create a new model to run.\n\n Parameters:\n -----------------------------------------------------------------------\n removeSwarmId: If not None, force a change to the current set of active\n swarms by removing this swarm. This is used in situations\n where we can't find any new unique models to create in\n this swarm. In these situations, we update the hypersearch\n state regardless of the timestamp of the last time another\n worker updated it.\n\n \"\"\"\n if self._hsState is None:\n self._hsState = HsState(self)\n\n # Read in current state from the DB\n self._hsState.readStateFromDB()\n\n # This will hold the list of completed swarms that we find\n completedSwarms = set()\n\n # Mark the exhausted swarm as completing/completed, if any\n if exhaustedSwarmId is not None:\n self.logger.info(\"Removing swarm %s from the active set \"\n \"because we can't find any new unique particle \"\n \"positions\" % (exhaustedSwarmId))\n # Is it completing or completed?\n (particles, _, _, _, _) = self._resultsDB.getParticleInfos(\n swarmId=exhaustedSwarmId, matured=False)\n if len(particles) > 0:\n exhaustedSwarmStatus = 'completing'\n else:\n exhaustedSwarmStatus = 'completed'\n\n # Kill all swarms that don't need to be explored based on the most recent\n # information.\n if self._killUselessSwarms:\n self._hsState.killUselessSwarms()\n\n # For all swarms that were in the 'completing' state, see if they have\n # completed yet.\n #\n # Note that we are not quite sure why this doesn't automatically get handled\n # when we receive notification that a model finally completed in a swarm.\n # But, we ARE running into a situation, when speculativeParticles is off,\n # where we have one or more swarms in the 'completing' state even though all\n # models have since finished. This logic will serve as a failsafe against\n # this situation.\n completingSwarms = self._hsState.getCompletingSwarms()\n for swarmId in completingSwarms:\n # Is it completed?\n (particles, _, _, _, _) = self._resultsDB.getParticleInfos(\n swarmId=swarmId, matured=False)\n if len(particles) == 0:\n completedSwarms.add(swarmId)\n\n # Are there any swarms we can remove (because they have matured)?\n completedSwarmGens = self._resultsDB.getMaturedSwarmGenerations()\n priorCompletedSwarms = self._hsState.getCompletedSwarms()\n for (swarmId, genIdx, errScore) in completedSwarmGens:\n\n # Don't need to report it if the swarm already completed\n if swarmId in priorCompletedSwarms:\n continue\n\n completedList = self._swarmTerminator.recordDataPoint(\n swarmId=swarmId, generation=genIdx, errScore=errScore)\n\n # Update status message\n statusMsg = \"Completed generation #%d of swarm '%s' with a best\" \\\n \" errScore of %g\" % (genIdx, swarmId, errScore)\n if len(completedList) > 0:\n statusMsg = \"%s. Matured swarm(s): %s\" % (statusMsg, completedList)\n self.logger.info(statusMsg)\n self._cjDAO.jobSetFields (jobID=self._jobID,\n fields=dict(engStatus=statusMsg),\n useConnectionID=False,\n ignoreUnchanged=True)\n\n # Special test mode to check which swarms have terminated\n if 'NTA_TEST_recordSwarmTerminations' in os.environ:\n while True:\n resultsStr = self._cjDAO.jobGetFields(self._jobID, ['results'])[0]\n if resultsStr is None:\n results = {}\n else:\n results = json.loads(resultsStr)\n if not 'terminatedSwarms' in results:\n results['terminatedSwarms'] = {}\n\n for swarm in completedList:\n if swarm not in results['terminatedSwarms']:\n results['terminatedSwarms'][swarm] = (genIdx,\n self._swarmTerminator.swarmScores[swarm])\n\n newResultsStr = json.dumps(results)\n if newResultsStr == resultsStr:\n break\n updated = self._cjDAO.jobSetFieldIfEqual(jobID=self._jobID,\n fieldName='results',\n curValue=resultsStr,\n newValue = json.dumps(results))\n if updated:\n break\n\n if len(completedList) > 0:\n for name in completedList:\n self.logger.info(\"Swarm matured: %s. Score at generation %d: \"\n \"%s\" % (name, genIdx, errScore))\n completedSwarms = completedSwarms.union(completedList)\n\n if len(completedSwarms)==0 and (exhaustedSwarmId is None):\n return\n\n # We need to mark one or more swarms as completed, keep trying until\n # successful, or until some other worker does it for us.\n while True:\n\n if exhaustedSwarmId is not None:\n self._hsState.setSwarmState(exhaustedSwarmId, exhaustedSwarmStatus)\n\n # Mark the completed swarms as completed\n for swarmId in completedSwarms:\n self._hsState.setSwarmState(swarmId, 'completed')\n\n # If nothing changed, we're done\n if not self._hsState.isDirty():\n return\n\n # Update the shared Hypersearch state now\n # This will do nothing and return False if some other worker beat us to it\n success = self._hsState.writeStateToDB()\n\n if success:\n # Go through and cancel all models that are still running, except for\n # the best model. Once the best model changes, the one that used to be\n # best (and has matured) will notice that and stop itself at that point.\n jobResultsStr = self._cjDAO.jobGetFields(self._jobID, ['results'])[0]\n if jobResultsStr is not None:\n jobResults = json.loads(jobResultsStr)\n bestModelId = jobResults.get('bestModel', None)\n else:\n bestModelId = None\n\n for swarmId in list(completedSwarms):\n (_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(\n swarmId=swarmId, completed=False)\n if bestModelId in modelIds:\n modelIds.remove(bestModelId)\n if len(modelIds) == 0:\n continue\n self.logger.info(\"Killing the following models in swarm '%s' because\"\n \"the swarm is being terminated: %s\" % (swarmId,\n str(modelIds)))\n\n for modelId in modelIds:\n self._cjDAO.modelSetFields(modelId,\n dict(engStop=ClientJobsDAO.STOP_REASON_KILLED),\n ignoreUnchanged = True)\n return\n\n # We were not able to change the state because some other worker beat us\n # to it.\n # Get the new state, and try again to apply our changes.\n self._hsState.readStateFromDB()\n self.logger.debug(\"New hsState has been set by some other worker to: \"\n \" \\n%s\" % (pprint.pformat(self._hsState._state, indent=4)))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _getCandidateParticleAndSwarm (self, exhaustedSwarmId=None):\n # Cancel search?\n jobCancel = self._cjDAO.jobGetFields(self._jobID, ['cancel'])[0]\n if jobCancel:\n self._jobCancelled = True\n # Did a worker cancel the job because of an error?\n (workerCmpReason, workerCmpMsg) = self._cjDAO.jobGetFields(self._jobID,\n ['workerCompletionReason', 'workerCompletionMsg'])\n if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:\n self.logger.info(\"Exiting due to job being cancelled\")\n self._cjDAO.jobSetFields(self._jobID,\n dict(workerCompletionMsg=\"Job was cancelled\"),\n useConnectionID=False, ignoreUnchanged=True)\n else:\n self.logger.error(\"Exiting because some worker set the \"\n \"workerCompletionReason to %s. WorkerCompletionMsg: %s\" %\n (workerCmpReason, workerCmpMsg))\n return (True, None, None)\n\n # Perform periodic updates on the Hypersearch state.\n if self._hsState is not None:\n priorActiveSwarms = self._hsState.getActiveSwarms()\n else:\n priorActiveSwarms = None\n\n # Update the HypersearchState, checking for matured swarms, and marking\n # the passed in swarm as exhausted, if any\n self._hsStatePeriodicUpdate(exhaustedSwarmId=exhaustedSwarmId)\n\n # The above call may have modified self._hsState['activeSwarmIds']\n # Log the current set of active swarms\n activeSwarms = self._hsState.getActiveSwarms()\n if activeSwarms != priorActiveSwarms:\n self.logger.info(\"Active swarms changed to %s (from %s)\" % (activeSwarms,\n priorActiveSwarms))\n self.logger.debug(\"Active swarms: %s\" % (activeSwarms))\n\n # If too many model errors were detected, exit\n totalCmpModels = self._resultsDB.getNumCompletedModels()\n if totalCmpModels > 5:\n numErrs = self._resultsDB.getNumErrModels()\n if (float(numErrs) / totalCmpModels) > self._maxPctErrModels:\n # Get one of the errors\n errModelIds = self._resultsDB.getErrModelIds()\n resInfo = self._cjDAO.modelsGetResultAndStatus([errModelIds[0]])[0]\n modelErrMsg = resInfo.completionMsg\n cmpMsg = \"%s: Exiting due to receiving too many models failing\" \\\n \" from exceptions (%d out of %d). \\nModel Exception: %s\" % \\\n (ErrorCodes.tooManyModelErrs, numErrs, totalCmpModels,\n modelErrMsg)\n self.logger.error(cmpMsg)\n\n # Cancel the entire job now, if it has not already been cancelled\n workerCmpReason = self._cjDAO.jobGetFields(self._jobID,\n ['workerCompletionReason'])[0]\n if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:\n self._cjDAO.jobSetFields(\n self._jobID,\n fields=dict(\n cancel=True,\n workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,\n workerCompletionMsg = cmpMsg),\n useConnectionID=False,\n ignoreUnchanged=True)\n return (True, None, None)\n\n # If HsState thinks the search is over, exit. It is seeing if the results\n # on the sprint we just completed are worse than a prior sprint.\n if self._hsState.isSearchOver():\n cmpMsg = \"Exiting because results did not improve in most recently\" \\\n \" completed sprint.\"\n self.logger.info(cmpMsg)\n self._cjDAO.jobSetFields(self._jobID,\n dict(workerCompletionMsg=cmpMsg),\n useConnectionID=False, ignoreUnchanged=True)\n return (True, None, None)\n\n # Search successive active sprints, until we can find a candidate particle\n # to work with\n sprintIdx = -1\n while True:\n # Is this sprint active?\n sprintIdx += 1\n (active, eos) = self._hsState.isSprintActive(sprintIdx)\n\n # If no more sprints to explore:\n if eos:\n # If any prior ones are still being explored, finish up exploring them\n if self._hsState.anyGoodSprintsActive():\n self.logger.info(\"No more sprints to explore, waiting for prior\"\n \" sprints to complete\")\n return (False, None, None)\n\n # Else, we're done\n else:\n cmpMsg = \"Exiting because we've evaluated all possible field \" \\\n \"combinations\"\n self._cjDAO.jobSetFields(self._jobID,\n dict(workerCompletionMsg=cmpMsg),\n useConnectionID=False, ignoreUnchanged=True)\n self.logger.info(cmpMsg)\n return (True, None, None)\n\n if not active:\n if not self._speculativeParticles:\n if not self._hsState.isSprintCompleted(sprintIdx):\n self.logger.info(\"Waiting for all particles in sprint %d to complete\"\n \"before evolving any more particles\" % (sprintIdx))\n return (False, None, None)\n continue\n\n\n # ====================================================================\n # Look for swarms that have particle \"holes\" in their generations. That is,\n # an earlier generation with less than minParticlesPerSwarm. This can\n # happen if a model that was started eariler got orphaned. If we detect\n # this, start a new particle in that generation.\n swarmIds = self._hsState.getActiveSwarms(sprintIdx)\n for swarmId in swarmIds:\n firstNonFullGenIdx = self._resultsDB.firstNonFullGeneration(\n swarmId=swarmId,\n minNumParticles=self._minParticlesPerSwarm)\n if firstNonFullGenIdx is None:\n continue\n\n if firstNonFullGenIdx < self._resultsDB.highestGeneration(swarmId):\n self.logger.info(\"Cloning an earlier model in generation %d of swarm \"\n \"%s (sprintIdx=%s) to replace an orphaned model\" % (\n firstNonFullGenIdx, swarmId, sprintIdx))\n\n # Clone a random orphaned particle from the incomplete generation\n (allParticles, allModelIds, errScores, completed, matured) = \\\n self._resultsDB.getOrphanParticleInfos(swarmId, firstNonFullGenIdx)\n\n if len(allModelIds) > 0:\n # We have seen instances where we get stuck in a loop incessantly\n # trying to clone earlier models (NUP-1511). My best guess is that\n # we've already successfully cloned each of the orphaned models at\n # least once, but still need at least one more. If we don't create\n # a new particleID, we will never be able to instantiate another\n # model (since particleID hash is a unique key in the models table).\n # So, on 1/8/2013 this logic was changed to create a new particleID\n # whenever we clone an orphan.\n newParticleId = True\n self.logger.info(\"Cloning an orphaned model\")\n\n # If there is no orphan, clone one of the other particles. We can\n # have no orphan if this was a speculative generation that only\n # continued particles completed in the prior generation.\n else:\n newParticleId = True\n self.logger.info(\"No orphans found, so cloning a non-orphan\")\n (allParticles, allModelIds, errScores, completed, matured) = \\\n self._resultsDB.getParticleInfos(swarmId=swarmId,\n genIdx=firstNonFullGenIdx)\n\n # Clone that model\n modelId = random.choice(allModelIds)\n self.logger.info(\"Cloning model %r\" % (modelId))\n (particleState, _, _, _, _) = self._resultsDB.getParticleInfo(modelId)\n particle = Particle(hsObj = self,\n resultsDB = self._resultsDB,\n flattenedPermuteVars=self._flattenedPermutations,\n newFromClone=particleState,\n newParticleId=newParticleId)\n return (False, particle, swarmId)\n\n\n # ====================================================================\n # Sort the swarms in priority order, trying the ones with the least\n # number of models first\n swarmSizes = numpy.array([self._resultsDB.numModels(x) for x in swarmIds])\n swarmSizeAndIdList = zip(swarmSizes, swarmIds)\n swarmSizeAndIdList.sort()\n for (_, swarmId) in swarmSizeAndIdList:\n\n # -------------------------------------------------------------------\n # 1.) The particle will be created from new (at generation #0) if there\n # are not already self._minParticlesPerSwarm particles in the swarm.\n (allParticles, allModelIds, errScores, completed, matured) = (\n self._resultsDB.getParticleInfos(swarmId))\n if len(allParticles) < self._minParticlesPerSwarm:\n particle = Particle(hsObj=self,\n resultsDB=self._resultsDB,\n flattenedPermuteVars=self._flattenedPermutations,\n swarmId=swarmId,\n newFarFrom=allParticles)\n\n # Jam in the best encoder state found from the first sprint\n bestPriorModel = None\n if sprintIdx >= 1:\n (bestPriorModel, errScore) = self._hsState.bestModelInSprint(0)\n\n if bestPriorModel is not None:\n self.logger.info(\"Best model and errScore from previous sprint(%d):\"\n \" %s, %g\" % (0, str(bestPriorModel), errScore))\n (baseState, modelId, errScore, completed, matured) \\\n = self._resultsDB.getParticleInfo(bestPriorModel)\n particle.copyEncoderStatesFrom(baseState)\n\n # Copy the best inference type from the earlier sprint\n particle.copyVarStatesFrom(baseState, ['modelParams|inferenceType'])\n\n # It's best to jiggle the best settings from the prior sprint, so\n # compute a new position starting from that previous best\n # Only jiggle the vars we copied from the prior model\n whichVars = []\n for varName in baseState['varStates']:\n if ':' in varName:\n whichVars.append(varName)\n particle.newPosition(whichVars)\n\n self.logger.debug(\"Particle after incorporating encoder vars from best \"\n \"model in previous sprint: \\n%s\" % (str(particle)))\n\n return (False, particle, swarmId)\n\n # -------------------------------------------------------------------\n # 2.) Look for a completed particle to evolve\n # Note that we use lastDescendent. We only want to evolve particles that\n # are at their most recent generation index.\n (readyParticles, readyModelIds, readyErrScores, _, _) = (\n self._resultsDB.getParticleInfos(swarmId, genIdx=None,\n matured=True, lastDescendent=True))\n\n # If we have at least 1 ready particle to evolve...\n if len(readyParticles) > 0:\n readyGenIdxs = [x['genIdx'] for x in readyParticles]\n sortedGenIdxs = sorted(set(readyGenIdxs))\n genIdx = sortedGenIdxs[0]\n\n # Now, genIdx has the generation of the particle we want to run,\n # Get a particle from that generation and evolve it.\n useParticle = None\n for particle in readyParticles:\n if particle['genIdx'] == genIdx:\n useParticle = particle\n break\n\n # If speculativeParticles is off, we don't want to evolve a particle\n # into the next generation until all particles in the current\n # generation have completed.\n if not self._speculativeParticles:\n (particles, _, _, _, _) = self._resultsDB.getParticleInfos(\n swarmId, genIdx=genIdx, matured=False)\n if len(particles) > 0:\n continue\n\n particle = Particle(hsObj=self,\n resultsDB=self._resultsDB,\n flattenedPermuteVars=self._flattenedPermutations,\n evolveFromState=useParticle)\n return (False, particle, swarmId)\n\n # END: for (swarmSize, swarmId) in swarmSizeAndIdList:\n # No success in this swarm, onto next swarm\n\n # ====================================================================\n # We couldn't find a particle in this sprint ready to evolve. If\n # speculative particles is OFF, we have to wait for one or more other\n # workers to finish up their particles before we can do anything.\n if not self._speculativeParticles:\n self.logger.info(\"Waiting for one or more of the %s swarms \"\n \"to complete a generation before evolving any more particles\" \\\n % (str(swarmIds)))\n return (False, None, None)", "response": "Find or create a candidate particle and swarm."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _okToExit(self):\n # Send an update status periodically to the JobTracker so that it doesn't\n # think this worker is dead.\n print >> sys.stderr, \"reporter:status:In hypersearchV2: _okToExit\"\n\n # Any immature models still running?\n if not self._jobCancelled:\n (_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(matured=False)\n if len(modelIds) > 0:\n self.logger.info(\"Ready to end hyperseach, but not all models have \" \\\n \"matured yet. Sleeping a bit to wait for all models \" \\\n \"to mature.\")\n # Sleep for a bit, no need to check for orphaned models very often\n time.sleep(5.0 * random.random())\n return False\n\n # All particles have matured, send a STOP signal to any that are still\n # running.\n (_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(completed=False)\n for modelId in modelIds:\n self.logger.info(\"Stopping model %d because the search has ended\" \\\n % (modelId))\n self._cjDAO.modelSetFields(modelId,\n dict(engStop=ClientJobsDAO.STOP_REASON_STOPPED),\n ignoreUnchanged = True)\n\n # Update the HsState to get the accurate field contributions.\n self._hsStatePeriodicUpdate()\n pctFieldContributions, absFieldContributions = \\\n self._hsState.getFieldContributions()\n\n\n # Update the results field with the new field contributions.\n jobResultsStr = self._cjDAO.jobGetFields(self._jobID, ['results'])[0]\n if jobResultsStr is not None:\n jobResults = json.loads(jobResultsStr)\n else:\n jobResults = {}\n\n # Update the fieldContributions field.\n if pctFieldContributions != jobResults.get('fieldContributions', None):\n jobResults['fieldContributions'] = pctFieldContributions\n jobResults['absoluteFieldContributions'] = absFieldContributions\n\n isUpdated = self._cjDAO.jobSetFieldIfEqual(self._jobID,\n fieldName='results',\n curValue=jobResultsStr,\n newValue=json.dumps(jobResults))\n if isUpdated:\n self.logger.info('Successfully updated the field contributions:%s',\n pctFieldContributions)\n else:\n self.logger.info('Failed updating the field contributions, ' \\\n 'another hypersearch worker must have updated it')\n\n return True", "response": "Test if the worker is OK to exit this worker."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate one or more new models for evaluation.", "response": "def createModels(self, numModels=1):\n \"\"\"Create one or more new models for evaluation. These should NOT be models\n that we already know are in progress (i.e. those that have been sent to us\n via recordModelProgress). We return a list of models to the caller\n (HypersearchWorker) and if one can be successfully inserted into\n the models table (i.e. it is not a duplicate) then HypersearchWorker will\n turn around and call our runModel() method, passing in this model. If it\n is a duplicate, HypersearchWorker will call this method again. A model\n is a duplicate if either the modelParamsHash or particleHash is\n identical to another entry in the model table.\n\n The numModels is provided by HypersearchWorker as a suggestion as to how\n many models to generate. This particular implementation only ever returns 1\n model.\n\n Before choosing some new models, we first do a sweep for any models that\n may have been abandonded by failed workers. If/when we detect an abandoned\n model, we mark it as complete and orphaned and hide it from any subsequent\n queries to our ResultsDB. This effectively considers it as if it never\n existed. We also change the paramsHash and particleHash in the model record\n of the models table so that we can create another model with the same\n params and particle status and run it (which we then do immediately).\n\n The modelParamsHash returned for each model should be a hash (max allowed\n size of ClientJobsDAO.hashMaxSize) that uniquely identifies this model by\n it's params and the optional particleHash should be a hash of the particleId\n and generation index. Every model that gets placed into the models database,\n either by this worker or another worker, will have these hashes computed for\n it. The recordModelProgress gets called for every model in the database and\n the hash is used to tell which, if any, are the same as the ones this worker\n generated.\n\n NOTE: We check first ourselves for possible duplicates using the paramsHash\n before we return a model. If HypersearchWorker failed to insert it (because\n some other worker beat us to it), it will turn around and call our\n recordModelProgress with that other model so that we now know about it. It\n will then call createModels() again.\n\n This methods returns an exit boolean and the model to evaluate. If there is\n no model to evalulate, we may return False for exit because we want to stay\n alive for a while, waiting for all other models to finish. This gives us\n a chance to detect and pick up any possibly orphaned model by another\n worker.\n\n Parameters:\n ----------------------------------------------------------------------\n numModels: number of models to generate\n retval: (exit, models)\n exit: true if this worker should exit.\n models: list of tuples, one for each model. Each tuple contains:\n (modelParams, modelParamsHash, particleHash)\n\n modelParams is a dictionary containing the following elements:\n\n structuredParams: dictionary containing all variables for\n this model, with encoders represented as a dict within\n this dict (or None if they are not included.\n\n particleState: dictionary containing the state of this\n particle. This includes the position and velocity of\n each of it's variables, the particleId, and the particle\n generation index. It contains the following keys:\n\n id: The particle Id of the particle we are using to\n generate/track this model. This is a string of the\n form .\n genIdx: the particle's generation index. This starts at 0\n and increments every time we move the particle to a\n new position.\n swarmId: The swarmId, which is a string of the form\n .... that describes this swarm\n varStates: dict of the variable states. The key is the\n variable name, the value is a dict of the variable's\n position, velocity, bestPosition, bestResult, etc.\n \"\"\"\n\n # Check for and mark orphaned models\n self._checkForOrphanedModels()\n\n modelResults = []\n for _ in xrange(numModels):\n candidateParticle = None\n\n # If we've reached the max # of model to evaluate, we're done.\n if (self._maxModels is not None and\n (self._resultsDB.numModels() - self._resultsDB.getNumErrModels()) >=\n self._maxModels):\n\n return (self._okToExit(), [])\n\n # If we don't already have a particle to work on, get a candidate swarm and\n # particle to work with. If None is returned for the particle it means\n # either that the search is over (if exitNow is also True) or that we need\n # to wait for other workers to finish up their models before we can pick\n # another particle to run (if exitNow is False).\n if candidateParticle is None:\n (exitNow, candidateParticle, candidateSwarm) = (\n self._getCandidateParticleAndSwarm())\n if candidateParticle is None:\n if exitNow:\n return (self._okToExit(), [])\n else:\n # Send an update status periodically to the JobTracker so that it doesn't\n # think this worker is dead.\n print >> sys.stderr, \"reporter:status:In hypersearchV2: speculativeWait\"\n time.sleep(self._speculativeWaitSecondsMax * random.random())\n return (False, [])\n useEncoders = candidateSwarm.split('.')\n numAttempts = 0\n\n # Loop until we can create a unique model that we haven't seen yet.\n while True:\n\n # If this is the Nth attempt with the same candidate, agitate it a bit\n # to find a new unique position for it.\n if numAttempts >= 1:\n self.logger.debug(\"Agitating particle to get unique position after %d \"\n \"failed attempts in a row\" % (numAttempts))\n candidateParticle.agitate()\n\n # Create the hierarchical params expected by the base description. Note\n # that this is where we incorporate encoders that have no permuted\n # values in them.\n position = candidateParticle.getPosition()\n structuredParams = dict()\n def _buildStructuredParams(value, keys):\n flatKey = _flattenKeys(keys)\n # If it's an encoder, either put in None if it's not used, or replace\n # all permuted constructor params with the actual position.\n if flatKey in self._encoderNames:\n if flatKey in useEncoders:\n # Form encoder dict, substituting in chosen permutation values.\n return value.getDict(flatKey, position)\n # Encoder not used.\n else:\n return None\n # Regular top-level variable.\n elif flatKey in position:\n return position[flatKey]\n # Fixed override of a parameter in the base description.\n else:\n return value\n\n structuredParams = rCopy(self._permutations,\n _buildStructuredParams,\n discardNoneKeys=False)\n\n # Create the modelParams.\n modelParams = dict(\n structuredParams=structuredParams,\n particleState = candidateParticle.getState()\n )\n\n # And the hashes.\n m = hashlib.md5()\n m.update(sortedJSONDumpS(structuredParams))\n m.update(self._baseDescriptionHash)\n paramsHash = m.digest()\n\n particleInst = \"%s.%s\" % (modelParams['particleState']['id'],\n modelParams['particleState']['genIdx'])\n particleHash = hashlib.md5(particleInst).digest()\n\n # Increase attempt counter\n numAttempts += 1\n\n # If this is a new one, and passes the filter test, exit with it.\n # TODO: There is currently a problem with this filters implementation as\n # it relates to self._maxUniqueModelAttempts. When there is a filter in\n # effect, we should try a lot more times before we decide we have\n # exhausted the parameter space for this swarm. The question is, how many\n # more times?\n if self._filterFunc and not self._filterFunc(structuredParams):\n valid = False\n else:\n valid = True\n if valid and self._resultsDB.getModelIDFromParamsHash(paramsHash) is None:\n break\n\n # If we've exceeded the max allowed number of attempts, mark this swarm\n # as completing or completed, so we don't try and allocate any more new\n # particles to it, and pick another.\n if numAttempts >= self._maxUniqueModelAttempts:\n (exitNow, candidateParticle, candidateSwarm) \\\n = self._getCandidateParticleAndSwarm(\n exhaustedSwarmId=candidateSwarm)\n if candidateParticle is None:\n if exitNow:\n return (self._okToExit(), [])\n else:\n time.sleep(self._speculativeWaitSecondsMax * random.random())\n return (False, [])\n numAttempts = 0\n useEncoders = candidateSwarm.split('.')\n\n # Log message\n if self.logger.getEffectiveLevel() <= logging.DEBUG:\n self.logger.debug(\"Submitting new potential model to HypersearchWorker: \\n%s\"\n % (pprint.pformat(modelParams, indent=4)))\n modelResults.append((modelParams, paramsHash, particleHash))\n return (False, modelResults)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrecord or update the results of a model.", "response": "def recordModelProgress(self, modelID, modelParams, modelParamsHash, results,\n completed, completionReason, matured, numRecords):\n \"\"\"Record or update the results for a model. This is called by the\n HSW whenever it gets results info for another model, or updated results\n on a model that is still running.\n\n The first time this is called for a given modelID, the modelParams will\n contain the params dict for that model and the modelParamsHash will contain\n the hash of the params. Subsequent updates of the same modelID will\n have params and paramsHash values of None (in order to save overhead).\n\n The Hypersearch object should save these results into it's own working\n memory into some table, which it then uses to determine what kind of\n new models to create next time createModels() is called.\n\n Parameters:\n ----------------------------------------------------------------------\n modelID: ID of this model in models table\n modelParams: params dict for this model, or None if this is just an update\n of a model that it already previously reported on.\n\n See the comments for the createModels() method for a\n description of this dict.\n\n modelParamsHash: hash of the modelParams dict, generated by the worker\n that put it into the model database.\n results: tuple containing (allMetrics, optimizeMetric). Each is a\n dict containing metricName:result pairs. .\n May be none if we have no results yet.\n completed: True if the model has completed evaluation, False if it\n is still running (and these are online results)\n completionReason: One of the ClientJobsDAO.CMPL_REASON_XXX equates\n matured: True if this model has matured. In most cases, once a\n model matures, it will complete as well. The only time a\n model matures and does not complete is if it's currently\n the best model and we choose to keep it running to generate\n predictions.\n numRecords: Number of records that have been processed so far by this\n model.\n \"\"\"\n if results is None:\n metricResult = None\n else:\n metricResult = results[1].values()[0]\n\n # Update our database.\n errScore = self._resultsDB.update(modelID=modelID,\n modelParams=modelParams,modelParamsHash=modelParamsHash,\n metricResult=metricResult, completed=completed,\n completionReason=completionReason, matured=matured,\n numRecords=numRecords)\n\n # Log message.\n self.logger.debug('Received progress on model %d: completed: %s, '\n 'cmpReason: %s, numRecords: %d, errScore: %s' ,\n modelID, completed, completionReason, numRecords, errScore)\n\n # Log best so far.\n (bestModelID, bestResult) = self._resultsDB.bestModelIdAndErrScore()\n self.logger.debug('Best err score seen so far: %s on model %s' % \\\n (bestResult, bestModelID))"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef runModel(self, modelID, jobID, modelParams, modelParamsHash,\n jobsDAO, modelCheckpointGUID):\n \"\"\"Run the given model.\n\n This runs the model described by 'modelParams'. Periodically, it updates\n the results seen on the model to the model database using the databaseAO\n (database Access Object) methods.\n\n Parameters:\n -------------------------------------------------------------------------\n modelID: ID of this model in models table\n\n jobID: ID for this hypersearch job in the jobs table\n\n modelParams: parameters of this specific model\n modelParams is a dictionary containing the name/value\n pairs of each variable we are permuting over. Note that\n variables within an encoder spec have their name\n structure as:\n .\n\n modelParamsHash: hash of modelParamValues\n\n jobsDAO jobs data access object - the interface to the jobs\n database where model information is stored\n\n modelCheckpointGUID: A persistent, globally-unique identifier for\n constructing the model checkpoint key\n \"\"\"\n\n # We're going to make an assumption that if we're not using streams, that\n # we also don't need checkpoints saved. For now, this assumption is OK\n # (if there are no streams, we're typically running on a single machine\n # and just save models to files) but we may want to break this out as\n # a separate controllable parameter in the future\n if not self._createCheckpoints:\n modelCheckpointGUID = None\n\n # Register this model in our database\n self._resultsDB.update(modelID=modelID,\n modelParams=modelParams,\n modelParamsHash=modelParamsHash,\n metricResult = None,\n completed = False,\n completionReason = None,\n matured = False,\n numRecords = 0)\n\n # Get the structured params, which we pass to the base description\n structuredParams = modelParams['structuredParams']\n\n if self.logger.getEffectiveLevel() <= logging.DEBUG:\n self.logger.debug(\"Running Model. \\nmodelParams: %s, \\nmodelID=%s, \" % \\\n (pprint.pformat(modelParams, indent=4), modelID))\n\n # Record time.clock() so that we can report on cpu time\n cpuTimeStart = time.clock()\n\n # Run the experiment. This will report the results back to the models\n # database for us as well.\n logLevel = self.logger.getEffectiveLevel()\n try:\n if self._dummyModel is None or self._dummyModel is False:\n (cmpReason, cmpMsg) = runModelGivenBaseAndParams(\n modelID=modelID,\n jobID=jobID,\n baseDescription=self._baseDescription,\n params=structuredParams,\n predictedField=self._predictedField,\n reportKeys=self._reportKeys,\n optimizeKey=self._optimizeKey,\n jobsDAO=jobsDAO,\n modelCheckpointGUID=modelCheckpointGUID,\n logLevel=logLevel,\n predictionCacheMaxRecords=self._predictionCacheMaxRecords)\n else:\n dummyParams = dict(self._dummyModel)\n dummyParams['permutationParams'] = structuredParams\n if self._dummyModelParamsFunc is not None:\n permInfo = dict(structuredParams)\n permInfo ['generation'] = modelParams['particleState']['genIdx']\n dummyParams.update(self._dummyModelParamsFunc(permInfo))\n\n (cmpReason, cmpMsg) = runDummyModel(\n modelID=modelID,\n jobID=jobID,\n params=dummyParams,\n predictedField=self._predictedField,\n reportKeys=self._reportKeys,\n optimizeKey=self._optimizeKey,\n jobsDAO=jobsDAO,\n modelCheckpointGUID=modelCheckpointGUID,\n logLevel=logLevel,\n predictionCacheMaxRecords=self._predictionCacheMaxRecords)\n\n # Write out the completion reason and message\n jobsDAO.modelSetCompleted(modelID,\n completionReason = cmpReason,\n completionMsg = cmpMsg,\n cpuTime = time.clock() - cpuTimeStart)\n\n\n except InvalidConnectionException, e:\n self.logger.warn(\"%s\", e)", "response": "Runs the given model."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _escape(s):\n assert isinstance(s, str), \\\n \"expected %s but got %s; value=%s\" % (type(str), type(s), s)\n s = s.replace(\"\\\\\", \"\\\\\\\\\")\n s = s.replace(\"\\n\", \"\\\\n\")\n s = s.replace(\"\\t\", \"\\\\t\")\n s = s.replace(\",\", \"\\t\")\n return s", "response": "Escape commas tabs newlines and dashes in a string\n Escape commas tabs newlines and dashes in a string\n Escape commas tabs newlines and dashes in a string\n athon the same way as the string\n athon the same way"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning True if the engine services are running \"\"\" Return False if the engine services are not running \"\"\" Return True if the engine services are running \"\"\" Return False if the engine services are running \"\"\" Return True if the engine services are running", "response": "def _engineServicesRunning():\n \"\"\" Return true if the engine services are running\n \"\"\"\n process = subprocess.Popen([\"ps\", \"aux\"], stdout=subprocess.PIPE)\n\n stdout = process.communicate()[0]\n result = process.returncode\n if result != 0:\n raise RuntimeError(\"Unable to check for running client job manager\")\n\n # See if the CJM is running\n running = False\n for line in stdout.split(\"\\n\"):\n if \"python\" in line and \"clientjobmanager.client_job_manager\" in line:\n running = True\n break\n\n return running"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef runWithConfig(swarmConfig, options,\n outDir=None, outputLabel=\"default\",\n permWorkDir=None, verbosity=1):\n \"\"\"\n Starts a swarm, given an dictionary configuration.\n @param swarmConfig {dict} A complete [swarm description](http://nupic.docs.numenta.org/0.7.0.dev0/guides/swarming/running.html#the-swarm-description) object.\n @param outDir {string} Optional path to write swarm details (defaults to\n current working directory).\n @param outputLabel {string} Optional label for output (defaults to \"default\").\n @param permWorkDir {string} Optional location of working directory (defaults\n to current working directory).\n @param verbosity {int} Optional (1,2,3) increasing verbosity of output.\n\n @returns {object} Model parameters\n \"\"\"\n global g_currentVerbosityLevel\n g_currentVerbosityLevel = verbosity\n\n # Generate the description and permutations.py files in the same directory\n # for reference.\n if outDir is None:\n outDir = os.getcwd()\n if permWorkDir is None:\n permWorkDir = os.getcwd()\n\n _checkOverwrite(options, outDir)\n\n _generateExpFilesFromSwarmDescription(swarmConfig, outDir)\n\n options[\"expDescConfig\"] = swarmConfig\n options[\"outputLabel\"] = outputLabel\n options[\"outDir\"] = outDir\n options[\"permWorkDir\"] = permWorkDir\n\n runOptions = _injectDefaultOptions(options)\n _validateOptions(runOptions)\n\n return _runAction(runOptions)", "response": "Starts a swarm with a given dictionary configuration."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef runWithJsonFile(expJsonFilePath, options, outputLabel, permWorkDir):\n if \"verbosityCount\" in options:\n verbosity = options[\"verbosityCount\"]\n del options[\"verbosityCount\"]\n else:\n verbosity = 1\n\n _setupInterruptHandling()\n\n with open(expJsonFilePath, \"r\") as jsonFile:\n expJsonConfig = json.loads(jsonFile.read())\n\n outDir = os.path.dirname(expJsonFilePath)\n return runWithConfig(expJsonConfig, options, outDir=outDir,\n outputLabel=outputLabel, permWorkDir=permWorkDir,\n verbosity=verbosity)", "response": "This function is meant to be used with a CLI wrapper that takes a path to a JSON file containing the complete\n configuration and runs it with the given options."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstart a swarm, given a path to a permutations.py script. This function is meant to be used with a CLI wrapper that passes command line arguments in through the options parameter. @param permutationsFilePath {string} Path to permutations.py. @param options {dict} CLI options. @param outputLabel {string} Label for output. @param permWorkDir {string} Location of working directory. @returns {object} Model parameters.", "response": "def runWithPermutationsScript(permutationsFilePath, options,\n outputLabel, permWorkDir):\n \"\"\"\n Starts a swarm, given a path to a permutations.py script.\n\n This function is meant to be used with a CLI wrapper that passes command line\n arguments in through the options parameter.\n\n @param permutationsFilePath {string} Path to permutations.py.\n @param options {dict} CLI options.\n @param outputLabel {string} Label for output.\n @param permWorkDir {string} Location of working directory.\n\n @returns {object} Model parameters.\n \"\"\"\n global g_currentVerbosityLevel\n if \"verbosityCount\" in options:\n g_currentVerbosityLevel = options[\"verbosityCount\"]\n del options[\"verbosityCount\"]\n else:\n g_currentVerbosityLevel = 1\n\n _setupInterruptHandling()\n\n options[\"permutationsScriptPath\"] = permutationsFilePath\n options[\"outputLabel\"] = outputLabel\n options[\"outDir\"] = permWorkDir\n options[\"permWorkDir\"] = permWorkDir\n\n # Assume it's a permutations python script\n runOptions = _injectDefaultOptions(options)\n _validateOptions(runOptions)\n\n return _runAction(runOptions)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nbacking up a file in the order of the file.", "response": "def _backupFile(filePath):\n \"\"\"Back up a file\n\n Parameters:\n ----------------------------------------------------------------------\n retval: Filepath of the back-up\n \"\"\"\n assert os.path.exists(filePath)\n\n stampNum = 0\n (prefix, suffix) = os.path.splitext(filePath)\n while True:\n backupPath = \"%s.%d%s\" % (prefix, stampNum, suffix)\n stampNum += 1\n if not os.path.exists(backupPath):\n break\n shutil.copyfile(filePath, backupPath)\n\n return backupPath"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an iterator that returns ModelInfo elements for the given modelIDs.", "response": "def _iterModels(modelIDs):\n \"\"\"Creates an iterator that returns ModelInfo elements for the given modelIDs\n\n WARNING: The order of ModelInfo elements returned by the iterator\n may not match the order of the given modelIDs\n\n Parameters:\n ----------------------------------------------------------------------\n modelIDs: A sequence of model identifiers (e.g., as returned by\n _HyperSearchJob.queryModelIDs()).\n retval: Iterator that returns ModelInfo elements for the given\n modelIDs (NOTE:possibly in a different order)\n \"\"\"\n\n class ModelInfoIterator(object):\n \"\"\"ModelInfo iterator implementation class\n \"\"\"\n\n # Maximum number of ModelInfo elements to load into cache whenever\n # cache empties\n __CACHE_LIMIT = 1000\n\n debug=False\n\n\n def __init__(self, modelIDs):\n \"\"\"\n Parameters:\n ----------------------------------------------------------------------\n modelIDs: a sequence of Nupic model identifiers for which this\n iterator will return _NupicModelInfo instances.\n NOTE: The returned instances are NOT guaranteed to be in\n the same order as the IDs in modelIDs sequence.\n retval: nothing\n \"\"\"\n # Make our own copy in case caller changes model id list during iteration\n self.__modelIDs = tuple(modelIDs)\n\n if self.debug:\n _emit(Verbosity.DEBUG,\n \"MODELITERATOR: __init__; numModelIDs=%s\" % len(self.__modelIDs))\n\n self.__nextIndex = 0\n self.__modelCache = collections.deque()\n return\n\n\n def __iter__(self):\n \"\"\"Iterator Protocol function\n\n Parameters:\n ----------------------------------------------------------------------\n retval: self\n \"\"\"\n return self\n\n\n\n def next(self):\n \"\"\"Iterator Protocol function\n\n Parameters:\n ----------------------------------------------------------------------\n retval: A _NupicModelInfo instance or raises StopIteration to\n signal end of iteration.\n \"\"\"\n return self.__getNext()\n\n\n\n def __getNext(self):\n \"\"\"Implementation of the next() Iterator Protocol function.\n\n When the modelInfo cache becomes empty, queries Nupic and fills the cache\n with the next set of NupicModelInfo instances.\n\n Parameters:\n ----------------------------------------------------------------------\n retval: A _NupicModelInfo instance or raises StopIteration to\n signal end of iteration.\n \"\"\"\n\n if self.debug:\n _emit(Verbosity.DEBUG,\n \"MODELITERATOR: __getNext(); modelCacheLen=%s\" % (\n len(self.__modelCache)))\n\n if not self.__modelCache:\n self.__fillCache()\n\n if not self.__modelCache:\n raise StopIteration()\n\n return self.__modelCache.popleft()\n\n\n\n def __fillCache(self):\n \"\"\"Queries Nupic and fills an empty modelInfo cache with the next set of\n _NupicModelInfo instances\n\n Parameters:\n ----------------------------------------------------------------------\n retval: nothing\n \"\"\"\n assert (not self.__modelCache)\n\n # Assemble a list of model IDs to look up\n numModelIDs = len(self.__modelIDs) if self.__modelIDs else 0\n\n if self.__nextIndex >= numModelIDs:\n return\n\n idRange = self.__nextIndex + self.__CACHE_LIMIT\n if idRange > numModelIDs:\n idRange = numModelIDs\n\n lookupIDs = self.__modelIDs[self.__nextIndex:idRange]\n\n self.__nextIndex += (idRange - self.__nextIndex)\n\n # Query Nupic for model info of all models in the look-up list\n # NOTE: the order of results may not be the same as lookupIDs\n infoList = _clientJobsDB().modelsInfo(lookupIDs)\n assert len(infoList) == len(lookupIDs), \\\n \"modelsInfo returned %s elements; expected %s.\" % \\\n (len(infoList), len(lookupIDs))\n\n # Create _NupicModelInfo instances and add them to cache\n for rawInfo in infoList:\n modelInfo = _NupicModelInfo(rawInfo=rawInfo)\n self.__modelCache.append(modelInfo)\n\n assert len(self.__modelCache) == len(lookupIDs), \\\n \"Added %s elements to modelCache; expected %s.\" % \\\n (len(self.__modelCache), len(lookupIDs))\n\n if self.debug:\n _emit(Verbosity.DEBUG,\n \"MODELITERATOR: Leaving __fillCache(); modelCacheLen=%s\" % \\\n (len(self.__modelCache),))\n\n\n return ModelInfoIterator(modelIDs)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npick up the latest search from a saved jobID and monitor it to completion Parameters: ---------------------------------------------------------------------- retval: nothing", "response": "def pickupSearch(self):\n \"\"\"Pick up the latest search from a saved jobID and monitor it to completion\n Parameters:\n ----------------------------------------------------------------------\n retval: nothing\n \"\"\"\n self.__searchJob = self.loadSavedHyperSearchJob(\n permWorkDir=self._options[\"permWorkDir\"],\n outputLabel=self._options[\"outputLabel\"])\n\n\n self.monitorSearchJob()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef monitorSearchJob(self):\n assert self.__searchJob is not None\n\n jobID = self.__searchJob.getJobID()\n\n startTime = time.time()\n lastUpdateTime = datetime.now()\n\n # Monitor HyperSearch and report progress\n\n # NOTE: may be -1 if it can't be determined\n expectedNumModels = self.__searchJob.getExpectedNumModels(\n searchMethod = self._options[\"searchMethod\"])\n\n lastNumFinished = 0\n finishedModelIDs = set()\n\n finishedModelStats = _ModelStats()\n\n # Keep track of the worker state, results, and milestones from the job\n # record\n lastWorkerState = None\n lastJobResults = None\n lastModelMilestones = None\n lastEngStatus = None\n\n hyperSearchFinished = False\n while not hyperSearchFinished:\n jobInfo = self.__searchJob.getJobStatus(self._workers)\n\n # Check for job completion BEFORE processing models; NOTE: this permits us\n # to process any models that we may not have accounted for in the\n # previous iteration.\n hyperSearchFinished = jobInfo.isFinished()\n\n # Look for newly completed models, and process them\n modelIDs = self.__searchJob.queryModelIDs()\n _emit(Verbosity.DEBUG,\n \"Current number of models is %d (%d of them completed)\" % (\n len(modelIDs), len(finishedModelIDs)))\n\n if len(modelIDs) > 0:\n # Build a list of modelIDs to check for completion\n checkModelIDs = []\n for modelID in modelIDs:\n if modelID not in finishedModelIDs:\n checkModelIDs.append(modelID)\n\n del modelIDs\n\n # Process newly completed models\n if checkModelIDs:\n _emit(Verbosity.DEBUG,\n \"Checking %d models...\" % (len(checkModelIDs)))\n errorCompletionMsg = None\n for (i, modelInfo) in enumerate(_iterModels(checkModelIDs)):\n _emit(Verbosity.DEBUG,\n \"[%s] Checking completion: %s\" % (i, modelInfo))\n if modelInfo.isFinished():\n finishedModelIDs.add(modelInfo.getModelID())\n\n finishedModelStats.update(modelInfo)\n\n if (modelInfo.getCompletionReason().isError() and\n not errorCompletionMsg):\n errorCompletionMsg = modelInfo.getCompletionMsg()\n\n # Update the set of all encountered metrics keys (we will use\n # these to print column names in reports.csv)\n metrics = modelInfo.getReportMetrics()\n self.__foundMetrcsKeySet.update(metrics.keys())\n\n numFinished = len(finishedModelIDs)\n\n # Print current completion stats\n if numFinished != lastNumFinished:\n lastNumFinished = numFinished\n\n if expectedNumModels is None:\n expModelsStr = \"\"\n else:\n expModelsStr = \"of %s\" % (expectedNumModels)\n\n stats = finishedModelStats\n print (\" %s %s models finished [success: %s; %s: %s; %s: \"\n \"%s; %s: %s; %s: %s; %s: %s; %s: %s]\" % (\n jobID,\n numFinished,\n expModelsStr,\n #stats.numCompletedSuccess,\n (stats.numCompletedEOF+stats.numCompletedStopped),\n \"EOF\" if stats.numCompletedEOF else \"eof\",\n stats.numCompletedEOF,\n \"STOPPED\" if stats.numCompletedStopped else \"stopped\",\n stats.numCompletedStopped,\n \"KILLED\" if stats.numCompletedKilled else \"killed\",\n stats.numCompletedKilled,\n \"ERROR\" if stats.numCompletedError else \"error\",\n stats.numCompletedError,\n \"ORPHANED\" if stats.numCompletedError else \"orphaned\",\n stats.numCompletedOrphaned,\n \"UNKNOWN\" if stats.numCompletedOther else \"unknown\",\n stats.numCompletedOther))\n\n # Print the first error message from the latest batch of completed\n # models\n if errorCompletionMsg:\n print \"ERROR MESSAGE: %s\" % errorCompletionMsg\n\n # Print the new worker state, if it changed\n workerState = jobInfo.getWorkerState()\n if workerState != lastWorkerState:\n print \"##>> UPDATED WORKER STATE: \\n%s\" % (pprint.pformat(workerState,\n indent=4))\n lastWorkerState = workerState\n\n # Print the new job results, if it changed\n jobResults = jobInfo.getResults()\n if jobResults != lastJobResults:\n print \"####>> UPDATED JOB RESULTS: \\n%s (elapsed time: %g secs)\" \\\n % (pprint.pformat(jobResults, indent=4), time.time()-startTime)\n lastJobResults = jobResults\n\n # Print the new model milestones if they changed\n modelMilestones = jobInfo.getModelMilestones()\n if modelMilestones != lastModelMilestones:\n print \"##>> UPDATED MODEL MILESTONES: \\n%s\" % (\n pprint.pformat(modelMilestones, indent=4))\n lastModelMilestones = modelMilestones\n\n # Print the new engine status if it changed\n engStatus = jobInfo.getEngStatus()\n if engStatus != lastEngStatus:\n print \"##>> UPDATED STATUS: \\n%s\" % (engStatus)\n lastEngStatus = engStatus\n\n # Sleep before next check\n if not hyperSearchFinished:\n if self._options[\"timeout\"] != None:\n if ((datetime.now() - lastUpdateTime) >\n timedelta(minutes=self._options[\"timeout\"])):\n print \"Timeout reached, exiting\"\n self.__cjDAO.jobCancel(jobID)\n sys.exit(1)\n time.sleep(1)\n\n # Tabulate results\n modelIDs = self.__searchJob.queryModelIDs()\n print \"Evaluated %s models\" % len(modelIDs)\n print \"HyperSearch finished!\"\n\n jobInfo = self.__searchJob.getJobStatus(self._workers)\n print \"Worker completion message: %s\" % (jobInfo.getWorkerCompletionMsg())", "response": "Monitor the HyperSearch job and process the results and milestones."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _launchWorkers(self, cmdLine, numWorkers):\n\n self._workers = []\n for i in range(numWorkers):\n stdout = tempfile.NamedTemporaryFile(delete=False)\n stderr = tempfile.NamedTemporaryFile(delete=False)\n p = subprocess.Popen(cmdLine, bufsize=1, env=os.environ, shell=True,\n stdin=None, stdout=stdout, stderr=stderr)\n p._stderr_file = stderr\n p._stdout_file = stdout\n self._workers.append(p)", "response": "Launches the given command line for each worker in the sequence."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting a new hypersearch or runs it inline for the dryRun action.", "response": "def __startSearch(self):\n \"\"\"Starts HyperSearch as a worker or runs it inline for the \"dryRun\" action\n\n Parameters:\n ----------------------------------------------------------------------\n retval: the new _HyperSearchJob instance representing the\n HyperSearch job\n \"\"\"\n # This search uses a pre-existing permutations script\n params = _ClientJobUtils.makeSearchJobParamsDict(options=self._options,\n forRunning=True)\n\n if self._options[\"action\"] == \"dryRun\":\n args = [sys.argv[0], \"--params=%s\" % (json.dumps(params))]\n\n print\n print \"==================================================================\"\n print \"RUNNING PERMUTATIONS INLINE as \\\"DRY RUN\\\"...\"\n print \"==================================================================\"\n jobID = hypersearch_worker.main(args)\n\n else:\n cmdLine = _setUpExports(self._options[\"exports\"])\n # Begin the new search. The {JOBID} string is replaced by the actual\n # jobID returned from jobInsert.\n cmdLine += \"$HYPERSEARCH\"\n maxWorkers = self._options[\"maxWorkers\"]\n\n jobID = self.__cjDAO.jobInsert(\n client=\"GRP\",\n cmdLine=cmdLine,\n params=json.dumps(params),\n minimumWorkers=1,\n maximumWorkers=maxWorkers,\n jobType=self.__cjDAO.JOB_TYPE_HS)\n\n cmdLine = \"python -m nupic.swarming.hypersearch_worker\" \\\n \" --jobID=%d\" % (jobID)\n self._launchWorkers(cmdLine, maxWorkers)\n\n searchJob = _HyperSearchJob(jobID)\n\n # Save search ID to file (this is used for report generation)\n self.__saveHyperSearchJobID(\n permWorkDir=self._options[\"permWorkDir\"],\n outputLabel=self._options[\"outputLabel\"],\n hyperSearchJob=searchJob)\n\n if self._options[\"action\"] == \"dryRun\":\n print \"Successfully executed \\\"dry-run\\\" hypersearch, jobID=%d\" % (jobID)\n else:\n print \"Successfully submitted new HyperSearch job, jobID=%d\" % (jobID)\n _emit(Verbosity.DEBUG,\n \"Each worker executing the command line: %s\" % (cmdLine,))\n\n return searchJob"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef generateReport(cls,\n options,\n replaceReport,\n hyperSearchJob,\n metricsKeys):\n \"\"\"Prints all available results in the given HyperSearch job and emits\n model information to the permutations report csv.\n\n The job may be completed or still in progress.\n\n Parameters:\n ----------------------------------------------------------------------\n options: NupicRunPermutations options dict\n replaceReport: True to replace existing report csv, if any; False to\n append to existing report csv, if any\n hyperSearchJob: _HyperSearchJob instance; if None, will get it from saved\n jobID, if any\n metricsKeys: sequence of report metrics key names to include in report;\n if None, will pre-scan all modelInfos to generate a complete\n list of metrics key names.\n retval: model parameters\n \"\"\"\n # Load _HyperSearchJob instance from storage, if not provided\n if hyperSearchJob is None:\n hyperSearchJob = cls.loadSavedHyperSearchJob(\n permWorkDir=options[\"permWorkDir\"],\n outputLabel=options[\"outputLabel\"])\n\n modelIDs = hyperSearchJob.queryModelIDs()\n bestModel = None\n\n # If metricsKeys was not provided, pre-scan modelInfos to create the list;\n # this is needed by _ReportCSVWriter\n # Also scan the parameters to generate a list of encoders and search\n # parameters\n metricstmp = set()\n searchVar = set()\n for modelInfo in _iterModels(modelIDs):\n if modelInfo.isFinished():\n vars = modelInfo.getParamLabels().keys()\n searchVar.update(vars)\n metrics = modelInfo.getReportMetrics()\n metricstmp.update(metrics.keys())\n if metricsKeys is None:\n metricsKeys = metricstmp\n # Create a csv report writer\n reportWriter = _ReportCSVWriter(hyperSearchJob=hyperSearchJob,\n metricsKeys=metricsKeys,\n searchVar=searchVar,\n outputDirAbsPath=options[\"permWorkDir\"],\n outputLabel=options[\"outputLabel\"],\n replaceReport=replaceReport)\n\n # Tallies of experiment dispositions\n modelStats = _ModelStats()\n #numCompletedOther = long(0)\n\n print \"\\nResults from all experiments:\"\n print \"----------------------------------------------------------------\"\n\n # Get common optimization metric info from permutations script\n searchParams = hyperSearchJob.getParams()\n\n (optimizationMetricKey, maximizeMetric) = (\n _PermutationUtils.getOptimizationMetricInfo(searchParams))\n\n # Print metrics, while looking for the best model\n formatStr = None\n # NOTE: we may find additional metrics if HyperSearch is still running\n foundMetricsKeySet = set(metricsKeys)\n sortedMetricsKeys = []\n\n # pull out best Model from jobs table\n jobInfo = _clientJobsDB().jobInfo(hyperSearchJob.getJobID())\n\n # Try to return a decent error message if the job was cancelled for some\n # reason.\n if jobInfo.cancel == 1:\n raise Exception(jobInfo.workerCompletionMsg)\n\n try:\n results = json.loads(jobInfo.results)\n except Exception, e:\n print \"json.loads(jobInfo.results) raised an exception. \" \\\n \"Here is some info to help with debugging:\"\n print \"jobInfo: \", jobInfo\n print \"jobInfo.results: \", jobInfo.results\n print \"EXCEPTION: \", e\n raise\n\n bestModelNum = results[\"bestModel\"]\n bestModelIterIndex = None\n\n # performance metrics for the entire job\n totalWallTime = 0\n totalRecords = 0\n\n # At the end, we will sort the models by their score on the optimization\n # metric\n scoreModelIDDescList = []\n for (i, modelInfo) in enumerate(_iterModels(modelIDs)):\n\n # Output model info to report csv\n reportWriter.emit(modelInfo)\n\n # Update job metrics\n totalRecords+=modelInfo.getNumRecords()\n format = \"%Y-%m-%d %H:%M:%S\"\n startTime = modelInfo.getStartTime()\n if modelInfo.isFinished():\n endTime = modelInfo.getEndTime()\n st = datetime.strptime(startTime, format)\n et = datetime.strptime(endTime, format)\n totalWallTime+=(et-st).seconds\n\n # Tabulate experiment dispositions\n modelStats.update(modelInfo)\n\n # For convenience\n expDesc = modelInfo.getModelDescription()\n reportMetrics = modelInfo.getReportMetrics()\n optimizationMetrics = modelInfo.getOptimizationMetrics()\n if modelInfo.getModelID() == bestModelNum:\n bestModel = modelInfo\n bestModelIterIndex=i\n bestMetric = optimizationMetrics.values()[0]\n\n # Keep track of the best-performing model\n if optimizationMetrics:\n assert len(optimizationMetrics) == 1, (\n \"expected 1 opt key, but got %d (%s) in %s\" % (\n len(optimizationMetrics), optimizationMetrics, modelInfo))\n\n # Append to our list of modelIDs and scores\n if modelInfo.getCompletionReason().isEOF():\n scoreModelIDDescList.append((optimizationMetrics.values()[0],\n modelInfo.getModelID(),\n modelInfo.getGeneratedDescriptionFile(),\n modelInfo.getParamLabels()))\n\n print \"[%d] Experiment %s\\n(%s):\" % (i, modelInfo, expDesc)\n if (modelInfo.isFinished() and\n not (modelInfo.getCompletionReason().isStopped or\n modelInfo.getCompletionReason().isEOF())):\n print \">> COMPLETION MESSAGE: %s\" % modelInfo.getCompletionMsg()\n\n if reportMetrics:\n # Update our metrics key set and format string\n foundMetricsKeySet.update(reportMetrics.iterkeys())\n if len(sortedMetricsKeys) != len(foundMetricsKeySet):\n sortedMetricsKeys = sorted(foundMetricsKeySet)\n\n maxKeyLen = max([len(k) for k in sortedMetricsKeys])\n formatStr = \" %%-%ds\" % (maxKeyLen+2)\n\n # Print metrics\n for key in sortedMetricsKeys:\n if key in reportMetrics:\n if key == optimizationMetricKey:\n m = \"%r (*)\" % reportMetrics[key]\n else:\n m = \"%r\" % reportMetrics[key]\n print formatStr % (key+\":\"), m\n print\n\n # Summarize results\n print \"--------------------------------------------------------------\"\n if len(modelIDs) > 0:\n print \"%d experiments total (%s).\\n\" % (\n len(modelIDs),\n (\"all completed successfully\"\n if (modelStats.numCompletedKilled + modelStats.numCompletedEOF) ==\n len(modelIDs)\n else \"WARNING: %d models have not completed or there were errors\" % (\n len(modelIDs) - (\n modelStats.numCompletedKilled + modelStats.numCompletedEOF +\n modelStats.numCompletedStopped))))\n\n if modelStats.numStatusOther > 0:\n print \"ERROR: models with unexpected status: %d\" % (\n modelStats.numStatusOther)\n\n print \"WaitingToStart: %d\" % modelStats.numStatusWaitingToStart\n print \"Running: %d\" % modelStats.numStatusRunning\n print \"Completed: %d\" % modelStats.numStatusCompleted\n if modelStats.numCompletedOther > 0:\n print \" ERROR: models with unexpected completion reason: %d\" % (\n modelStats.numCompletedOther)\n print \" ran to EOF: %d\" % modelStats.numCompletedEOF\n print \" ran to stop signal: %d\" % modelStats.numCompletedStopped\n print \" were orphaned: %d\" % modelStats.numCompletedOrphaned\n print \" killed off: %d\" % modelStats.numCompletedKilled\n print \" failed: %d\" % modelStats.numCompletedError\n\n assert modelStats.numStatusOther == 0, \"numStatusOther=%s\" % (\n modelStats.numStatusOther)\n assert modelStats.numCompletedOther == 0, \"numCompletedOther=%s\" % (\n modelStats.numCompletedOther)\n\n else:\n print \"0 experiments total.\"\n\n # Print out the field contributions\n print\n global gCurrentSearch\n jobStatus = hyperSearchJob.getJobStatus(gCurrentSearch._workers)\n jobResults = jobStatus.getResults()\n if \"fieldContributions\" in jobResults:\n print \"Field Contributions:\"\n pprint.pprint(jobResults[\"fieldContributions\"], indent=4)\n else:\n print \"Field contributions info not available\"\n\n # Did we have an optimize key?\n if bestModel is not None:\n maxKeyLen = max([len(k) for k in sortedMetricsKeys])\n maxKeyLen = max(maxKeyLen, len(optimizationMetricKey))\n formatStr = \" %%-%ds\" % (maxKeyLen+2)\n bestMetricValue = bestModel.getOptimizationMetrics().values()[0]\n optimizationMetricName = bestModel.getOptimizationMetrics().keys()[0]\n print\n print \"Best results on the optimization metric %s (maximize=%s):\" % (\n optimizationMetricName, maximizeMetric)\n print \"[%d] Experiment %s (%s):\" % (\n bestModelIterIndex, bestModel, bestModel.getModelDescription())\n print formatStr % (optimizationMetricName+\":\"), bestMetricValue\n print\n print \"Total number of Records processed: %d\" % totalRecords\n print\n print \"Total wall time for all models: %d\" % totalWallTime\n\n hsJobParams = hyperSearchJob.getParams()\n\n # Were we asked to write out the top N model description files?\n if options[\"genTopNDescriptions\"] > 0:\n print \"\\nGenerating description files for top %d models...\" % (\n options[\"genTopNDescriptions\"])\n scoreModelIDDescList.sort()\n scoreModelIDDescList = scoreModelIDDescList[\n 0:options[\"genTopNDescriptions\"]]\n\n i = -1\n for (score, modelID, description, paramLabels) in scoreModelIDDescList:\n i += 1\n outDir = os.path.join(options[\"permWorkDir\"], \"model_%d\" % (i))\n print \"Generating description file for model %s at %s\" % \\\n (modelID, outDir)\n if not os.path.exists(outDir):\n os.makedirs(outDir)\n\n # Fix up the location to the base description file.\n # importBaseDescription() chooses the file relative to the calling file.\n # The calling file is in outDir.\n # The base description is in the user-specified \"outDir\"\n base_description_path = os.path.join(options[\"outDir\"],\n \"description.py\")\n base_description_relpath = os.path.relpath(base_description_path,\n start=outDir)\n description = description.replace(\n \"importBaseDescription('base.py', config)\",\n \"importBaseDescription('%s', config)\" % base_description_relpath)\n fd = open(os.path.join(outDir, \"description.py\"), \"wb\")\n fd.write(description)\n fd.close()\n\n # Generate a csv file with the parameter settings in it\n fd = open(os.path.join(outDir, \"params.csv\"), \"wb\")\n writer = csv.writer(fd)\n colNames = paramLabels.keys()\n colNames.sort()\n writer.writerow(colNames)\n row = [paramLabels[x] for x in colNames]\n writer.writerow(row)\n fd.close()\n\n print \"Generating model params file...\"\n # Generate a model params file alongside the description.py\n mod = imp.load_source(\"description\", os.path.join(outDir,\n \"description.py\"))\n model_description = mod.descriptionInterface.getModelDescription()\n fd = open(os.path.join(outDir, \"model_params.py\"), \"wb\")\n fd.write(\"%s\\nMODEL_PARAMS = %s\" % (getCopyrightHead(),\n pprint.pformat(model_description)))\n fd.close()\n\n print\n\n reportWriter.finalize()\n return model_description", "response": "Generates a report from all available results in the given HyperSearch job and emits all available results to the permutations report csv."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload a _HyperSearchJob instance from info saved in file", "response": "def loadSavedHyperSearchJob(cls, permWorkDir, outputLabel):\n \"\"\"Instantiates a _HyperSearchJob instance from info saved in file\n\n Parameters:\n ----------------------------------------------------------------------\n permWorkDir: Directory path for saved jobID file\n outputLabel: Label string for incorporating into file name for saved jobID\n retval: _HyperSearchJob instance; raises exception if not found\n \"\"\"\n jobID = cls.__loadHyperSearchJobID(permWorkDir=permWorkDir,\n outputLabel=outputLabel)\n\n searchJob = _HyperSearchJob(nupicJobID=jobID)\n return searchJob"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsave the given _HyperSearchJob instance s jobID to file.", "response": "def __saveHyperSearchJobID(cls, permWorkDir, outputLabel, hyperSearchJob):\n \"\"\"Saves the given _HyperSearchJob instance's jobID to file\n\n Parameters:\n ----------------------------------------------------------------------\n permWorkDir: Directory path for saved jobID file\n outputLabel: Label string for incorporating into file name for saved jobID\n hyperSearchJob: _HyperSearchJob instance\n retval: nothing\n \"\"\"\n jobID = hyperSearchJob.getJobID()\n filePath = cls.__getHyperSearchJobIDFilePath(permWorkDir=permWorkDir,\n outputLabel=outputLabel)\n\n if os.path.exists(filePath):\n _backupFile(filePath)\n\n d = dict(hyperSearchJobID = jobID)\n\n with open(filePath, \"wb\") as jobIdPickleFile:\n pickle.dump(d, jobIdPickleFile)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads a saved jobID from file and returns it.", "response": "def __loadHyperSearchJobID(cls, permWorkDir, outputLabel):\n \"\"\"Loads a saved jobID from file\n\n Parameters:\n ----------------------------------------------------------------------\n permWorkDir: Directory path for saved jobID file\n outputLabel: Label string for incorporating into file name for saved jobID\n retval: HyperSearch jobID; raises exception if not found.\n \"\"\"\n filePath = cls.__getHyperSearchJobIDFilePath(permWorkDir=permWorkDir,\n outputLabel=outputLabel)\n\n jobID = None\n with open(filePath, \"r\") as jobIdPickleFile:\n jobInfo = pickle.load(jobIdPickleFile)\n jobID = jobInfo[\"hyperSearchJobID\"]\n\n return jobID"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the filepath where to store the HyperSearch job ID", "response": "def __getHyperSearchJobIDFilePath(cls, permWorkDir, outputLabel):\n \"\"\"Returns filepath where to store HyperSearch JobID\n\n Parameters:\n ----------------------------------------------------------------------\n permWorkDir: Directory path for saved jobID file\n outputLabel: Label string for incorporating into file name for saved jobID\n retval: Filepath where to store HyperSearch JobID\n \"\"\"\n # Get the base path and figure out the path of the report file.\n basePath = permWorkDir\n\n # Form the name of the output csv file that will contain all the results\n filename = \"%s_HyperSearchJobID.pkl\" % (outputLabel,)\n filepath = os.path.join(basePath, filename)\n\n return filepath"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef emit(self, modelInfo):\n # Open/init csv file, if needed\n if self.__csvFileObj is None:\n # sets up self.__sortedVariableNames and self.__csvFileObj\n self.__openAndInitCSVFile(modelInfo)\n\n csv = self.__csvFileObj\n\n # Emit model info row to report.csv\n print >> csv, \"%s, \" % (self.__searchJobID),\n print >> csv, \"%s, \" % (modelInfo.getModelID()),\n print >> csv, \"%s, \" % (modelInfo.statusAsString()),\n if modelInfo.isFinished():\n print >> csv, \"%s, \" % (modelInfo.getCompletionReason()),\n else:\n print >> csv, \"NA, \",\n if not modelInfo.isWaitingToStart():\n print >> csv, \"%s, \" % (modelInfo.getStartTime()),\n else:\n print >> csv, \"NA, \",\n if modelInfo.isFinished():\n dateFormat = \"%Y-%m-%d %H:%M:%S\"\n startTime = modelInfo.getStartTime()\n endTime = modelInfo.getEndTime()\n print >> csv, \"%s, \" % endTime,\n st = datetime.strptime(startTime, dateFormat)\n et = datetime.strptime(endTime, dateFormat)\n print >> csv, \"%s, \" % (str((et - st).seconds)),\n else:\n print >> csv, \"NA, \",\n print >> csv, \"NA, \",\n print >> csv, \"%s, \" % str(modelInfo.getModelDescription()),\n print >> csv, \"%s, \" % str(modelInfo.getNumRecords()),\n paramLabelsDict = modelInfo.getParamLabels()\n for key in self.__sortedVariableNames:\n # Some values are complex structures,.. which need to be represented as\n # strings\n if key in paramLabelsDict:\n print >> csv, \"%s, \" % (paramLabelsDict[key]),\n else:\n print >> csv, \"None, \",\n metrics = modelInfo.getReportMetrics()\n for key in self.__sortedMetricsKeys:\n value = metrics.get(key, \"NA\")\n value = str(value)\n value = value.replace(\"\\n\", \" \")\n print >> csv, \"%s, \" % (value),\n\n print >> csv", "response": "Emit model info to csv file."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfinalizes the report csv file.", "response": "def finalize(self):\n \"\"\"Close file and print report/backup csv file paths\n\n Parameters:\n ----------------------------------------------------------------------\n retval: nothing\n \"\"\"\n if self.__csvFileObj is not None:\n # Done with file\n self.__csvFileObj.close()\n self.__csvFileObj = None\n\n print \"Report csv saved in %s\" % (self.__reportCSVPath,)\n\n if self.__backupCSVPath:\n print \"Previous report csv file was backed up to %s\" % \\\n (self.__backupCSVPath,)\n else:\n print \"Nothing was written to report csv file.\""} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __openAndInitCSVFile(self, modelInfo):\n # Get the base path and figure out the path of the report file.\n basePath = self.__outputDirAbsPath\n\n # Form the name of the output csv file that will contain all the results\n reportCSVName = \"%s_Report.csv\" % (self.__outputLabel,)\n reportCSVPath = self.__reportCSVPath = os.path.join(basePath, reportCSVName)\n\n # If a report CSV file already exists, back it up\n backupCSVPath = None\n if os.path.exists(reportCSVPath):\n backupCSVPath = self.__backupCSVPath = _backupFile(reportCSVPath)\n\n\n # Open report file\n if self.__replaceReport:\n mode = \"w\"\n else:\n mode = \"a\"\n csv = self.__csvFileObj = open(reportCSVPath, mode)\n\n # If we are appending, add some blank line separators\n if not self.__replaceReport and backupCSVPath:\n print >> csv\n print >> csv\n\n # Print the column names\n print >> csv, \"jobID, \",\n print >> csv, \"modelID, \",\n print >> csv, \"status, \" ,\n print >> csv, \"completionReason, \",\n print >> csv, \"startTime, \",\n print >> csv, \"endTime, \",\n print >> csv, \"runtime(s), \" ,\n print >> csv, \"expDesc, \",\n print >> csv, \"numRecords, \",\n\n for key in self.__sortedVariableNames:\n print >> csv, \"%s, \" % key,\n for key in self.__sortedMetricsKeys:\n print >> csv, \"%s, \" % key,\n print >> csv", "response": "Opens and initializes the csv file for the nupic model."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getJobStatus(self, workers):\n jobInfo = self.JobStatus(self.__nupicJobID, workers)\n return jobInfo", "response": "Returns the status of this nupic job."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nqueries the model IDs associated with this HyperSearch job.", "response": "def queryModelIDs(self):\n \"\"\"Queuries DB for model IDs of all currently instantiated models\n associated with this HyperSearch job.\n\n See also: _iterModels()\n\n Parameters:\n ----------------------------------------------------------------------\n retval: A sequence of Nupic modelIDs\n \"\"\"\n jobID = self.getJobID()\n modelCounterPairs = _clientJobsDB().modelsGetUpdateCounters(jobID)\n modelIDs = tuple(x[0] for x in modelCounterPairs)\n\n return modelIDs"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef makeSearchJobParamsDict(cls, options, forRunning=False):\n if options[\"searchMethod\"] == \"v2\":\n hsVersion = \"v2\"\n else:\n raise Exception(\"Unsupported search method: %r\" % options[\"searchMethod\"])\n\n maxModels = options[\"maxPermutations\"]\n if options[\"action\"] == \"dryRun\" and maxModels is None:\n maxModels = 1\n\n useTerminators = options[\"useTerminators\"]\n if useTerminators is None:\n params = {\n \"hsVersion\": hsVersion,\n \"maxModels\": maxModels,\n }\n else:\n params = {\n \"hsVersion\": hsVersion,\n \"useTerminators\": useTerminators,\n \"maxModels\": maxModels,\n }\n\n if forRunning:\n params[\"persistentJobGUID\"] = str(uuid.uuid1())\n\n if options[\"permutationsScriptPath\"]:\n params[\"permutationsPyFilename\"] = options[\"permutationsScriptPath\"]\n elif options[\"expDescConfig\"]:\n params[\"description\"] = options[\"expDescConfig\"]\n else:\n with open(options[\"expDescJsonPath\"], mode=\"r\") as fp:\n params[\"description\"] = json.load(fp)\n\n return params", "response": "Creates a dictionary of HyperSearch parameters suitable for converting to json and passing as the params argument to ClientJobsDAO. jobInsert"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the optimization key name and optimization function.", "response": "def getOptimizationMetricInfo(cls, searchJobParams):\n \"\"\"Retrives the optimization key name and optimization function.\n\n Parameters:\n ---------------------------------------------------------\n searchJobParams:\n Parameter for passing as the searchParams arg to\n Hypersearch constructor.\n retval: (optimizationMetricKey, maximize)\n optimizationMetricKey: which report key to optimize for\n maximize: True if we should try and maximize the optimizeKey\n metric. False if we should minimize it.\n \"\"\"\n if searchJobParams[\"hsVersion\"] == \"v2\":\n search = HypersearchV2(searchParams=searchJobParams)\n else:\n raise RuntimeError(\"Unsupported hypersearch version \\\"%s\\\"\" % \\\n (searchJobParams[\"hsVersion\"]))\n\n info = search.getOptimizationMetricInfo()\n return info"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getModelDescription(self):\n params = self.__unwrapParams()\n\n if \"experimentName\" in params:\n return params[\"experimentName\"]\n\n else:\n paramSettings = self.getParamLabels()\n # Form a csv friendly string representation of this model\n items = []\n for key, value in paramSettings.items():\n items.append(\"%s_%s\" % (key, value))\n return \".\".join(items)", "response": "Returns a string representation of the model."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a dictionary of model parameter labels for each entry in the entry .", "response": "def getParamLabels(self):\n \"\"\"\n Parameters:\n ----------------------------------------------------------------------\n retval: a dictionary of model parameter labels. For each entry\n the key is the name of the parameter and the value\n is the value chosen for it.\n \"\"\"\n params = self.__unwrapParams()\n\n # Hypersearch v2 stores the flattened parameter settings in \"particleState\"\n if \"particleState\" in params:\n retval = dict()\n queue = [(pair, retval) for pair in\n params[\"particleState\"][\"varStates\"].iteritems()]\n while len(queue) > 0:\n pair, output = queue.pop()\n k, v = pair\n if (\"position\" in v and \"bestPosition\" in v and\n \"velocity\" in v):\n output[k] = v[\"position\"]\n else:\n if k not in output:\n output[k] = dict()\n queue.extend((pair, output[k]) for pair in v.iteritems())\n return retval"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __unwrapParams(self):\n if self.__cachedParams is None:\n self.__cachedParams = json.loads(self.__rawInfo.params)\n assert self.__cachedParams is not None, \\\n \"%s resulted in None\" % self.__rawInfo.params\n\n return self.__cachedParams", "response": "Unwraps self. params into the equivalent python dictionary\n and caches it in self. params"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a dictionary of metrics that combines all report and optimization metrics with the current model.", "response": "def getAllMetrics(self):\n \"\"\"Retrives a dictionary of metrics that combines all report and\n optimization metrics\n\n Parameters:\n ----------------------------------------------------------------------\n retval: a dictionary of optimization metrics that were collected\n for the model; an empty dictionary if there aren't any.\n \"\"\"\n result = self.getReportMetrics()\n result.update(self.getOptimizationMetrics())\n return result"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nunwraps self. rawInfo. results and caches it in self. __cachedResults", "response": "def __unwrapResults(self):\n \"\"\"Unwraps self.__rawInfo.results and caches it in self.__cachedResults;\n Returns the unwrapped params\n\n Parameters:\n ----------------------------------------------------------------------\n retval: ModelResults namedtuple instance\n \"\"\"\n if self.__cachedResults is None:\n if self.__rawInfo.results is not None:\n resultList = json.loads(self.__rawInfo.results)\n assert len(resultList) == 2, \\\n \"Expected 2 elements, but got %s (%s).\" % (\n len(resultList), resultList)\n self.__cachedResults = self.ModelResults(\n reportMetrics=resultList[0],\n optimizationMetrics=resultList[1])\n else:\n self.__cachedResults = self.ModelResults(\n reportMetrics={},\n optimizationMetrics={})\n\n\n return self.__cachedResults"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the next n values for the distribution as a list.", "response": "def getData(self, n):\n \"\"\"Returns the next n values for the distribution as a list.\"\"\"\n\n records = [self.getNext() for x in range(n)]\n return records"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getTerminationCallbacks(self, terminationFunc):\n activities = [None] * len(ModelTerminator._MILESTONES)\n for index, (iteration, _) in enumerate(ModelTerminator._MILESTONES):\n cb = functools.partial(terminationFunc, index=index)\n activities[index] = PeriodicActivityRequest(repeating =False,\n period = iteration,\n cb=cb)", "response": "Returns the periodic checks to see if the model should\n continue running."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlike itertools.groupby, with the following additions: - Supports multiple sequences. Instead of returning (k, g), each iteration returns (k, g0, g1, ...), with one `g` for each input sequence. The value of each `g` is either a non-empty iterator or `None`. - It treats the value `None` as an empty sequence. So you can make subsequent calls to groupby2 on any `g` value. .. note:: Read up on groupby here: https://docs.python.org/dev/library/itertools.html#itertools.groupby :param args: (list) Parameters alternating between sorted lists and their respective key functions. The lists should be sorted with respect to their key function. :returns: (tuple) A n + 1 dimensional tuple, where the first element is the key of the iteration, and the other n entries are groups of objects that share this key. Each group corresponds to the an input sequence. `groupby2` is a generator that returns a tuple for every iteration. If an input sequence has no members with the current key, None is returned in place of a generator.", "response": "def groupby2(*args):\n \"\"\" Like itertools.groupby, with the following additions:\n\n - Supports multiple sequences. Instead of returning (k, g), each iteration\n returns (k, g0, g1, ...), with one `g` for each input sequence. The value of\n each `g` is either a non-empty iterator or `None`.\n - It treats the value `None` as an empty sequence. So you can make subsequent\n calls to groupby2 on any `g` value.\n\n .. note:: Read up on groupby here:\n https://docs.python.org/dev/library/itertools.html#itertools.groupby\n\n :param args: (list) Parameters alternating between sorted lists and their\n respective key functions. The lists should be sorted with\n respect to their key function.\n\n :returns: (tuple) A n + 1 dimensional tuple, where the first element is the\n key of the iteration, and the other n entries are groups of\n objects that share this key. Each group corresponds to the an\n input sequence. `groupby2` is a generator that returns a tuple\n for every iteration. If an input sequence has no members with\n the current key, None is returned in place of a generator.\n \"\"\"\n generatorList = [] # list of each list's (k, group) tuples\n\n if len(args) % 2 == 1:\n raise ValueError(\"Must have a key function for every list.\")\n\n advanceList = []\n\n # populate above lists\n for i in xrange(0, len(args), 2):\n listn = args[i]\n fn = args[i + 1]\n if listn is not None:\n generatorList.append(groupby(listn, fn))\n advanceList.append(True) # start by advancing everyone.\n else:\n generatorList.append(None)\n advanceList.append(False)\n\n n = len(generatorList)\n\n nextList = [None] * n\n # while all lists aren't exhausted walk through each group in order\n while True:\n for i in xrange(n):\n if advanceList[i]:\n try:\n nextList[i] = generatorList[i].next()\n except StopIteration:\n nextList[i] = None\n\n # no more values to process in any of the generators\n if all(entry is None for entry in nextList):\n break\n\n # the minimum key value in the nextList\n minKeyVal = min(nextVal[0] for nextVal in nextList\n if nextVal is not None)\n\n # populate the tuple to return based on minKeyVal\n retGroups = [minKeyVal]\n for i in xrange(n):\n if nextList[i] is not None and nextList[i][0] == minKeyVal:\n retGroups.append(nextList[i][1])\n advanceList[i] = True\n else:\n advanceList[i] = False\n retGroups.append(None)\n\n yield tuple(retGroups)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _openStream(dataUrl,\n isBlocking, # pylint: disable=W0613\n maxTimeout, # pylint: disable=W0613\n bookmark,\n firstRecordIdx):\n \"\"\"Open the underlying file stream\n This only supports 'file://' prefixed paths.\n\n :returns: record stream instance\n :rtype: FileRecordStream\n \"\"\"\n filePath = dataUrl[len(FILE_PREF):]\n if not os.path.isabs(filePath):\n filePath = os.path.join(os.getcwd(), filePath)\n return FileRecordStream(streamID=filePath,\n write=False,\n bookmark=bookmark,\n firstRecord=firstRecordIdx)", "response": "Open the underlying file stream."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getNextRecord(self):\n\n\n # Keep reading from the raw input till we get enough for an aggregated\n # record\n while True:\n\n # Reached EOF due to lastRow constraint?\n if self._sourceLastRecordIdx is not None and \\\n self._recordStore.getNextRecordIdx() >= self._sourceLastRecordIdx:\n preAggValues = None # indicates EOF\n bookmark = self._recordStore.getBookmark()\n\n else:\n # Get the raw record and bookmark\n preAggValues = self._recordStore.getNextRecord()\n bookmark = self._recordStore.getBookmark()\n\n if preAggValues == (): # means timeout error occurred\n if self._eofOnTimeout:\n preAggValues = None # act as if we got EOF\n else:\n return preAggValues # Timeout indicator\n\n self._logger.debug('Read source record #%d: %r',\n self._recordStore.getNextRecordIdx()-1, preAggValues)\n\n # Perform aggregation\n (fieldValues, aggBookmark) = self._aggregator.next(preAggValues, bookmark)\n\n # Update the aggregated record bookmark if we got a real record back\n if fieldValues is not None:\n self._aggBookmark = aggBookmark\n\n # Reached EOF?\n if preAggValues is None and fieldValues is None:\n return None\n\n # Return it if we have a record\n if fieldValues is not None:\n break\n\n\n # Do we need to re-order the fields in the record?\n if self._needFieldsFiltering:\n values = []\n srcDict = dict(zip(self._recordStoreFieldNames, fieldValues))\n for name in self._streamFieldNames:\n values.append(srcDict[name])\n fieldValues = values\n\n\n # Write to debug output?\n if self._writer is not None:\n self._writer.appendRecord(fieldValues)\n\n self._recordCount += 1\n\n self._logger.debug('Returning aggregated record #%d from getNextRecord(): '\n '%r. Bookmark: %r',\n self._recordCount-1, fieldValues, self._aggBookmark)\n return fieldValues", "response": "Returns the next record from all sources."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the number of records in the datastream after aggregation.", "response": "def getDataRowCount(self):\n \"\"\"\n Iterates through stream to calculate total records after aggregation.\n This will alter the bookmark state.\n \"\"\"\n inputRowCountAfterAggregation = 0\n while True:\n record = self.getNextRecord()\n if record is None:\n return inputRowCountAfterAggregation\n inputRowCountAfterAggregation += 1\n\n if inputRowCountAfterAggregation > 10000:\n raise RuntimeError('No end of datastream found.')"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getStats(self):\n\n # The record store returns a dict of stats, each value in this dict is\n # a list with one item per field of the record store\n # {\n # 'min' : [f1_min, f2_min, f3_min],\n # 'max' : [f1_max, f2_max, f3_max]\n # }\n recordStoreStats = self._recordStore.getStats()\n\n # We need to convert each item to represent the fields of the *stream*\n streamStats = dict()\n for (key, values) in recordStoreStats.items():\n fieldStats = dict(zip(self._recordStoreFieldNames, values))\n streamValues = []\n for name in self._streamFieldNames:\n streamValues.append(fieldStats[name])\n streamStats[key] = streamValues\n\n return streamStats", "response": "Returns the stats for the current record store."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a pattern for a number.", "response": "def get(self, number):\n \"\"\"\n Return a pattern for a number.\n\n @param number (int) Number of pattern\n\n @return (set) Indices of on bits\n \"\"\"\n if not number in self._patterns:\n raise IndexError(\"Invalid number\")\n\n return self._patterns[number]"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef addNoise(self, bits, amount):\n newBits = set()\n\n for bit in bits:\n if self._random.getReal64() < amount:\n newBits.add(self._random.getUInt32(self._n))\n else:\n newBits.add(bit)\n\n return newBits", "response": "Add noise to noisy pattern."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef numbersForBit(self, bit):\n if bit >= self._n:\n raise IndexError(\"Invalid bit\")\n\n numbers = set()\n\n for index, pattern in self._patterns.iteritems():\n if bit in pattern:\n numbers.add(index)\n\n return numbers", "response": "Returns the set of pattern numbers that match a bit."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef numberMapForBits(self, bits):\n numberMap = dict()\n\n for bit in bits:\n numbers = self.numbersForBit(bit)\n\n for number in numbers:\n if not number in numberMap:\n numberMap[number] = set()\n\n numberMap[number].add(bit)\n\n return numberMap", "response": "Returns a dictionary mapping from number to matching on bits for all numbers that match a set of bits."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef prettyPrintPattern(self, bits, verbosity=1):\n numberMap = self.numberMapForBits(bits)\n text = \"\"\n\n numberList = []\n numberItems = sorted(numberMap.iteritems(),\n key=lambda (number, bits): len(bits),\n reverse=True)\n\n for number, bits in numberItems:\n\n if verbosity > 2:\n strBits = [str(n) for n in bits]\n numberText = \"{0} (bits: {1})\".format(number, \",\".join(strBits))\n elif verbosity > 1:\n numberText = \"{0} ({1} bits)\".format(number, len(bits))\n else:\n numberText = str(number)\n\n numberList.append(numberText)\n\n text += \"[{0}]\".format(\", \".join(numberList))\n\n return text", "response": "Pretty print a pattern."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating set of random patterns.", "response": "def _generate(self):\n \"\"\"\n Generates set of random patterns.\n \"\"\"\n candidates = np.array(range(self._n), np.uint32)\n for i in xrange(self._num):\n self._random.shuffle(candidates)\n pattern = candidates[0:self._getW()]\n self._patterns[i] = set(pattern)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _getW(self):\n w = self._w\n\n if type(w) is list:\n return w[self._random.getUInt32(len(w))]\n else:\n return w", "response": "Gets a value of w for use in generating a pattern."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates set of consecutive patterns.", "response": "def _generate(self):\n \"\"\"\n Generates set of consecutive patterns.\n \"\"\"\n n = self._n\n w = self._w\n\n assert type(w) is int, \"List for w not supported\"\n\n for i in xrange(n / w):\n pattern = set(xrange(i * w, (i+1) * w))\n self._patterns[i] = pattern"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprocesses one input sample. This method is called by outer loop code outside the nupic-engine. We use this instead of the nupic engine compute() because our inputs and outputs aren't fixed size vectors of reals. :param recordNum: Record number of this input pattern. Record numbers normally increase sequentially by 1 each time unless there are missing records in the dataset. Knowing this information insures that we don't get confused by missing records. :param patternNZ: List of the active indices from the output below. When the input is from TemporalMemory, this list should be the indices of the active cells. :param classification: Dict of the classification information where: - bucketIdx: list of indices of the encoder bucket - actValue: list of actual values going into the encoder Classification could be None for inference mode. :param learn: (bool) if true, learn this sample :param infer: (bool) if true, perform inference :return: Dict containing inference results, there is one entry for each step in self.steps, where the key is the number of steps, and the value is an array containing the relative likelihood for each bucketIdx starting from bucketIdx 0. There is also an entry containing the average actual value to use for each bucket. The key is 'actualValues'. for example: .. code-block:: python {1 : [0.1, 0.3, 0.2, 0.7], 4 : [0.2, 0.4, 0.3, 0.5], 'actualValues': [1.5, 3,5, 5,5, 7.6], }", "response": "def compute(self, recordNum, patternNZ, classification, learn, infer):\n \"\"\"\n Process one input sample.\n\n This method is called by outer loop code outside the nupic-engine. We\n use this instead of the nupic engine compute() because our inputs and\n outputs aren't fixed size vectors of reals.\n\n\n :param recordNum: Record number of this input pattern. Record numbers\n normally increase sequentially by 1 each time unless there are missing\n records in the dataset. Knowing this information insures that we don't get\n confused by missing records.\n\n :param patternNZ: List of the active indices from the output below. When the\n input is from TemporalMemory, this list should be the indices of the\n active cells.\n\n :param classification: Dict of the classification information where:\n\n - bucketIdx: list of indices of the encoder bucket\n - actValue: list of actual values going into the encoder\n\n Classification could be None for inference mode.\n :param learn: (bool) if true, learn this sample\n :param infer: (bool) if true, perform inference\n\n :return: Dict containing inference results, there is one entry for each\n step in self.steps, where the key is the number of steps, and\n the value is an array containing the relative likelihood for\n each bucketIdx starting from bucketIdx 0.\n\n There is also an entry containing the average actual value to\n use for each bucket. The key is 'actualValues'.\n\n for example:\n\n .. code-block:: python\n\n {1 : [0.1, 0.3, 0.2, 0.7],\n 4 : [0.2, 0.4, 0.3, 0.5],\n 'actualValues': [1.5, 3,5, 5,5, 7.6],\n }\n \"\"\"\n if self.verbosity >= 1:\n print \" learn:\", learn\n print \" recordNum:\", recordNum\n print \" patternNZ (%d):\" % len(patternNZ), patternNZ\n print \" classificationIn:\", classification\n\n # ensures that recordNum increases monotonically\n if len(self._patternNZHistory) > 0:\n if recordNum < self._patternNZHistory[-1][0]:\n raise ValueError(\"the record number has to increase monotonically\")\n\n # Store pattern in our history if this is a new record\n if len(self._patternNZHistory) == 0 or \\\n recordNum > self._patternNZHistory[-1][0]:\n self._patternNZHistory.append((recordNum, patternNZ))\n\n # To allow multi-class classification, we need to be able to run learning\n # without inference being on. So initialize retval outside\n # of the inference block.\n retval = {}\n\n # Update maxInputIdx and augment weight matrix with zero padding\n if max(patternNZ) > self._maxInputIdx:\n newMaxInputIdx = max(patternNZ)\n for nSteps in self.steps:\n self._weightMatrix[nSteps] = numpy.concatenate((\n self._weightMatrix[nSteps],\n numpy.zeros(shape=(newMaxInputIdx-self._maxInputIdx,\n self._maxBucketIdx+1))), axis=0)\n self._maxInputIdx = int(newMaxInputIdx)\n\n # Get classification info\n if classification is not None:\n if type(classification[\"bucketIdx\"]) is not list:\n bucketIdxList = [classification[\"bucketIdx\"]]\n actValueList = [classification[\"actValue\"]]\n numCategory = 1\n else:\n bucketIdxList = classification[\"bucketIdx\"]\n actValueList = classification[\"actValue\"]\n numCategory = len(classification[\"bucketIdx\"])\n else:\n if learn:\n raise ValueError(\"classification cannot be None when learn=True\")\n actValueList = None\n bucketIdxList = None\n # ------------------------------------------------------------------------\n # Inference:\n # For each active bit in the activationPattern, get the classification\n # votes\n if infer:\n retval = self.infer(patternNZ, actValueList)\n\n\n if learn and classification[\"bucketIdx\"] is not None:\n for categoryI in range(numCategory):\n bucketIdx = bucketIdxList[categoryI]\n actValue = actValueList[categoryI]\n\n # Update maxBucketIndex and augment weight matrix with zero padding\n if bucketIdx > self._maxBucketIdx:\n for nSteps in self.steps:\n self._weightMatrix[nSteps] = numpy.concatenate((\n self._weightMatrix[nSteps],\n numpy.zeros(shape=(self._maxInputIdx+1,\n bucketIdx-self._maxBucketIdx))), axis=1)\n\n self._maxBucketIdx = int(bucketIdx)\n\n # Update rolling average of actual values if it's a scalar. If it's\n # not, it must be a category, in which case each bucket only ever\n # sees one category so we don't need a running average.\n while self._maxBucketIdx > len(self._actualValues) - 1:\n self._actualValues.append(None)\n if self._actualValues[bucketIdx] is None:\n self._actualValues[bucketIdx] = actValue\n else:\n if (isinstance(actValue, int) or\n isinstance(actValue, float) or\n isinstance(actValue, long)):\n self._actualValues[bucketIdx] = ((1.0 - self.actValueAlpha)\n * self._actualValues[bucketIdx]\n + self.actValueAlpha * actValue)\n else:\n self._actualValues[bucketIdx] = actValue\n\n for (learnRecordNum, learnPatternNZ) in self._patternNZHistory:\n error = self._calculateError(recordNum, bucketIdxList)\n\n nSteps = recordNum - learnRecordNum\n if nSteps in self.steps:\n for bit in learnPatternNZ:\n self._weightMatrix[nSteps][bit, :] += self.alpha * error[nSteps]\n\n # ------------------------------------------------------------------------\n # Verbose print\n if infer and self.verbosity >= 1:\n print \" inference: combined bucket likelihoods:\"\n print \" actual bucket values:\", retval[\"actualValues\"]\n for (nSteps, votes) in retval.items():\n if nSteps == \"actualValues\":\n continue\n print \" %d steps: \" % (nSteps), _pFormatArray(votes)\n bestBucketIdx = votes.argmax()\n print (\" most likely bucket idx: \"\n \"%d, value: %s\" % (bestBucketIdx,\n retval[\"actualValues\"][bestBucketIdx]))\n print\n\n return retval"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninfers the actual value of a pattern from one input sample.", "response": "def infer(self, patternNZ, actValueList):\n \"\"\"\n Return the inference value from one input sample. The actual\n learning happens in compute().\n\n :param patternNZ: list of the active indices from the output below\n :param classification: dict of the classification information:\n bucketIdx: index of the encoder bucket\n actValue: actual value going into the encoder\n\n :return: dict containing inference results, one entry for each step in\n self.steps. The key is the number of steps, the value is an\n array containing the relative likelihood for each bucketIdx\n starting from bucketIdx 0.\n\n for example:\n\n .. code-block:: python\n\n {'actualValues': [0.0, 1.0, 2.0, 3.0]\n 1 : [0.1, 0.3, 0.2, 0.7]\n 4 : [0.2, 0.4, 0.3, 0.5]}\n \"\"\"\n\n # Return value dict. For buckets which we don't have an actual value\n # for yet, just plug in any valid actual value. It doesn't matter what\n # we use because that bucket won't have non-zero likelihood anyways.\n\n # NOTE: If doing 0-step prediction, we shouldn't use any knowledge\n # of the classification input during inference.\n if self.steps[0] == 0 or actValueList is None:\n defaultValue = 0\n else:\n defaultValue = actValueList[0]\n actValues = [x if x is not None else defaultValue\n for x in self._actualValues]\n retval = {\"actualValues\": actValues}\n\n for nSteps in self.steps:\n predictDist = self.inferSingleStep(patternNZ, self._weightMatrix[nSteps])\n\n retval[nSteps] = predictDist\n\n return retval"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef inferSingleStep(self, patternNZ, weightMatrix):\n outputActivation = weightMatrix[patternNZ].sum(axis=0)\n\n # softmax normalization\n outputActivation = outputActivation - numpy.max(outputActivation)\n expOutputActivation = numpy.exp(outputActivation)\n predictDist = expOutputActivation / numpy.sum(expOutputActivation)\n return predictDist", "response": "Perform inference for a single step. Given an SDR input and a weight matrix return a predicted class label distribution."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the error at the output layer.", "response": "def _calculateError(self, recordNum, bucketIdxList):\n \"\"\"\n Calculate error signal\n\n :param bucketIdxList: list of encoder buckets\n\n :return: dict containing error. The key is the number of steps\n The value is a numpy array of error at the output layer\n \"\"\"\n error = dict()\n targetDist = numpy.zeros(self._maxBucketIdx + 1)\n numCategories = len(bucketIdxList)\n for bucketIdx in bucketIdxList:\n targetDist[bucketIdx] = 1.0/numCategories\n\n for (learnRecordNum, learnPatternNZ) in self._patternNZHistory:\n nSteps = recordNum - learnRecordNum\n if nSteps in self.steps:\n predictDist = self.inferSingleStep(learnPatternNZ,\n self._weightMatrix[nSteps])\n error[nSteps] = targetDist - predictDist\n\n return error"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsorts a potentially big file by a list of field names.", "response": "def sort(filename, key, outputFile, fields=None, watermark=1024 * 1024 * 100):\n \"\"\"Sort a potentially big file\n\n filename - the input file (standard File format)\n key - a list of field names to sort by\n outputFile - the name of the output file\n fields - a list of fields that should be included (all fields if None)\n watermark - when available memory goes bellow the watermark create a new chunk\n\n sort() works by reading as records from the file into memory\n and calling _sortChunk() on each chunk. In the process it gets\n rid of unneeded fields if any. Once all the chunks have been sorted and\n written to chunk files it calls _merge() to merge all the chunks into a\n single sorted file.\n\n Note, that sort() gets a key that contains field names, which it converts\n into field indices for _sortChunk() becuase _sortChunk() doesn't need to know\n the field name.\n\n sort() figures out by itself how many chunk files to use by reading records\n from the file until the low watermark value of availabel memory is hit and\n then it sorts the current records, generates a chunk file, clears the sorted\n records and starts on a new chunk.\n\n The key field names are turned into indices\n \"\"\"\n if fields is not None:\n assert set(key).issubset(set([f[0] for f in fields]))\n\n with FileRecordStream(filename) as f:\n\n\n # Find the indices of the requested fields\n if fields:\n fieldNames = [ff[0] for ff in fields]\n indices = [f.getFieldNames().index(name) for name in fieldNames]\n assert len(indices) == len(fields)\n else:\n fileds = f.getFields()\n fieldNames = f.getFieldNames()\n indices = None\n\n # turn key fields to key indices\n key = [fieldNames.index(name) for name in key]\n\n chunk = 0\n records = []\n for i, r in enumerate(f):\n # Select requested fields only\n if indices:\n temp = []\n for i in indices:\n temp.append(r[i])\n r = temp\n # Store processed record\n records.append(r)\n\n # Check memory\n available_memory = psutil.avail_phymem()\n\n # If bellow the watermark create a new chunk, reset and keep going\n if available_memory < watermark:\n _sortChunk(records, key, chunk, fields)\n records = []\n chunk += 1\n\n # Sort and write the remainder\n if len(records) > 0:\n _sortChunk(records, key, chunk, fields)\n chunk += 1\n\n # Marge all the files\n _mergeFiles(key, chunk, outputFile, fields)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsort in memory chunk of records by chunkIndex", "response": "def _sortChunk(records, key, chunkIndex, fields):\n \"\"\"Sort in memory chunk of records\n\n records - a list of records read from the original dataset\n key - a list of indices to sort the records by\n chunkIndex - the index of the current chunk\n\n The records contain only the fields requested by the user.\n\n _sortChunk() will write the sorted records to a standard File\n named \"chunk_.csv\" (chunk_0.csv, chunk_1.csv,...).\n \"\"\"\n title(additional='(key=%s, chunkIndex=%d)' % (str(key), chunkIndex))\n\n assert len(records) > 0\n\n # Sort the current records\n records.sort(key=itemgetter(*key))\n\n # Write to a chunk file\n if chunkIndex is not None:\n filename = 'chunk_%d.csv' % chunkIndex\n with FileRecordStream(filename, write=True, fields=fields) as o:\n for r in records:\n o.appendRecord(r)\n\n assert os.path.getsize(filename) > 0\n\n return records"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _mergeFiles(key, chunkCount, outputFile, fields):\n title()\n\n # Open all chun files\n files = [FileRecordStream('chunk_%d.csv' % i) for i in range(chunkCount)]\n\n # Open output file\n with FileRecordStream(outputFile, write=True, fields=fields) as o:\n # Open all chunk files\n files = [FileRecordStream('chunk_%d.csv' % i) for i in range(chunkCount)]\n records = [f.getNextRecord() for f in files]\n\n # This loop will run until all files are exhausted\n while not all(r is None for r in records):\n # Cleanup None values (files that were exhausted)\n indices = [i for i,r in enumerate(records) if r is not None]\n records = [records[i] for i in indices]\n files = [files[i] for i in indices]\n\n # Find the current record\n r = min(records, key=itemgetter(*key))\n # Write it to the file\n o.appendRecord(r)\n\n # Find the index of file that produced the current record\n index = records.index(r)\n # Read a new record from the file\n records[index] = files[index].getNextRecord()\n\n # Cleanup chunk files\n for i, f in enumerate(files):\n f.close()\n os.remove('chunk_%d.csv' % i)", "response": "Merge sorted chunk files into a sorted output file\n"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompute the state of the temporal memory.", "response": "def compute(self, activeColumns, learn=True):\n \"\"\"\n Feeds input record through TM, performing inference and learning.\n Updates member variables with new state.\n\n @param activeColumns (set) Indices of active columns in `t`\n \"\"\"\n bottomUpInput = numpy.zeros(self.numberOfCols, dtype=dtype)\n bottomUpInput[list(activeColumns)] = 1\n super(TemporalMemoryShim, self).compute(bottomUpInput,\n enableLearn=learn,\n enableInference=True)\n\n predictedState = self.getPredictedState()\n self.predictiveCells = set(numpy.flatnonzero(predictedState))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef read(cls, proto):\n tm = super(TemporalMemoryShim, cls).read(proto.baseTM)\n tm.predictiveCells = set(proto.predictedState)\n tm.connections = Connections.read(proto.conncetions)", "response": "Deserialize from proto instance."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npopulates serialization proto instance.", "response": "def write(self, proto):\n \"\"\"Populate serialization proto instance.\n\n :param proto: (TemporalMemoryShimProto) the proto instance to populate\n \"\"\"\n super(TemporalMemoryShim, self).write(proto.baseTM)\n proto.connections.write(self.connections)\n proto.predictiveCells = self.predictiveCells"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprint a message to the console.", "response": "def cPrint(self, level, message, *args, **kw):\n \"\"\"Print a message to the console.\n\n Prints only if level <= self.consolePrinterVerbosity\n Printing with level 0 is equivalent to using a print statement,\n and should normally be avoided.\n\n :param level: (int) indicating the urgency of the message with\n lower values meaning more urgent (messages at level 0 are the most\n urgent and are always printed)\n\n :param message: (string) possibly with format specifiers\n\n :param args: specifies the values for any format specifiers in message\n\n :param kw: newline is the only keyword argument. True (default) if a newline\n should be printed\n \"\"\"\n\n if level > self.consolePrinterVerbosity:\n return\n\n if len(kw) > 1:\n raise KeyError(\"Invalid keywords for cPrint: %s\" % str(kw.keys()))\n\n newline = kw.get(\"newline\", True)\n if len(kw) == 1 and 'newline' not in kw:\n raise KeyError(\"Invalid keyword for cPrint: %s\" % kw.keys()[0])\n\n if len(args) == 0:\n if newline:\n print message\n else:\n print message,\n else:\n if newline:\n print message % args\n else:\n print message % args,"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef profileTM(tmClass, tmDim, nRuns):\n\n # create TM instance to measure\n tm = tmClass(numberOfCols=tmDim)\n\n # generate input data\n data = numpy.random.randint(0, 2, [tmDim, nRuns]).astype('float32')\n\n for i in xrange(nRuns):\n # new data every time, this is the worst case performance\n # real performance would be better, as the input data would not be completely random\n d = data[:,i]\n\n # the actual function to profile!\n tm.compute(d, True)", "response": "profile performance of a TemporalMemory"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating a simple dataset.", "response": "def _generateCategory(filename=\"simple.csv\", numSequences=2, elementsPerSeq=1, \n numRepeats=10, resets=False):\n \"\"\" Generate a simple dataset. This contains a bunch of non-overlapping\n sequences. \n \n Parameters:\n ----------------------------------------------------\n filename: name of the file to produce, including extension. It will\n be created in a 'datasets' sub-directory within the \n directory containing this script. \n numSequences: how many sequences to generate\n elementsPerSeq: length of each sequence\n numRepeats: how many times to repeat each sequence in the output \n resets: if True, turn on reset at start of each sequence\n \"\"\"\n \n # Create the output file\n scriptDir = os.path.dirname(__file__)\n pathname = os.path.join(scriptDir, 'datasets', filename)\n print \"Creating %s...\" % (pathname)\n fields = [('reset', 'int', 'R'), ('category', 'int', 'C'),\n ('field1', 'string', '')] \n outFile = FileRecordStream(pathname, write=True, fields=fields)\n \n # Create the sequences\n sequences = []\n for i in range(numSequences):\n seq = [x for x in range(i*elementsPerSeq, (i+1)*elementsPerSeq)]\n sequences.append(seq)\n \n # Write out the sequences in random order\n seqIdxs = []\n for i in range(numRepeats):\n seqIdxs += range(numSequences)\n random.shuffle(seqIdxs)\n \n for seqIdx in seqIdxs:\n reset = int(resets)\n seq = sequences[seqIdx]\n for x in seq:\n outFile.appendRecord([reset, str(seqIdx), str(x)])\n reset = 0\n\n outFile.close()"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nencode the SDR into a numpy array.", "response": "def encodeIntoArray(self, inputData, output):\n \"\"\"\n See `nupic.encoders.base.Encoder` for more information.\n\n :param: inputData (tuple) Contains speed (float), longitude (float),\n latitude (float), altitude (float)\n :param: output (numpy.array) Stores encoded SDR in this numpy array\n \"\"\"\n altitude = None\n if len(inputData) == 4:\n (speed, longitude, latitude, altitude) = inputData\n else:\n (speed, longitude, latitude) = inputData\n coordinate = self.coordinateForPosition(longitude, latitude, altitude)\n radius = self.radiusForSpeed(speed)\n super(GeospatialCoordinateEncoder, self).encodeIntoArray(\n (coordinate, radius), output)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the coordinate that maps the given GPS position.", "response": "def coordinateForPosition(self, longitude, latitude, altitude=None):\n \"\"\"\n Returns coordinate for given GPS position.\n\n :param: longitude (float) Longitude of position\n :param: latitude (float) Latitude of position\n :param: altitude (float) Altitude of position\n :returns: (numpy.array) Coordinate that the given GPS position\n maps to\n \"\"\"\n coords = PROJ(longitude, latitude)\n\n if altitude is not None:\n coords = transform(PROJ, geocentric, coords[0], coords[1], altitude)\n\n coordinate = numpy.array(coords)\n coordinate = coordinate / self.scale\n return coordinate.astype(int)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the radius of the atom in the order of the readings in the given speed.", "response": "def radiusForSpeed(self, speed):\n \"\"\"\n Returns radius for given speed.\n\n Tries to get the encodings of consecutive readings to be\n adjacent with some overlap.\n\n :param: speed (float) Speed (in meters per second)\n :returns: (int) Radius for given speed\n \"\"\"\n overlap = 1.5\n coordinatesPerTimestep = speed * self.timestep / self.scale\n radius = int(round(float(coordinatesPerTimestep) / 2 * overlap))\n minRadius = int(math.ceil((math.sqrt(self.w) - 1) / 2))\n return max(radius, minRadius)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getSearch(rootDir):\n \n # Form the stream definition\n dataPath = os.path.abspath(os.path.join(rootDir, 'datasets', 'scalar_1.csv'))\n streamDef = dict(\n version = 1, \n info = \"testSpatialClassification\",\n streams = [\n dict(source=\"file://%s\" % (dataPath), \n info=\"scalar_1.csv\", \n columns=[\"*\"],\n ),\n ],\n )\n\n # Generate the experiment description\n expDesc = {\n \"environment\": 'nupic',\n \"inferenceArgs\":{\n \"predictedField\":\"classification\",\n \"predictionSteps\": [0],\n },\n \"inferenceType\": \"MultiStep\",\n \"streamDef\": streamDef,\n \"includedFields\": [\n { \"fieldName\": \"field1\",\n \"fieldType\": \"float\",\n },\n { \"fieldName\": \"classification\",\n \"fieldType\": \"string\",\n },\n { \"fieldName\": \"randomData\",\n \"fieldType\": \"float\",\n },\n ],\n \"iterationCount\": -1,\n }\n \n \n return expDesc", "response": "This method returns the search description for the given dictionary"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nencodes value into output array.", "response": "def encodeIntoArray(self, value, output):\n \"\"\" See method description in base.py \"\"\"\n denseInput = numpy.zeros(output.shape)\n try:\n denseInput[value] = 1\n except IndexError:\n if isinstance(value, numpy.ndarray):\n raise ValueError(\n \"Numpy array must have integer dtype but got {}\".format(\n value.dtype))\n raise\n super(SparsePassThroughEncoder, self).encodeIntoArray(denseInput, output)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading serialized object from file.", "response": "def readFromFile(cls, f, packed=True):\n \"\"\"\n Read serialized object from file.\n\n :param f: input file\n :param packed: If true, will assume content is packed\n :return: first-class instance initialized from proto obj\n \"\"\"\n # Get capnproto schema from instance\n schema = cls.getSchema()\n\n # Read from file\n if packed:\n proto = schema.read_packed(f)\n else:\n proto = schema.read(f)\n\n # Return first-class instance initialized from proto obj\n return cls.read(proto)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef writeToFile(self, f, packed=True):\n # Get capnproto schema from instance\n schema = self.getSchema()\n\n # Construct new message, otherwise refered to as `proto`\n proto = schema.new_message()\n\n # Populate message w/ `write()` instance method\n self.write(proto)\n\n # Finally, write to file\n if packed:\n proto.write_packed(f)\n else:\n proto.write(f)", "response": "Writes serialized object to file."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread a TwoGramModel from a proto message.", "response": "def read(cls, proto):\n \"\"\"\n :param proto: capnp TwoGramModelProto message reader\n \"\"\"\n instance = object.__new__(cls)\n super(TwoGramModel, instance).__init__(proto=proto.modelBase)\n\n instance._logger = opf_utils.initLogger(instance)\n\n instance._reset = proto.reset\n instance._hashToValueDict = {x.hash: x.value\n for x in proto.hashToValueDict}\n instance._learningEnabled = proto.learningEnabled\n instance._encoder = encoders.MultiEncoder.read(proto.encoder)\n instance._fieldNames = instance._encoder.getScalarNames()\n instance._prevValues = list(proto.prevValues)\n instance._twoGramDicts = [dict() for _ in xrange(len(proto.twoGramDicts))]\n for idx, field in enumerate(proto.twoGramDicts):\n for entry in field:\n prev = None if entry.value == -1 else entry.value\n instance._twoGramDicts[idx][prev] = collections.defaultdict(int)\n for bucket in entry.buckets:\n instance._twoGramDicts[idx][prev][bucket.index] = bucket.count\n\n return instance"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrite the model to the proto message builder", "response": "def write(self, proto):\n \"\"\"\n :param proto: capnp TwoGramModelProto message builder\n \"\"\"\n super(TwoGramModel, self).writeBaseToProto(proto.modelBase)\n\n proto.reset = self._reset\n proto.learningEnabled = self._learningEnabled\n proto.prevValues = self._prevValues\n self._encoder.write(proto.encoder)\n proto.hashToValueDict = [{\"hash\": h, \"value\": v}\n for h, v in self._hashToValueDict.items()]\n\n twoGramDicts = []\n for items in self._twoGramDicts:\n twoGramArr = []\n for prev, values in items.iteritems():\n buckets = [{\"index\": index, \"count\": count}\n for index, count in values.iteritems()]\n if prev is None:\n prev = -1\n twoGramArr.append({\"value\": prev, \"buckets\": buckets})\n\n twoGramDicts.append(twoGramArr)\n\n proto.twoGramDicts = twoGramDicts"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef requireAnomalyModel(func):\n @wraps(func)\n def _decorator(self, *args, **kwargs):\n if not self.getInferenceType() == InferenceType.TemporalAnomaly:\n raise RuntimeError(\"Method required a TemporalAnomaly model.\")\n if self._getAnomalyClassifier() is None:\n raise RuntimeError(\"Model does not support this command. Model must\"\n \"be an active anomalyDetector model.\")\n return func(self, *args, **kwargs)\n return _decorator", "response": "Decorator for functions that require anomaly models."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nremoves labels from the anomaly classifier within this model.", "response": "def anomalyRemoveLabels(self, start, end, labelFilter):\n \"\"\"\n Remove labels from the anomaly classifier within this model. Removes all\n records if ``labelFilter==None``, otherwise only removes the labels equal to\n ``labelFilter``.\n\n :param start: (int) index to start removing labels\n :param end: (int) index to end removing labels\n :param labelFilter: (string) If specified, only removes records that match\n \"\"\"\n self._getAnomalyClassifier().getSelf().removeLabels(start, end, labelFilter)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a label to the anomaly classifier within this model.", "response": "def anomalyAddLabel(self, start, end, labelName):\n \"\"\"\n Add labels from the anomaly classifier within this model.\n\n :param start: (int) index to start label\n :param end: (int) index to end label\n :param labelName: (string) name of label\n \"\"\"\n self._getAnomalyClassifier().getSelf().addLabel(start, end, labelName)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef anomalyGetLabels(self, start, end):\n return self._getAnomalyClassifier().getSelf().getLabels(start, end)", "response": "Get the labels from the anomaly classifier within this model."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a SensorInput object which represents the parsed representation of the input record", "response": "def _getSensorInputRecord(self, inputRecord):\n \"\"\"\n inputRecord - dict containing the input to the sensor\n\n Return a 'SensorInput' object, which represents the 'parsed'\n representation of the input record\n \"\"\"\n sensor = self._getSensorRegion()\n dataRow = copy.deepcopy(sensor.getSelf().getOutputValues('sourceOut'))\n dataDict = copy.deepcopy(inputRecord)\n inputRecordEncodings = sensor.getSelf().getOutputValues('sourceEncodings')\n inputRecordCategory = int(sensor.getOutputData('categoryOut')[0])\n resetOut = sensor.getOutputData('resetOut')[0]\n\n return SensorInput(dataRow=dataRow,\n dataDict=dataDict,\n dataEncodings=inputRecordEncodings,\n sequenceReset=resetOut,\n category=inputRecordCategory)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a ClassifierInput object which contains the value of the predicted field in the inputRecord", "response": "def _getClassifierInputRecord(self, inputRecord):\n \"\"\"\n inputRecord - dict containing the input to the sensor\n\n Return a 'ClassifierInput' object, which contains the mapped\n bucket index for input Record\n \"\"\"\n absoluteValue = None\n bucketIdx = None\n\n if self._predictedFieldName is not None and self._classifierInputEncoder is not None:\n absoluteValue = inputRecord[self._predictedFieldName]\n bucketIdx = self._classifierInputEncoder.getBucketIndices(absoluteValue)[0]\n\n return ClassifierInput(dataRow=absoluteValue,\n bucketIndex=bucketIdx)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _anomalyCompute(self):\n inferenceType = self.getInferenceType()\n\n inferences = {}\n sp = self._getSPRegion()\n score = None\n if inferenceType == InferenceType.NontemporalAnomaly:\n score = sp.getOutputData(\"anomalyScore\")[0] #TODO move from SP to Anomaly ?\n\n elif inferenceType == InferenceType.TemporalAnomaly:\n tm = self._getTPRegion()\n\n if sp is not None:\n activeColumns = sp.getOutputData(\"bottomUpOut\").nonzero()[0]\n else:\n sensor = self._getSensorRegion()\n activeColumns = sensor.getOutputData('dataOut').nonzero()[0]\n\n if not self._predictedFieldName in self._input:\n raise ValueError(\n \"Expected predicted field '%s' in input row, but was not found!\"\n % self._predictedFieldName\n )\n # Calculate the anomaly score using the active columns\n # and previous predicted columns.\n score = tm.getOutputData(\"anomalyScore\")[0]\n\n # Calculate the classifier's output and use the result as the anomaly\n # label. Stores as string of results.\n\n # TODO: make labels work with non-SP models\n if sp is not None:\n self._getAnomalyClassifier().setParameter(\n \"activeColumnCount\", len(activeColumns))\n self._getAnomalyClassifier().prepareInputs()\n self._getAnomalyClassifier().compute()\n\n labels = self._getAnomalyClassifier().getSelf().getLabelResults()\n inferences[InferenceElement.anomalyLabel] = \"%s\" % labels\n\n inferences[InferenceElement.anomalyScore] = score\n return inferences", "response": "Compute anomaly score for the current anomaly in the current region."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _handleSDRClassifierMultiStep(self, patternNZ,\n inputTSRecordIdx,\n rawInput):\n \"\"\" Handle the CLA Classifier compute logic when implementing multi-step\n prediction. This is where the patternNZ is associated with one of the\n other fields from the dataset 0 to N steps in the future. This method is\n used by each type of network (encoder only, SP only, SP +TM) to handle the\n compute logic through the CLA Classifier. It fills in the inference dict with\n the results of the compute.\n\n Parameters:\n -------------------------------------------------------------------\n patternNZ: The input to the CLA Classifier as a list of active input indices\n inputTSRecordIdx: The index of the record as computed from the timestamp\n and aggregation interval. This normally increments by 1\n each time unless there are missing records. If there is no\n aggregation interval or timestamp in the data, this will be\n None.\n rawInput: The raw input to the sensor, as a dict.\n \"\"\"\n inferenceArgs = self.getInferenceArgs()\n predictedFieldName = inferenceArgs.get('predictedField', None)\n if predictedFieldName is None:\n raise ValueError(\n \"No predicted field was enabled! Did you call enableInference()?\"\n )\n self._predictedFieldName = predictedFieldName\n\n classifier = self._getClassifierRegion()\n if not self._hasCL or classifier is None:\n # No classifier so return an empty dict for inferences.\n return {}\n\n sensor = self._getSensorRegion()\n minLikelihoodThreshold = self._minLikelihoodThreshold\n maxPredictionsPerStep = self._maxPredictionsPerStep\n needLearning = self.isLearningEnabled()\n inferences = {}\n\n # Get the classifier input encoder, if we don't have it already\n if self._classifierInputEncoder is None:\n if predictedFieldName is None:\n raise RuntimeError(\"This experiment description is missing \"\n \"the 'predictedField' in its config, which is required \"\n \"for multi-step prediction inference.\")\n\n encoderList = sensor.getSelf().encoder.getEncoderList()\n self._numFields = len(encoderList)\n\n # This is getting index of predicted field if being fed to CLA.\n fieldNames = sensor.getSelf().encoder.getScalarNames()\n if predictedFieldName in fieldNames:\n self._predictedFieldIdx = fieldNames.index(predictedFieldName)\n else:\n # Predicted field was not fed into the network, only to the classifier\n self._predictedFieldIdx = None\n\n # In a multi-step model, the classifier input encoder is separate from\n # the other encoders and always disabled from going into the bottom of\n # the network.\n if sensor.getSelf().disabledEncoder is not None:\n encoderList = sensor.getSelf().disabledEncoder.getEncoderList()\n else:\n encoderList = []\n if len(encoderList) >= 1:\n fieldNames = sensor.getSelf().disabledEncoder.getScalarNames()\n self._classifierInputEncoder = encoderList[fieldNames.index(\n predictedFieldName)]\n else:\n # Legacy multi-step networks don't have a separate encoder for the\n # classifier, so use the one that goes into the bottom of the network\n encoderList = sensor.getSelf().encoder.getEncoderList()\n self._classifierInputEncoder = encoderList[self._predictedFieldIdx]\n\n\n\n # Get the actual value and the bucket index for this sample. The\n # predicted field may not be enabled for input to the network, so we\n # explicitly encode it outside of the sensor\n # TODO: All this logic could be simpler if in the encoder itself\n if not predictedFieldName in rawInput:\n raise ValueError(\"Input row does not contain a value for the predicted \"\n \"field configured for this model. Missing value for '%s'\"\n % predictedFieldName)\n absoluteValue = rawInput[predictedFieldName]\n bucketIdx = self._classifierInputEncoder.getBucketIndices(absoluteValue)[0]\n\n # Convert the absolute values to deltas if necessary\n # The bucket index should be handled correctly by the underlying delta encoder\n if isinstance(self._classifierInputEncoder, DeltaEncoder):\n # Make the delta before any values have been seen 0 so that we do not mess up the\n # range for the adaptive scalar encoder.\n if not hasattr(self,\"_ms_prevVal\"):\n self._ms_prevVal = absoluteValue\n prevValue = self._ms_prevVal\n self._ms_prevVal = absoluteValue\n actualValue = absoluteValue - prevValue\n else:\n actualValue = absoluteValue\n\n if isinstance(actualValue, float) and math.isnan(actualValue):\n actualValue = SENTINEL_VALUE_FOR_MISSING_DATA\n\n\n # Pass this information to the classifier's custom compute method\n # so that it can assign the current classification to possibly\n # multiple patterns from the past and current, and also provide\n # the expected classification for some time step(s) in the future.\n classifier.setParameter('inferenceMode', True)\n classifier.setParameter('learningMode', needLearning)\n classificationIn = {'bucketIdx': bucketIdx,\n 'actValue': actualValue}\n\n # Handle missing records\n if inputTSRecordIdx is not None:\n recordNum = inputTSRecordIdx\n else:\n recordNum = self.__numRunCalls\n clResults = classifier.getSelf().customCompute(recordNum=recordNum,\n patternNZ=patternNZ,\n classification=classificationIn)\n\n # ---------------------------------------------------------------\n # Get the prediction for every step ahead learned by the classifier\n predictionSteps = classifier.getParameter('steps')\n predictionSteps = [int(x) for x in predictionSteps.split(',')]\n\n # We will return the results in this dict. The top level keys\n # are the step number, the values are the relative likelihoods for\n # each classification value in that time step, represented as\n # another dict where the keys are the classification values and\n # the values are the relative likelihoods.\n inferences[InferenceElement.multiStepPredictions] = dict()\n inferences[InferenceElement.multiStepBestPredictions] = dict()\n inferences[InferenceElement.multiStepBucketLikelihoods] = dict()\n\n\n # ======================================================================\n # Plug in the predictions for each requested time step.\n for steps in predictionSteps:\n # From the clResults, compute the predicted actual value. The\n # SDRClassifier classifies the bucket index and returns a list of\n # relative likelihoods for each bucket. Let's find the max one\n # and then look up the actual value from that bucket index\n likelihoodsVec = clResults[steps]\n bucketValues = clResults['actualValues']\n\n # Create a dict of value:likelihood pairs. We can't simply use\n # dict(zip(bucketValues, likelihoodsVec)) because there might be\n # duplicate bucketValues (this happens early on in the model when\n # it doesn't have actual values for each bucket so it returns\n # multiple buckets with the same default actual value).\n likelihoodsDict = dict()\n bestActValue = None\n bestProb = None\n for (actValue, prob) in zip(bucketValues, likelihoodsVec):\n if actValue in likelihoodsDict:\n likelihoodsDict[actValue] += prob\n else:\n likelihoodsDict[actValue] = prob\n # Keep track of best\n if bestProb is None or likelihoodsDict[actValue] > bestProb:\n bestProb = likelihoodsDict[actValue]\n bestActValue = actValue\n\n\n # Remove entries with 0 likelihood or likelihood less than\n # minLikelihoodThreshold, but don't leave an empty dict.\n likelihoodsDict = HTMPredictionModel._removeUnlikelyPredictions(\n likelihoodsDict, minLikelihoodThreshold, maxPredictionsPerStep)\n\n # calculate likelihood for each bucket\n bucketLikelihood = {}\n for k in likelihoodsDict.keys():\n bucketLikelihood[self._classifierInputEncoder.getBucketIndices(k)[0]] = (\n likelihoodsDict[k])\n\n # ---------------------------------------------------------------------\n # If we have a delta encoder, we have to shift our predicted output value\n # by the sum of the deltas\n if isinstance(self._classifierInputEncoder, DeltaEncoder):\n # Get the prediction history for this number of timesteps.\n # The prediction history is a store of the previous best predicted values.\n # This is used to get the final shift from the current absolute value.\n if not hasattr(self, '_ms_predHistories'):\n self._ms_predHistories = dict()\n predHistories = self._ms_predHistories\n if not steps in predHistories:\n predHistories[steps] = deque()\n predHistory = predHistories[steps]\n\n # Find the sum of the deltas for the steps and use this to generate\n # an offset from the current absolute value\n sumDelta = sum(predHistory)\n offsetDict = dict()\n for (k, v) in likelihoodsDict.iteritems():\n if k is not None:\n # Reconstruct the absolute value based on the current actual value,\n # the best predicted values from the previous iterations,\n # and the current predicted delta\n offsetDict[absoluteValue+float(k)+sumDelta] = v\n\n # calculate likelihood for each bucket\n bucketLikelihoodOffset = {}\n for k in offsetDict.keys():\n bucketLikelihoodOffset[self._classifierInputEncoder.getBucketIndices(k)[0]] = (\n offsetDict[k])\n\n\n # Push the current best delta to the history buffer for reconstructing the final delta\n if bestActValue is not None:\n predHistory.append(bestActValue)\n # If we don't need any more values in the predictionHistory, pop off\n # the earliest one.\n if len(predHistory) >= steps:\n predHistory.popleft()\n\n # Provide the offsetDict as the return value\n if len(offsetDict)>0:\n inferences[InferenceElement.multiStepPredictions][steps] = offsetDict\n inferences[InferenceElement.multiStepBucketLikelihoods][steps] = bucketLikelihoodOffset\n else:\n inferences[InferenceElement.multiStepPredictions][steps] = likelihoodsDict\n inferences[InferenceElement.multiStepBucketLikelihoods][steps] = bucketLikelihood\n\n if bestActValue is None:\n inferences[InferenceElement.multiStepBestPredictions][steps] = None\n else:\n inferences[InferenceElement.multiStepBestPredictions][steps] = (\n absoluteValue + sumDelta + bestActValue)\n\n # ---------------------------------------------------------------------\n # Normal case, no delta encoder. Just plug in all our multi-step predictions\n # with likelihoods as well as our best prediction\n else:\n # The multiStepPredictions element holds the probabilities for each\n # bucket\n inferences[InferenceElement.multiStepPredictions][steps] = (\n likelihoodsDict)\n inferences[InferenceElement.multiStepBestPredictions][steps] = (\n bestActValue)\n inferences[InferenceElement.multiStepBucketLikelihoods][steps] = (\n bucketLikelihood)\n\n\n return inferences", "response": "This method handles the CLA Classifier multi - step inference logic when implementing multi - step prediction."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _removeUnlikelyPredictions(cls, likelihoodsDict, minLikelihoodThreshold,\n maxPredictionsPerStep):\n \"\"\"Remove entries with 0 likelihood or likelihood less than\n minLikelihoodThreshold, but don't leave an empty dict.\n \"\"\"\n maxVal = (None, None)\n for (k, v) in likelihoodsDict.items():\n if len(likelihoodsDict) <= 1:\n break\n if maxVal[0] is None or v >= maxVal[1]:\n if maxVal[0] is not None and maxVal[1] < minLikelihoodThreshold:\n del likelihoodsDict[maxVal[0]]\n maxVal = (k, v)\n elif v < minLikelihoodThreshold:\n del likelihoodsDict[k]\n # Limit the number of predictions to include.\n likelihoodsDict = dict(sorted(likelihoodsDict.iteritems(),\n key=itemgetter(1),\n reverse=True)[:maxPredictionsPerStep])\n return likelihoodsDict", "response": "Remove entries with 0 likelihood or likelihood less than minLikelihoodThreshold but don t leave an empty dict."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getRuntimeStats(self):\n ret = {\"numRunCalls\" : self.__numRunCalls}\n\n #--------------------------------------------------\n # Query temporal network stats\n temporalStats = dict()\n if self._hasTP:\n for stat in self._netInfo.statsCollectors:\n sdict = stat.getStats()\n temporalStats.update(sdict)\n\n ret[InferenceType.getLabel(InferenceType.TemporalNextStep)] = temporalStats\n\n\n return ret", "response": "Returns a dictionary of stats for a stat called numRunCalls."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the Classifier region in the network", "response": "def _getClassifierRegion(self):\n \"\"\"\n Returns reference to the network's Classifier region\n \"\"\"\n if (self._netInfo.net is not None and\n \"Classifier\" in self._netInfo.net.regions):\n return self._netInfo.net.regions[\"Classifier\"]\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a CLA network and return it.", "response": "def __createHTMNetwork(self, sensorParams, spEnable, spParams, tmEnable,\n tmParams, clEnable, clParams, anomalyParams):\n \"\"\" Create a CLA network and return it.\n\n description: HTMPredictionModel description dictionary (TODO: define schema)\n Returns: NetworkInfo instance;\n \"\"\"\n\n #--------------------------------------------------\n # Create the network\n n = Network()\n\n\n #--------------------------------------------------\n # Add the Sensor\n n.addRegion(\"sensor\", \"py.RecordSensor\", json.dumps(dict(verbosity=sensorParams['verbosity'])))\n sensor = n.regions['sensor'].getSelf()\n\n enabledEncoders = copy.deepcopy(sensorParams['encoders'])\n for name, params in enabledEncoders.items():\n if params is not None:\n classifierOnly = params.pop('classifierOnly', False)\n if classifierOnly:\n enabledEncoders.pop(name)\n\n # Disabled encoders are encoders that are fed to SDRClassifierRegion but not\n # SP or TM Regions. This is to handle the case where the predicted field\n # is not fed through the SP/TM. We typically just have one of these now.\n disabledEncoders = copy.deepcopy(sensorParams['encoders'])\n for name, params in disabledEncoders.items():\n if params is None:\n disabledEncoders.pop(name)\n else:\n classifierOnly = params.pop('classifierOnly', False)\n if not classifierOnly:\n disabledEncoders.pop(name)\n\n encoder = MultiEncoder(enabledEncoders)\n\n sensor.encoder = encoder\n sensor.disabledEncoder = MultiEncoder(disabledEncoders)\n sensor.dataSource = DataBuffer()\n\n prevRegion = \"sensor\"\n prevRegionWidth = encoder.getWidth()\n\n # SP is not enabled for spatial classification network\n if spEnable:\n spParams = spParams.copy()\n spParams['inputWidth'] = prevRegionWidth\n self.__logger.debug(\"Adding SPRegion; spParams: %r\" % spParams)\n n.addRegion(\"SP\", \"py.SPRegion\", json.dumps(spParams))\n\n # Link SP region\n n.link(\"sensor\", \"SP\", \"UniformLink\", \"\")\n n.link(\"sensor\", \"SP\", \"UniformLink\", \"\", srcOutput=\"resetOut\",\n destInput=\"resetIn\")\n\n n.link(\"SP\", \"sensor\", \"UniformLink\", \"\", srcOutput=\"spatialTopDownOut\",\n destInput=\"spatialTopDownIn\")\n n.link(\"SP\", \"sensor\", \"UniformLink\", \"\", srcOutput=\"temporalTopDownOut\",\n destInput=\"temporalTopDownIn\")\n\n prevRegion = \"SP\"\n prevRegionWidth = spParams['columnCount']\n\n if tmEnable:\n tmParams = tmParams.copy()\n if prevRegion == 'sensor':\n tmParams['inputWidth'] = tmParams['columnCount'] = prevRegionWidth\n else:\n assert tmParams['columnCount'] == prevRegionWidth\n tmParams['inputWidth'] = tmParams['columnCount']\n\n self.__logger.debug(\"Adding TMRegion; tmParams: %r\" % tmParams)\n n.addRegion(\"TM\", \"py.TMRegion\", json.dumps(tmParams))\n\n # Link TM region\n n.link(prevRegion, \"TM\", \"UniformLink\", \"\")\n if prevRegion != \"sensor\":\n n.link(\"TM\", prevRegion, \"UniformLink\", \"\", srcOutput=\"topDownOut\",\n destInput=\"topDownIn\")\n else:\n n.link(\"TM\", prevRegion, \"UniformLink\", \"\", srcOutput=\"topDownOut\",\n destInput=\"temporalTopDownIn\")\n n.link(\"sensor\", \"TM\", \"UniformLink\", \"\", srcOutput=\"resetOut\",\n destInput=\"resetIn\")\n\n prevRegion = \"TM\"\n prevRegionWidth = tmParams['inputWidth']\n\n if clEnable and clParams is not None:\n clParams = clParams.copy()\n clRegionName = clParams.pop('regionName')\n self.__logger.debug(\"Adding %s; clParams: %r\" % (clRegionName,\n clParams))\n n.addRegion(\"Classifier\", \"py.%s\" % str(clRegionName), json.dumps(clParams))\n\n # SDR Classifier-specific links\n if str(clRegionName) == \"SDRClassifierRegion\":\n n.link(\"sensor\", \"Classifier\", \"UniformLink\", \"\", srcOutput=\"actValueOut\",\n destInput=\"actValueIn\")\n n.link(\"sensor\", \"Classifier\", \"UniformLink\", \"\", srcOutput=\"bucketIdxOut\",\n destInput=\"bucketIdxIn\")\n\n # This applies to all (SDR and KNN) classifiers\n n.link(\"sensor\", \"Classifier\", \"UniformLink\", \"\", srcOutput=\"categoryOut\",\n destInput=\"categoryIn\")\n\n n.link(prevRegion, \"Classifier\", \"UniformLink\", \"\")\n\n if self.getInferenceType() == InferenceType.TemporalAnomaly:\n anomalyClParams = dict(\n trainRecords=anomalyParams.get('autoDetectWaitRecords', None),\n cacheSize=anomalyParams.get('anomalyCacheRecords', None)\n )\n self._addAnomalyClassifierRegion(n, anomalyClParams, spEnable, tmEnable)\n\n #--------------------------------------------------\n # NuPIC doesn't initialize the network until you try to run it\n # but users may want to access components in a setup callback\n n.initialize()\n\n return NetworkInfo(net=n, statsCollectors=[])"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef write(self, proto):\n super(HTMPredictionModel, self).writeBaseToProto(proto.modelBase)\n\n proto.numRunCalls = self.__numRunCalls\n proto.minLikelihoodThreshold = self._minLikelihoodThreshold\n proto.maxPredictionsPerStep = self._maxPredictionsPerStep\n\n self._netInfo.net.write(proto.network)\n proto.spLearningEnabled = self.__spLearningEnabled\n proto.tpLearningEnabled = self.__tpLearningEnabled\n if self._predictedFieldIdx is None:\n proto.predictedFieldIdx.none = None\n else:\n proto.predictedFieldIdx.value = self._predictedFieldIdx\n if self._predictedFieldName is None:\n proto.predictedFieldName.none = None\n else:\n proto.predictedFieldName.value = self._predictedFieldName\n if self._numFields is None:\n proto.numFields.none = None\n else:\n proto.numFields.value = self._numFields\n proto.trainSPNetOnlyIfRequested = self.__trainSPNetOnlyIfRequested\n proto.finishedLearning = self.__finishedLearning", "response": "writes the current state of this object to the specified proto message builder"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read(cls, proto):\n obj = object.__new__(cls)\n # model.capnp\n super(HTMPredictionModel, obj).__init__(proto=proto.modelBase)\n\n # HTMPredictionModelProto.capnp\n obj._minLikelihoodThreshold = round(proto.minLikelihoodThreshold,\n EPSILON_ROUND)\n obj._maxPredictionsPerStep = proto.maxPredictionsPerStep\n\n network = Network.read(proto.network)\n obj._hasSP = (\"SP\" in network.regions)\n obj._hasTP = (\"TM\" in network.regions)\n obj._hasCL = (\"Classifier\" in network.regions)\n obj._netInfo = NetworkInfo(net=network, statsCollectors=[])\n\n obj.__spLearningEnabled = bool(proto.spLearningEnabled)\n obj.__tpLearningEnabled = bool(proto.tpLearningEnabled)\n obj.__numRunCalls = proto.numRunCalls\n\n obj._classifierInputEncoder = None\n if proto.predictedFieldIdx.which() == \"none\":\n obj._predictedFieldIdx = None\n else:\n obj._predictedFieldIdx = proto.predictedFieldIdx.value\n if proto.predictedFieldName.which() == \"none\":\n obj._predictedFieldName = None\n else:\n obj._predictedFieldName = proto.predictedFieldName.value\n obj._numFields = proto.numFields\n if proto.numFields.which() == \"none\":\n obj._numFields = None\n else:\n obj._numFields = proto.numFields.value\n obj.__trainSPNetOnlyIfRequested = proto.trainSPNetOnlyIfRequested\n obj.__finishedLearning = proto.finishedLearning\n obj._input = None\n sensor = network.regions['sensor'].getSelf()\n sensor.dataSource = DataBuffer()\n network.initialize()\n\n\n obj.__logger = initLogger(obj)\n obj.__logger.debug(\"Instantiating %s.\" % obj.__myClassName)\n\n # Mark end of restoration from state\n obj.__restoringFromState = False\n obj.__restoringFromV1 = False\n\n return obj", "response": "Reads an HTMPredictionModel from a proto message."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _deSerializeExtraData(self, extraDataDir):\n assert self.__restoringFromState\n\n #--------------------------------------------------\n # Check to make sure that our Network member wasn't restored from\n # serialized data\n assert (self._netInfo.net is None), \"Network was already unpickled\"\n\n #--------------------------------------------------\n # Restore the network\n stateDir = self.__getNetworkStateDirectory(extraDataDir=extraDataDir)\n\n self.__logger.debug(\n \"(%s) De-serializing network...\", self)\n\n self._netInfo.net = Network(stateDir)\n\n self.__logger.debug(\n \"(%s) Finished de-serializing network\", self)\n\n\n # NuPIC doesn't initialize the network until you try to run it\n # but users may want to access components in a setup callback\n self._netInfo.net.initialize()\n\n\n # Used for backwards compatibility for anomaly classification models.\n # Previous versions used the HTMPredictionModelClassifierHelper class for utilizing\n # the KNN classifier. Current version uses KNNAnomalyClassifierRegion to\n # encapsulate all the classifier functionality.\n if self.getInferenceType() == InferenceType.TemporalAnomaly:\n classifierType = self._getAnomalyClassifier().getSelf().__class__.__name__\n if classifierType is 'KNNClassifierRegion':\n\n anomalyClParams = dict(\n trainRecords=self._classifier_helper._autoDetectWaitRecords,\n cacheSize=self._classifier_helper._history_length,\n )\n\n spEnable = (self._getSPRegion() is not None)\n tmEnable = True\n\n # Store original KNN region\n knnRegion = self._getAnomalyClassifier().getSelf()\n\n # Add new KNNAnomalyClassifierRegion\n self._addAnomalyClassifierRegion(self._netInfo.net, anomalyClParams,\n spEnable, tmEnable)\n\n # Restore state\n self._getAnomalyClassifier().getSelf()._iteration = self.__numRunCalls\n self._getAnomalyClassifier().getSelf()._recordsCache = (\n self._classifier_helper.saved_states)\n self._getAnomalyClassifier().getSelf().saved_categories = (\n self._classifier_helper.saved_categories)\n self._getAnomalyClassifier().getSelf()._knnclassifier = knnRegion\n\n # Set TM to output neccessary information\n self._getTPRegion().setParameter('anomalyMode', True)\n\n # Remove old classifier_helper\n del self._classifier_helper\n\n self._netInfo.net.initialize()\n\n #--------------------------------------------------\n # Mark end of restoration from state\n self.__restoringFromState = False\n\n self.__logger.debug(\"(%s) Finished restoring from state\", self)\n\n return", "response": "This method de - serialize the extra data of the current model into a new NuPIC structure."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _addAnomalyClassifierRegion(self, network, params, spEnable, tmEnable):\n\n allParams = copy.deepcopy(params)\n knnParams = dict(k=1,\n distanceMethod='rawOverlap',\n distanceNorm=1,\n doBinarization=1,\n replaceDuplicates=0,\n maxStoredPatterns=1000)\n allParams.update(knnParams)\n\n # Set defaults if not set\n if allParams['trainRecords'] is None:\n allParams['trainRecords'] = DEFAULT_ANOMALY_TRAINRECORDS\n\n if allParams['cacheSize'] is None:\n allParams['cacheSize'] = DEFAULT_ANOMALY_CACHESIZE\n\n # Remove current instance if already created (used for deserializing)\n if self._netInfo is not None and self._netInfo.net is not None \\\n and self._getAnomalyClassifier() is not None:\n self._netInfo.net.removeRegion('AnomalyClassifier')\n\n network.addRegion(\"AnomalyClassifier\",\n \"py.KNNAnomalyClassifierRegion\",\n json.dumps(allParams))\n\n # Attach link to SP\n if spEnable:\n network.link(\"SP\", \"AnomalyClassifier\", \"UniformLink\", \"\",\n srcOutput=\"bottomUpOut\", destInput=\"spBottomUpOut\")\n else:\n network.link(\"sensor\", \"AnomalyClassifier\", \"UniformLink\", \"\",\n srcOutput=\"dataOut\", destInput=\"spBottomUpOut\")\n\n # Attach link to TM\n if tmEnable:\n network.link(\"TM\", \"AnomalyClassifier\", \"UniformLink\", \"\",\n srcOutput=\"topDownOut\", destInput=\"tpTopDownOut\")\n network.link(\"TM\", \"AnomalyClassifier\", \"UniformLink\", \"\",\n srcOutput=\"lrnActiveStateT\", destInput=\"tpLrnActiveStateT\")\n else:\n raise RuntimeError(\"TemporalAnomaly models require a TM region.\")", "response": "Adds an AnomalyClassifier region to the network."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __getNetworkStateDirectory(self, extraDataDir):\n if self.__restoringFromV1:\n if self.getInferenceType() == InferenceType.TemporalNextStep:\n leafName = 'temporal'+ \"-network.nta\"\n else:\n leafName = 'nonTemporal'+ \"-network.nta\"\n else:\n leafName = InferenceType.getLabel(self.getInferenceType()) + \"-network.nta\"\n path = os.path.join(extraDataDir, leafName)\n path = os.path.abspath(path)\n return path", "response": "Returns the absolute path to the network state directory for saving CLA Network."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __manglePrivateMemberName(self, privateMemberName, skipCheck=False):\n\n assert privateMemberName.startswith(\"__\"), \\\n \"%r doesn't start with __\" % privateMemberName\n assert not privateMemberName.startswith(\"___\"), \\\n \"%r starts with ___\" % privateMemberName\n assert not privateMemberName.endswith(\"__\"), \\\n \"%r ends with more than one underscore\" % privateMemberName\n\n realName = \"_\" + (self.__myClassName).lstrip(\"_\") + privateMemberName\n\n if not skipCheck:\n # This will throw an exception if the member is missing\n getattr(self, realName)\n\n return realName", "response": "Mangles the given private member name into a single unmangled member name."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset the radius resolution and range of the encoder.", "response": "def _setEncoderParams(self):\n \"\"\"\n Set the radius, resolution and range. These values are updated when minval\n and/or maxval change.\n \"\"\"\n\n self.rangeInternal = float(self.maxval - self.minval)\n\n self.resolution = float(self.rangeInternal) / (self.n - self.w)\n self.radius = self.w * self.resolution\n self.range = self.rangeInternal + self.resolution\n\n # nInternal represents the output area excluding the possible padding on each side\n self.nInternal = self.n - 2 * self.padding\n\n # Invalidate the bucket values cache so that they get recomputed\n self._bucketValues = None"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef setFieldStats(self, fieldName, fieldStats):\n #If the stats are not fully formed, ignore.\n if fieldStats[fieldName]['min'] == None or \\\n fieldStats[fieldName]['max'] == None:\n return\n self.minval = fieldStats[fieldName]['min']\n self.maxval = fieldStats[fieldName]['max']\n if self.minval == self.maxval:\n self.maxval+=1\n self._setEncoderParams()", "response": "Sets the min and max values of the fieldStats for the specified fieldName."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\noverriding this method to set the min and max values for the next bucket.", "response": "def getBucketIndices(self, input, learn=None):\n \"\"\"\n [overrides nupic.encoders.scalar.ScalarEncoder.getBucketIndices]\n \"\"\"\n\n self.recordNum +=1\n if learn is None:\n learn = self._learningEnabled\n\n if type(input) is float and math.isnan(input):\n input = SENTINEL_VALUE_FOR_MISSING_DATA\n\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n return [None]\n else:\n self._setMinAndMax(input, learn)\n return super(AdaptiveScalarEncoder, self).getBucketIndices(input)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef encodeIntoArray(self, input, output,learn=None):\n\n self.recordNum +=1\n if learn is None:\n learn = self._learningEnabled\n if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n output[0:self.n] = 0\n elif not math.isnan(input):\n self._setMinAndMax(input, learn)\n\n super(AdaptiveScalarEncoder, self).encodeIntoArray(input, output)", "response": "Override encodeIntoArray to set the min and max values for the next record."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\noverriding this method to return encoderResult for multi - valued buckets.", "response": "def getBucketInfo(self, buckets):\n \"\"\"\n [overrides nupic.encoders.scalar.ScalarEncoder.getBucketInfo]\n \"\"\"\n\n if self.minval is None or self.maxval is None:\n return [EncoderResult(value=0, scalar=0,\n encoding=numpy.zeros(self.n))]\n\n return super(AdaptiveScalarEncoder, self).getBucketInfo(buckets)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef topDownCompute(self, encoded):\n\n if self.minval is None or self.maxval is None:\n return [EncoderResult(value=0, scalar=0,\n encoding=numpy.zeros(self.n))]\n return super(AdaptiveScalarEncoder, self).topDownCompute(encoded)", "response": "Compute the top down of the encoder."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef recordDataPoint(self, swarmId, generation, errScore):\n terminatedSwarms = []\n\n # Append score to existing swarm.\n if swarmId in self.swarmScores:\n entry = self.swarmScores[swarmId]\n assert(len(entry) == generation)\n entry.append(errScore)\n\n entry = self.swarmBests[swarmId]\n entry.append(min(errScore, entry[-1]))\n\n assert(len(self.swarmBests[swarmId]) == len(self.swarmScores[swarmId]))\n else:\n # Create list of scores for a new swarm\n assert (generation == 0)\n self.swarmScores[swarmId] = [errScore]\n self.swarmBests[swarmId] = [errScore]\n\n # If the current swarm hasn't completed at least MIN_GENERATIONS, it should\n # not be candidate for maturation or termination. This prevents the initial\n # allocation of particles in PSO from killing off a field combination too\n # early.\n if generation + 1 < self.MATURITY_WINDOW:\n return terminatedSwarms\n\n # If the swarm has completed more than MAX_GENERATIONS, it should be marked\n # as mature, regardless of how its value is changing.\n if self.MAX_GENERATIONS is not None and generation > self.MAX_GENERATIONS:\n self._logger.info(\n 'Swarm %s has matured (more than %d generations). Stopping' %\n (swarmId, self.MAX_GENERATIONS))\n terminatedSwarms.append(swarmId)\n\n if self._isTerminationEnabled:\n terminatedSwarms.extend(self._getTerminatedSwarms(generation))\n\n # Return which swarms to kill when we've reached maturity\n # If there is no change in the swarm's best for some time,\n # Mark it dead\n cumulativeBestScores = self.swarmBests[swarmId]\n if cumulativeBestScores[-1] == cumulativeBestScores[-self.MATURITY_WINDOW]:\n self._logger.info('Swarm %s has matured (no change in %d generations).'\n 'Stopping...'% (swarmId, self.MATURITY_WINDOW))\n terminatedSwarms.append(swarmId)\n\n self.terminatedSwarms = self.terminatedSwarms.union(terminatedSwarms)\n return terminatedSwarms", "response": "Record the best score for a given swarm. Returns a list of swarmIds to terminate when we reach the given generation index."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a dictionary of all the state of the current object.", "response": "def getState(self):\n \"\"\"See comments in base class.\"\"\"\n return dict(_position = self._position,\n position = self.getPosition(),\n velocity = self._velocity,\n bestPosition = self._bestPosition,\n bestResult = self._bestResult)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the state of the internal state of the internal state of the internal state.", "response": "def setState(self, state):\n \"\"\"See comments in base class.\"\"\"\n self._position = state['_position']\n self._velocity = state['velocity']\n self._bestPosition = state['bestPosition']\n self._bestResult = state['bestResult']"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getPosition(self):\n if self.stepSize is None:\n return self._position\n\n # Find nearest step\n numSteps = (self._position - self.min) / self.stepSize\n numSteps = int(round(numSteps))\n position = self.min + (numSteps * self.stepSize)\n position = max(self.min, position)\n position = min(self.max, position)\n return position", "response": "Returns the position of the next step in the sequence."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef agitate(self):\n # Increase velocity enough that it will be higher the next time\n # newPosition() is called. We know that newPosition multiplies by inertia,\n # so take that into account.\n self._velocity *= 1.5 / self._inertia\n\n # Clip velocity\n maxV = (self.max - self.min)/2\n if self._velocity > maxV:\n self._velocity = maxV\n elif self._velocity < -maxV:\n self._velocity = -maxV\n\n # if we at the max or min, reverse direction\n if self._position == self.max and self._velocity > 0:\n self._velocity *= -1\n if self._position == self.min and self._velocity < 0:\n self._velocity *= -1", "response": "This method is used to agitate the current time series."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef newPosition(self, globalBestPosition, rng):\n # First, update the velocity. The new velocity is given as:\n # v = (inertia * v) + (cogRate * r1 * (localBest-pos))\n # + (socRate * r2 * (globalBest-pos))\n #\n # where r1 and r2 are random numbers between 0 and 1.0\n lb=float(Configuration.get(\"nupic.hypersearch.randomLowerBound\"))\n ub=float(Configuration.get(\"nupic.hypersearch.randomUpperBound\"))\n\n self._velocity = (self._velocity * self._inertia + rng.uniform(lb, ub) *\n self._cogRate * (self._bestPosition - self.getPosition()))\n if globalBestPosition is not None:\n self._velocity += rng.uniform(lb, ub) * self._socRate * (\n globalBestPosition - self.getPosition())\n\n # update position based on velocity\n self._position += self._velocity\n\n # Clip it\n self._position = max(self.min, self._position)\n self._position = min(self.max, self._position)\n\n # Return it\n return self.getPosition()", "response": "Update the position based on the current position and the global best position."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npush a particle from another set of positions.", "response": "def pushAwayFrom(self, otherPositions, rng):\n \"\"\"See comments in base class.\"\"\"\n # If min and max are the same, nothing to do\n if self.max == self.min:\n return\n\n # How many potential other positions to evaluate?\n numPositions = len(otherPositions) * 4\n if numPositions == 0:\n return\n\n # Assign a weight to each potential position based on how close it is\n # to other particles.\n stepSize = float(self.max-self.min) / numPositions\n positions = numpy.arange(self.min, self.max + stepSize, stepSize)\n\n # Get rid of duplicates.\n numPositions = len(positions)\n weights = numpy.zeros(numPositions)\n\n # Assign a weight to each potential position, based on a gaussian falloff\n # from each existing variable. The weight of a variable to each potential\n # position is given as:\n # e ^ -(dist^2/stepSize^2)\n maxDistanceSq = -1 * (stepSize ** 2)\n for pos in otherPositions:\n distances = pos - positions\n varWeights = numpy.exp(numpy.power(distances, 2) / maxDistanceSq)\n weights += varWeights\n\n\n # Put this particle at the position with smallest weight.\n positionIdx = weights.argmin()\n self._position = positions[positionIdx]\n\n # Set its best position to this.\n self._bestPosition = self.getPosition()\n\n # Give it a random direction.\n self._velocity *= rng.choice([1, -1])"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nresets the velocity of the current node to a random value.", "response": "def resetVelocity(self, rng):\n \"\"\"See comments in base class.\"\"\"\n maxVelocity = (self.max - self.min) / 5.0\n self._velocity = maxVelocity #min(abs(self._velocity), maxVelocity)\n self._velocity *= rng.choice([1, -1])"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getPosition(self):\n position = super(PermuteInt, self).getPosition()\n position = int(round(position))\n return position", "response": "Returns the position of the permute integer."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getState(self):\n return dict(_position = self.getPosition(),\n position = self.getPosition(),\n velocity = None,\n bestPosition = self.choices[self._bestPositionIdx],\n bestResult = self._bestResult)", "response": "Returns a dictionary of state of the current page."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef setState(self, state):\n self._positionIdx = self.choices.index(state['_position'])\n self._bestPositionIdx = self.choices.index(state['bestPosition'])\n self._bestResult = state['bestResult']", "response": "Sets the state of the current selection."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef newPosition(self, globalBestPosition, rng):\n # Compute the mean score per choice.\n numChoices = len(self.choices)\n meanScorePerChoice = []\n overallSum = 0\n numResults = 0\n\n for i in range(numChoices):\n if len(self._resultsPerChoice[i]) > 0:\n data = numpy.array(self._resultsPerChoice[i])\n meanScorePerChoice.append(data.mean())\n overallSum += data.sum()\n numResults += data.size\n else:\n meanScorePerChoice.append(None)\n\n if numResults == 0:\n overallSum = 1.0\n numResults = 1\n\n # For any choices we don't have a result for yet, set to the overall mean.\n for i in range(numChoices):\n if meanScorePerChoice[i] is None:\n meanScorePerChoice[i] = overallSum / numResults\n\n # Now, pick a new choice based on the above probabilities. Note that the\n # best result is the lowest result. We want to make it more likely to\n # pick the choice that produced the lowest results. So, we need to invert\n # the scores (someLargeNumber - score).\n meanScorePerChoice = numpy.array(meanScorePerChoice)\n\n # Invert meaning.\n meanScorePerChoice = (1.1 * meanScorePerChoice.max()) - meanScorePerChoice\n\n # If you want the scores to quickly converge to the best choice, raise the\n # results to a power. This will cause lower scores to become lower\n # probability as you see more results, until it eventually should\n # assymptote to only choosing the best choice.\n if self._fixEarly:\n meanScorePerChoice **= (numResults * self._fixEarlyFactor / numChoices)\n # Normalize.\n total = meanScorePerChoice.sum()\n if total == 0:\n total = 1.0\n meanScorePerChoice /= total\n\n # Get distribution and choose one based on those probabilities.\n distribution = meanScorePerChoice.cumsum()\n r = rng.random() * distribution[-1]\n choiceIdx = numpy.where(r <= distribution)[0][0]\n\n self._positionIdx = choiceIdx\n return self.getPosition()", "response": "This method is used to set the best position of the new choice."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pushAwayFrom(self, otherPositions, rng):\n # Get the count of how many in each position\n positions = [self.choices.index(x) for x in otherPositions]\n\n positionCounts = [0] * len(self.choices)\n for pos in positions:\n positionCounts[pos] += 1\n\n self._positionIdx = numpy.array(positionCounts).argmin()\n self._bestPositionIdx = self._positionIdx", "response": "Pushes a way from another position to the current one."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a dict that can be used to construct this encoder.", "response": "def getDict(self, encoderName, flattenedChosenValues):\n \"\"\" Return a dict that can be used to construct this encoder. This dict\n can be passed directly to the addMultipleEncoders() method of the\n multi encoder.\n\n Parameters:\n ----------------------------------------------------------------------\n encoderName: name of the encoder\n flattenedChosenValues: dict of the flattened permutation variables. Any\n variables within this dict whose key starts\n with encoderName will be substituted for\n encoder constructor args which are being\n permuted over.\n \"\"\"\n encoder = dict(fieldname=self.fieldName,\n name=self.name)\n\n # Get the position of each encoder argument\n for encoderArg, value in self.kwArgs.iteritems():\n # If a permuted variable, get its chosen value.\n if isinstance(value, PermuteVariable):\n value = flattenedChosenValues[\"%s:%s\" % (encoderName, encoderArg)]\n\n encoder[encoderArg] = value\n\n # Special treatment for DateEncoder timeOfDay and dayOfWeek stuff. In the\n # permutations file, the class can be one of:\n # DateEncoder.timeOfDay\n # DateEncoder.dayOfWeek\n # DateEncoder.season\n # If one of these, we need to intelligently set the constructor args.\n if '.' in self.encoderClass:\n (encoder['type'], argName) = self.encoderClass.split('.')\n argValue = (encoder['w'], encoder['radius'])\n encoder[argName] = argValue\n encoder.pop('w')\n encoder.pop('radius')\n else:\n encoder['type'] = self.encoderClass\n\n return encoder"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _translateMetricsToJSON(self, metrics, label):\n\n # Transcode the MetricValueElement values into JSON-compatible\n # structure\n metricsDict = metrics\n\n # Convert the structure to a display-friendly JSON string\n def _mapNumpyValues(obj):\n \"\"\"\n \"\"\"\n import numpy\n\n if isinstance(obj, numpy.float32):\n return float(obj)\n\n elif isinstance(obj, numpy.bool_):\n return bool(obj)\n\n elif isinstance(obj, numpy.ndarray):\n return obj.tolist()\n\n else:\n raise TypeError(\"UNEXPECTED OBJ: %s; class=%s\" % (obj, obj.__class__))\n\n\n jsonString = json.dumps(metricsDict, indent=4, default=_mapNumpyValues)\n\n return jsonString", "response": "Translate the given metrics value to a JSON string."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __openDatafile(self, modelResult):\n\n # Write reset bit\n resetFieldMeta = FieldMetaInfo(\n name=\"reset\",\n type=FieldMetaType.integer,\n special = FieldMetaSpecial.reset)\n\n self.__outputFieldsMeta.append(resetFieldMeta)\n\n\n # -----------------------------------------------------------------------\n # Write each of the raw inputs that go into the encoders\n rawInput = modelResult.rawInput\n rawFields = rawInput.keys()\n rawFields.sort()\n for field in rawFields:\n if field.startswith('_') or field == 'reset':\n continue\n value = rawInput[field]\n meta = FieldMetaInfo(name=field, type=FieldMetaType.string,\n special=FieldMetaSpecial.none)\n self.__outputFieldsMeta.append(meta)\n self._rawInputNames.append(field)\n\n\n # -----------------------------------------------------------------------\n # Handle each of the inference elements\n for inferenceElement, value in modelResult.inferences.iteritems():\n inferenceLabel = InferenceElement.getLabel(inferenceElement)\n\n # TODO: Right now we assume list inferences are associated with\n # The input field metadata\n if type(value) in (list, tuple):\n # Append input and prediction field meta-info\n self.__outputFieldsMeta.extend(self.__getListMetaInfo(inferenceElement))\n\n elif isinstance(value, dict):\n self.__outputFieldsMeta.extend(self.__getDictMetaInfo(inferenceElement,\n value))\n else:\n\n if InferenceElement.getInputElement(inferenceElement):\n self.__outputFieldsMeta.append(FieldMetaInfo(name=inferenceLabel+\".actual\",\n type=FieldMetaType.string, special = ''))\n self.__outputFieldsMeta.append(FieldMetaInfo(name=inferenceLabel,\n type=FieldMetaType.string, special = ''))\n\n if self.__metricNames:\n for metricName in self.__metricNames:\n metricField = FieldMetaInfo(\n name = metricName,\n type = FieldMetaType.float,\n special = FieldMetaSpecial.none)\n\n self.__outputFieldsMeta.append(metricField)\n\n # Create the inference directory for our experiment\n inferenceDir = _FileUtils.createExperimentInferenceDir(self.__experimentDir)\n\n # Consctruct the prediction dataset file path\n filename = (self.__label + \".\" +\n opf_utils.InferenceType.getLabel(self.__inferenceType) +\n \".predictionLog.csv\")\n self.__datasetPath = os.path.join(inferenceDir, filename)\n\n # Create the output dataset\n print \"OPENING OUTPUT FOR PREDICTION WRITER AT: %r\" % self.__datasetPath\n print \"Prediction field-meta: %r\" % ([tuple(i) for i in self.__outputFieldsMeta],)\n self.__dataset = FileRecordStream(streamID=self.__datasetPath, write=True,\n fields=self.__outputFieldsMeta)\n\n # Copy data from checkpoint cache\n if self.__checkpointCache is not None:\n self.__checkpointCache.seek(0)\n\n reader = csv.reader(self.__checkpointCache, dialect='excel')\n\n # Skip header row\n try:\n header = reader.next()\n except StopIteration:\n print \"Empty record checkpoint initializer for %r\" % (self.__datasetPath,)\n else:\n assert tuple(self.__dataset.getFieldNames()) == tuple(header), \\\n \"dataset.getFieldNames(): %r; predictionCheckpointFieldNames: %r\" % (\n tuple(self.__dataset.getFieldNames()), tuple(header))\n\n # Copy the rows from checkpoint\n numRowsCopied = 0\n while True:\n try:\n row = reader.next()\n except StopIteration:\n break\n\n #print \"DEBUG: restoring row from checkpoint: %r\" % (row,)\n\n self.__dataset.appendRecord(row)\n numRowsCopied += 1\n\n self.__dataset.flush()\n\n print \"Restored %d rows from checkpoint for %r\" % (\n numRowsCopied, self.__datasetPath)\n\n # Dispose of our checkpoint cache\n self.__checkpointCache.close()\n self.__checkpointCache = None\n\n return", "response": "Open the data file and write the header row"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef setLoggedMetrics(self, metricNames):\n if metricNames is None:\n self.__metricNames = set([])\n else:\n self.__metricNames = set(metricNames)", "response": "Tells the writer which metrics should be written"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget the field meta information for the list inferences that are of list type", "response": "def __getListMetaInfo(self, inferenceElement):\n \"\"\" Get field metadata information for inferences that are of list type\n TODO: Right now we assume list inferences are associated with the input field\n metadata\n \"\"\"\n fieldMetaInfo = []\n inferenceLabel = InferenceElement.getLabel(inferenceElement)\n\n for inputFieldMeta in self.__inputFieldsMeta:\n if InferenceElement.getInputElement(inferenceElement):\n outputFieldMeta = FieldMetaInfo(\n name=inputFieldMeta.name + \".actual\",\n type=inputFieldMeta.type,\n special=inputFieldMeta.special\n )\n\n predictionField = FieldMetaInfo(\n name=inputFieldMeta.name + \".\" + inferenceLabel,\n type=inputFieldMeta.type,\n special=inputFieldMeta.special\n )\n\n fieldMetaInfo.append(outputFieldMeta)\n fieldMetaInfo.append(predictionField)\n\n return fieldMetaInfo"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __getDictMetaInfo(self, inferenceElement, inferenceDict):\n fieldMetaInfo = []\n inferenceLabel = InferenceElement.getLabel(inferenceElement)\n\n if InferenceElement.getInputElement(inferenceElement):\n fieldMetaInfo.append(FieldMetaInfo(name=inferenceLabel+\".actual\",\n type=FieldMetaType.string,\n special = ''))\n\n keys = sorted(inferenceDict.keys())\n for key in keys:\n fieldMetaInfo.append(FieldMetaInfo(name=inferenceLabel+\".\"+str(key),\n type=FieldMetaType.string,\n special=''))\n\n\n return fieldMetaInfo", "response": "Get field metadate information for the given inference element and dict"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef append(self, modelResult):\n\n #print \"DEBUG: _BasicPredictionWriter: writing modelResult: %r\" % (modelResult,)\n\n # If there are no inferences, don't write anything\n inferences = modelResult.inferences\n hasInferences = False\n if inferences is not None:\n for value in inferences.itervalues():\n hasInferences = hasInferences or (value is not None)\n\n if not hasInferences:\n return\n\n if self.__dataset is None:\n self.__openDatafile(modelResult)\n\n inputData = modelResult.sensorInput\n\n sequenceReset = int(bool(inputData.sequenceReset))\n outputRow = [sequenceReset]\n\n\n # -----------------------------------------------------------------------\n # Write out the raw inputs\n rawInput = modelResult.rawInput\n for field in self._rawInputNames:\n outputRow.append(str(rawInput[field]))\n\n # -----------------------------------------------------------------------\n # Write out the inference element info\n for inferenceElement, outputVal in inferences.iteritems():\n inputElement = InferenceElement.getInputElement(inferenceElement)\n if inputElement:\n inputVal = getattr(inputData, inputElement)\n else:\n inputVal = None\n\n if type(outputVal) in (list, tuple):\n assert type(inputVal) in (list, tuple, None)\n\n for iv, ov in zip(inputVal, outputVal):\n # Write actual\n outputRow.append(str(iv))\n\n # Write inferred\n outputRow.append(str(ov))\n elif isinstance(outputVal, dict):\n if inputVal is not None:\n # If we have a predicted field, include only that in the actuals\n if modelResult.predictedFieldName is not None:\n outputRow.append(str(inputVal[modelResult.predictedFieldName]))\n else:\n outputRow.append(str(inputVal))\n for key in sorted(outputVal.keys()):\n outputRow.append(str(outputVal[key]))\n else:\n if inputVal is not None:\n outputRow.append(str(inputVal))\n outputRow.append(str(outputVal))\n\n metrics = modelResult.metrics\n for metricName in self.__metricNames:\n outputRow.append(metrics.get(metricName, 0.0))\n\n #print \"DEBUG: _BasicPredictionWriter: writing outputRow: %r\" % (outputRow,)\n\n self.__dataset.appendRecord(outputRow)\n\n self.__dataset.flush()\n\n return", "response": "Emits a single prediction as input versus\n predicted."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsave a checkpoint of the prediction output.", "response": "def checkpoint(self, checkpointSink, maxRows):\n \"\"\" [virtual method override] Save a checkpoint of the prediction output\n stream. The checkpoint comprises up to maxRows of the most recent inference\n records.\n\n Parameters:\n ----------------------------------------------------------------------\n checkpointSink: A File-like object where predictions checkpoint data, if\n any, will be stored.\n maxRows: Maximum number of most recent inference rows\n to checkpoint.\n \"\"\"\n\n checkpointSink.truncate()\n\n if self.__dataset is None:\n if self.__checkpointCache is not None:\n self.__checkpointCache.seek(0)\n shutil.copyfileobj(self.__checkpointCache, checkpointSink)\n checkpointSink.flush()\n return\n else:\n # Nothing to checkpoint\n return\n\n self.__dataset.flush()\n totalDataRows = self.__dataset.getDataRowCount()\n\n if totalDataRows == 0:\n # Nothing to checkpoint\n return\n\n # Open reader of prediction file (suppress missingValues conversion)\n reader = FileRecordStream(self.__datasetPath, missingValues=[])\n\n # Create CSV writer for writing checkpoint rows\n writer = csv.writer(checkpointSink)\n\n # Write the header row to checkpoint sink -- just field names\n writer.writerow(reader.getFieldNames())\n\n # Determine number of rows to checkpoint\n numToWrite = min(maxRows, totalDataRows)\n\n # Skip initial rows to get to the rows that we actually need to checkpoint\n numRowsToSkip = totalDataRows - numToWrite\n for i in xrange(numRowsToSkip):\n reader.next()\n\n # Write the data rows to checkpoint sink\n numWritten = 0\n while True:\n row = reader.getNextRecord()\n if row is None:\n break;\n\n row = [str(element) for element in row]\n\n #print \"DEBUG: _BasicPredictionWriter: checkpointing row: %r\" % (row,)\n\n writer.writerow(row)\n\n numWritten +=1\n\n assert numWritten == numToWrite, \\\n \"numWritten (%s) != numToWrite (%s)\" % (numWritten, numToWrite)\n\n\n checkpointSink.flush()\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating the inference output directory for the given experiment", "response": "def createExperimentInferenceDir(cls, experimentDir):\n \"\"\" Creates the inference output directory for the given experiment\n\n experimentDir: experiment directory path that contains description.py\n\n Returns: path of the inference output directory\n \"\"\"\n path = cls.getExperimentInferenceDirPath(experimentDir)\n\n cls.makeDirectory(path)\n\n return path"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates the initial first order transition sequence and second order transition sequence for model0.", "response": "def _generateModel0(numCategories):\n \"\"\" Generate the initial, first order, and second order transition\n probabilities for 'model0'. For this model, we generate the following\n set of sequences:\n \n 1-2-3 (4X)\n 1-2-4 (1X)\n 5-2-3 (1X)\n 5-2-4 (4X)\n \n \n Parameters:\n ----------------------------------------------------------------------\n numCategories: Number of categories\n retval: (initProb, firstOrder, secondOrder, seqLen)\n initProb: Initial probability for each category. This is a vector\n of length len(categoryList).\n firstOrder: A dictionary of the 1st order probabilities. The key\n is the 1st element of the sequence, the value is\n the probability of each 2nd element given the first. \n secondOrder: A dictionary of the 2nd order probabilities. The key\n is the first 2 elements of the sequence, the value is\n the probability of each possible 3rd element given the \n first two. \n seqLen: Desired length of each sequence. The 1st element will\n be generated using the initProb, the 2nd element by the\n firstOrder table, and the 3rd and all successive \n elements by the secondOrder table. \n\n\n Here is an example of some return values:\n initProb: [0.7, 0.2, 0.1]\n \n firstOrder: {'[0]': [0.3, 0.3, 0.4],\n '[1]': [0.3, 0.3, 0.4],\n '[2]': [0.3, 0.3, 0.4]}\n \n secondOrder: {'[0,0]': [0.3, 0.3, 0.4],\n '[0,1]': [0.3, 0.3, 0.4],\n '[0,2]': [0.3, 0.3, 0.4],\n '[1,0]': [0.3, 0.3, 0.4],\n '[1,1]': [0.3, 0.3, 0.4],\n '[1,2]': [0.3, 0.3, 0.4],\n '[2,0]': [0.3, 0.3, 0.4],\n '[2,1]': [0.3, 0.3, 0.4],\n '[2,2]': [0.3, 0.3, 0.4]}\n \"\"\"\n\n # ===============================================================\n # Let's model the following:\n # a-b-c (4X)\n # a-b-d (1X)\n # e-b-c (1X)\n # e-b-d (4X)\n\n\n # --------------------------------------------------------------------\n # Initial probabilities, 'a' and 'e' equally likely\n initProb = numpy.zeros(numCategories)\n initProb[0] = 0.5\n initProb[4] = 0.5\n \n\n # --------------------------------------------------------------------\n # 1st order transitions\n # both 'a' and 'e' should lead to 'b'\n firstOrder = dict()\n for catIdx in range(numCategories):\n key = str([catIdx])\n probs = numpy.ones(numCategories) / numCategories\n if catIdx == 0 or catIdx == 4:\n probs.fill(0)\n probs[1] = 1.0 # lead only to b\n firstOrder[key] = probs\n \n # --------------------------------------------------------------------\n # 2nd order transitions\n # a-b should lead to c 80% and d 20%\n # e-b should lead to c 20% and d 80%\n secondOrder = dict()\n for firstIdx in range(numCategories):\n for secondIdx in range(numCategories):\n key = str([firstIdx, secondIdx])\n probs = numpy.ones(numCategories) / numCategories\n if key == str([0,1]):\n probs.fill(0)\n probs[2] = 0.80 # 'ab' leads to 'c' 80% of the time\n probs[3] = 0.20 # 'ab' leads to 'd' 20% of the time\n elif key == str([4,1]):\n probs.fill(0)\n probs[2] = 0.20 # 'eb' leads to 'c' 20% of the time\n probs[3] = 0.80 # 'eb' leads to 'd' 80% of the time\n \n secondOrder[key] = probs\n \n return (initProb, firstOrder, secondOrder, 3)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _generateModel1(numCategories):\n\n\n # --------------------------------------------------------------------\n # Initial probabilities, 0 and 1 equally likely\n initProb = numpy.zeros(numCategories)\n initProb[0] = 0.5\n initProb[1] = 0.5\n \n\n # --------------------------------------------------------------------\n # 1st order transitions\n # both 0 and 1 should lead to 10,11,12,13,14 with equal probability\n firstOrder = dict()\n for catIdx in range(numCategories):\n key = str([catIdx])\n probs = numpy.ones(numCategories) / numCategories\n if catIdx == 0 or catIdx == 1:\n indices = numpy.array([10,11,12,13,14])\n probs.fill(0)\n probs[indices] = 1.0 # lead only to b\n probs /= probs.sum()\n firstOrder[key] = probs\n \n # --------------------------------------------------------------------\n # 2nd order transitions\n # 0-10 should lead to 15\n # 0-11 to 16\n # ...\n # 1-10 should lead to 20\n # 1-11 shold lean to 21\n # ...\n secondOrder = dict()\n for firstIdx in range(numCategories):\n for secondIdx in range(numCategories):\n key = str([firstIdx, secondIdx])\n probs = numpy.ones(numCategories) / numCategories\n if key == str([0,10]):\n probs.fill(0)\n probs[15] = 1\n elif key == str([0,11]):\n probs.fill(0)\n probs[16] = 1\n elif key == str([0,12]):\n probs.fill(0)\n probs[17] = 1\n elif key == str([0,13]):\n probs.fill(0)\n probs[18] = 1\n elif key == str([0,14]):\n probs.fill(0)\n probs[19] = 1\n \n elif key == str([1,10]):\n probs.fill(0)\n probs[20] = 1\n elif key == str([1,11]):\n probs.fill(0)\n probs[21] = 1\n elif key == str([1,12]):\n probs.fill(0)\n probs[22] = 1\n elif key == str([1,13]):\n probs.fill(0)\n probs[23] = 1\n elif key == str([1,14]):\n probs.fill(0)\n probs[24] = 1\n \n secondOrder[key] = probs\n \n return (initProb, firstOrder, secondOrder, 3)", "response": "Generates the initial first order transition sequence and second order transition sequence for model1."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _generateModel2(numCategories, alpha=0.25):\n\n\n # --------------------------------------------------------------------\n # All initial probabilities, are equally likely\n initProb = numpy.ones(numCategories)/numCategories\n\n def generatePeakedProbabilities(lastIdx,\n\t\t\t\t numCategories=numCategories, \n\t\t\t\t alpha=alpha):\n probs = numpy.random.dirichlet(alpha=[alpha]*numCategories)\n probs[lastIdx] = 0.0\n probs /= probs.sum()\n return probs \n\n # --------------------------------------------------------------------\n # 1st order transitions\n firstOrder = dict()\n for catIdx in range(numCategories):\n key = str([catIdx])\n probs = generatePeakedProbabilities(catIdx) \n firstOrder[key] = probs\n\n # --------------------------------------------------------------------\n # 2nd order transitions\n secondOrder = dict()\n for firstIdx in range(numCategories):\n for secondIdx in range(numCategories):\n key = str([firstIdx, secondIdx])\n probs = generatePeakedProbabilities(secondIdx) \n secondOrder[key] = probs\n\n return (initProb, firstOrder, secondOrder, None)", "response": "Generates the initial first order second order transition probabilities for the model2."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates a set of records reflecting a set of probabilities.", "response": "def _generateFile(filename, numRecords, categoryList, initProb, \n firstOrderProb, secondOrderProb, seqLen, numNoise=0, resetsEvery=None):\n \"\"\" Generate a set of records reflecting a set of probabilities.\n \n Parameters:\n ----------------------------------------------------------------\n filename: name of .csv file to generate\n numRecords: number of records to generate\n categoryList: list of category names\n initProb: Initial probability for each category. This is a vector\n of length len(categoryList).\n firstOrderProb: A dictionary of the 1st order probabilities. The key\n is the 1st element of the sequence, the value is\n the probability of each 2nd element given the first. \n secondOrderProb: A dictionary of the 2nd order probabilities. The key\n is the first 2 elements of the sequence, the value is\n the probability of each possible 3rd element given the \n first two. \n seqLen: Desired length of each sequence. The 1st element will\n be generated using the initProb, the 2nd element by the\n firstOrder table, and the 3rd and all successive \n elements by the secondOrder table. None means infinite\n length. \n numNoise: Number of noise elements to place between each \n sequence. The noise elements are evenly distributed from \n all categories. \n resetsEvery: If not None, generate a reset every N records\n \n \n Here is an example of some parameters:\n \n categoryList: ['cat1', 'cat2', 'cat3']\n \n initProb: [0.7, 0.2, 0.1]\n \n firstOrderProb: {'[0]': [0.3, 0.3, 0.4],\n '[1]': [0.3, 0.3, 0.4],\n '[2]': [0.3, 0.3, 0.4]}\n \n secondOrderProb: {'[0,0]': [0.3, 0.3, 0.4],\n '[0,1]': [0.3, 0.3, 0.4],\n '[0,2]': [0.3, 0.3, 0.4],\n '[1,0]': [0.3, 0.3, 0.4],\n '[1,1]': [0.3, 0.3, 0.4],\n '[1,2]': [0.3, 0.3, 0.4],\n '[2,0]': [0.3, 0.3, 0.4],\n '[2,1]': [0.3, 0.3, 0.4],\n '[2,2]': [0.3, 0.3, 0.4]}\n \n \"\"\"\n \n # Create the file\n print \"Creating %s...\" % (filename)\n fields = [('reset', 'int', 'R'), ('name', 'string', '')]\n outFile = FileRecordStream(filename, write=True, fields=fields)\n \n # --------------------------------------------------------------------\n # Convert the probabilitie tables into cumulative probabilities\n initCumProb = initProb.cumsum()\n \n firstOrderCumProb = dict()\n for (key,value) in firstOrderProb.iteritems():\n firstOrderCumProb[key] = value.cumsum()\n \n secondOrderCumProb = dict()\n for (key,value) in secondOrderProb.iteritems():\n secondOrderCumProb[key] = value.cumsum()\n \n\n # --------------------------------------------------------------------\n # Write out the sequences\n elementsInSeq = []\n numElementsSinceReset = 0\n maxCatIdx = len(categoryList) - 1\n for i in xrange(numRecords):\n\n # Generate a reset?\n if numElementsSinceReset == 0:\n reset = 1\n else:\n reset = 0\n \n # Pick the next element, based on how are we are into the 2nd order\n # sequence. \n rand = numpy.random.rand()\n if len(elementsInSeq) == 0:\n catIdx = numpy.searchsorted(initCumProb, rand)\n elif len(elementsInSeq) == 1:\n catIdx = numpy.searchsorted(firstOrderCumProb[str(elementsInSeq)], rand)\n elif (len(elementsInSeq) >=2) and \\\n (seqLen is None or len(elementsInSeq) < seqLen-numNoise):\n catIdx = numpy.searchsorted(secondOrderCumProb[str(elementsInSeq[-2:])], rand)\n else: # random \"noise\"\n catIdx = numpy.random.randint(len(categoryList))\n \n # Write out the record\n catIdx = min(maxCatIdx, catIdx)\n outFile.appendRecord([reset,categoryList[catIdx]]) \n #print categoryList[catIdx]\n \n # ------------------------------------------------------------\n # Increment counters\n elementsInSeq.append(catIdx)\n numElementsSinceReset += 1\n \n # Generate another reset?\n if resetsEvery is not None and numElementsSinceReset == resetsEvery:\n numElementsSinceReset = 0\n elementsInSeq = []\n \n # Start another 2nd order sequence?\n if seqLen is not None and (len(elementsInSeq) == seqLen+numNoise):\n elementsInSeq = []\n \n \n outFile.close()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _allow_new_attributes(f):\n def decorated(self, *args, **kw):\n \"\"\"The decorated function that replaces __init__() or __setstate__()\n\n \"\"\"\n # Run the original function\n if not hasattr(self, '_canAddAttributes'):\n self.__dict__['_canAddAttributes'] = 1\n else:\n self._canAddAttributes += 1\n assert self._canAddAttributes >= 1\n\n # Save add attribute counter\n count = self._canAddAttributes\n f(self, *args, **kw)\n\n # Restore _CanAddAttributes if deleted from dict (can happen in __setstte__)\n if hasattr(self, '_canAddAttributes'):\n self._canAddAttributes -= 1\n else:\n self._canAddAttributes = count - 1\n\n assert self._canAddAttributes >= 0\n if self._canAddAttributes == 0:\n del self._canAddAttributes\n\n decorated.__doc__ = f.__doc__\n decorated.__name__ = f.__name__\n return decorated", "response": "A function that allows setting new attributes on an object."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _simple_init(self, *args, **kw):\n type(self).__base__.__init__(self, *args, **kw)", "response": "simple init method that just calls base class s __init__ method"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating a table where each cell represents a frequency of pairs .", "response": "def generatePlot(outputs, origData):\n \"\"\" Generates a table where each cell represent a frequency of pairs\n as described below.\n x coordinate is the % difference between input records (origData list),\n y coordinate is the % difference between corresponding output records.\n \"\"\"\n\n PLOT_PRECISION = 100\n\n distribMatrix = np.zeros((PLOT_PRECISION+1,PLOT_PRECISION+1))\n\n outputSize = len(outputs)\n\n for i in range(0,outputSize):\n for j in range(i+1,outputSize):\n\n in1 = outputs[i]\n in2 = outputs[j]\n dist = (abs(in1-in2) > 0.1)\n intDist = int(dist.sum()/2+0.1)\n\n orig1 = origData[i]\n orig2 = origData[j]\n origDist = (abs(orig1-orig2) > 0.1)\n intOrigDist = int(origDist.sum()/2+0.1)\n\n if intDist < 2 and intOrigDist > 10:\n print 'Elements %d,%d has very small SP distance: %d' % (i, j, intDist)\n print 'Input elements distance is %d' % intOrigDist\n\n x = int(PLOT_PRECISION*intDist/40.0)\n y = int(PLOT_PRECISION*intOrigDist/42.0)\n if distribMatrix[x, y] < 0.1:\n distribMatrix[x, y] = 3\n else:\n if distribMatrix[x, y] < 10:\n distribMatrix[x, y] += 1\n\n # Add some elements for the scale drawing\n distribMatrix[4, 50] = 3\n distribMatrix[4, 52] = 4\n distribMatrix[4, 54] = 5\n distribMatrix[4, 56] = 6\n distribMatrix[4, 58] = 7\n distribMatrix[4, 60] = 8\n distribMatrix[4, 62] = 9\n distribMatrix[4, 64] = 10\n\n return distribMatrix"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates a set of random input for the specified number of records.", "response": "def generateRandomInput(numRecords, elemSize = 400, numSet = 42):\n \"\"\" Generates a set of input record\n\n Params:\n numRecords - how many records to generate\n elemSize - the size of each record (num 0s or 1s)\n numSet - how many 1s in each record\n\n Returns: a list of inputs\n \"\"\"\n\n inputs = []\n\n for _ in xrange(numRecords):\n\n input = np.zeros(elemSize, dtype=realDType)\n for _ in range(0,numSet):\n ind = np.random.random_integers(0, elemSize-1, 1)[0]\n input[ind] = 1\n while abs(input.sum() - numSet) > 0.1:\n ind = np.random.random_integers(0, elemSize-1, 1)[0]\n input[ind] = 1\n\n inputs.append(input)\n\n return inputs"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate an one - off record for each record in the inputs. Appends a new one - off record for each record in the inputs.", "response": "def appendInputWithSimilarValues(inputs):\n \"\"\" Creates an 'one-off' record for each record in the inputs. Appends new\n records to the same inputs list.\n \"\"\"\n numInputs = len(inputs)\n for i in xrange(numInputs):\n input = inputs[i]\n for j in xrange(len(input)-1):\n if input[j] == 1 and input[j+1] == 0:\n newInput = copy.deepcopy(input)\n newInput[j] = 0\n newInput[j+1] = 1\n inputs.append(newInput)\n break"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a neighboring record for each record in the inputs and adds a new record for each input that has the same number of values.", "response": "def appendInputWithNSimilarValues(inputs, numNear = 10):\n \"\"\" Creates a neighboring record for each record in the inputs and adds\n new records at the end of the inputs list\n \"\"\"\n numInputs = len(inputs)\n skipOne = False\n for i in xrange(numInputs):\n input = inputs[i]\n numChanged = 0\n newInput = copy.deepcopy(input)\n for j in xrange(len(input)-1):\n if skipOne:\n skipOne = False\n continue\n if input[j] == 1 and input[j+1] == 0:\n newInput[j] = 0\n newInput[j+1] = 1\n inputs.append(newInput)\n newInput = copy.deepcopy(newInput)\n #print input\n #print newInput\n numChanged += 1\n skipOne = True\n if numChanged == numNear:\n break"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmodify the inputVal by maxChanges number of bits in the alphabetical order.", "response": "def modifyBits(inputVal, maxChanges):\n \"\"\" Modifies up to maxChanges number of bits in the inputVal\n \"\"\"\n changes = np.random.random_integers(0, maxChanges, 1)[0]\n\n if changes == 0:\n return inputVal\n\n inputWidth = len(inputVal)\n\n whatToChange = np.random.random_integers(0, 41, changes)\n\n runningIndex = -1\n numModsDone = 0\n for i in xrange(inputWidth):\n if numModsDone >= changes:\n break\n if inputVal[i] == 1:\n runningIndex += 1\n if runningIndex in whatToChange:\n if i != 0 and inputVal[i-1] == 0:\n inputVal[i-1] = 1\n inputVal[i] = 0\n\n return inputVal"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a random selection from the inputSpace with randomly modified ultimately maxChanges number of bits.", "response": "def getRandomWithMods(inputSpace, maxChanges):\n \"\"\" Returns a random selection from the inputSpace with randomly modified\n up to maxChanges number of bits.\n \"\"\"\n size = len(inputSpace)\n ind = np.random.random_integers(0, size-1, 1)[0]\n\n value = copy.deepcopy(inputSpace[ind])\n\n if maxChanges == 0:\n return value\n\n return modifyBits(value, maxChanges)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate and returns a new encoder for the specified type of file - name.", "response": "def createEncoder():\n \"\"\"\n Creates and returns a #MultiEncoder including a ScalarEncoder for\n energy consumption and a DateEncoder for the time of the day.\n\n @see nupic/encoders/__init__.py for type to file-name mapping\n @see nupic/encoders for encoder source files\n \"\"\"\n encoder = MultiEncoder()\n encoder.addMultipleEncoders({\n \"consumption\": {\"fieldname\": u\"consumption\",\n \"type\": \"ScalarEncoder\",\n \"name\": u\"consumption\",\n \"minval\": 0.0,\n \"maxval\": 100.0,\n \"clipInput\": True,\n \"w\": 21,\n \"n\": 500},\n \"timestamp_timeOfDay\": {\"fieldname\": u\"timestamp\",\n \"type\": \"DateEncoder\",\n \"name\": u\"timestamp_timeOfDay\",\n \"timeOfDay\": (21, 9.5)}\n })\n return encoder"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a RecordSensor region that allows us to specify a file record stream as the input source.", "response": "def createRecordSensor(network, name, dataSource):\n \"\"\"\n Creates a RecordSensor region that allows us to specify a file record\n stream as the input source.\n \"\"\"\n\n # Specific type of region. Possible options can be found in /nupic/regions/\n regionType = \"py.RecordSensor\"\n\n # Creates a json from specified dictionary.\n regionParams = json.dumps({\"verbosity\": _VERBOSITY})\n network.addRegion(name, regionType, regionParams)\n\n # getSelf returns the actual region, instead of a region wrapper\n sensorRegion = network.regions[name].getSelf()\n\n # Specify how RecordSensor encodes input values\n sensorRegion.encoder = createEncoder()\n\n # Specify which sub-encoder should be used for \"actValueOut\"\n network.regions[name].setParameter(\"predictedField\", \"consumption\")\n\n # Specify the dataSource as a file record stream instance\n sensorRegion.dataSource = dataSource\n return sensorRegion"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef createNetwork(dataSource):\n network = Network()\n\n # Create and add a record sensor and a SP region\n sensor = createRecordSensor(network, name=_RECORD_SENSOR,\n dataSource=dataSource)\n createSpatialPooler(network, name=_L1_SPATIAL_POOLER,\n inputWidth=sensor.encoder.getWidth())\n\n # Link the SP region to the sensor input\n linkType = \"UniformLink\"\n linkParams = \"\"\n network.link(_RECORD_SENSOR, _L1_SPATIAL_POOLER, linkType, linkParams)\n\n # Create and add a TM region\n l1temporalMemory = createTemporalMemory(network, _L1_TEMPORAL_MEMORY)\n\n # Link SP region to TM region in the feedforward direction\n network.link(_L1_SPATIAL_POOLER, _L1_TEMPORAL_MEMORY, linkType, linkParams)\n\n # Add a classifier\n classifierParams = { # Learning rate. Higher values make it adapt faster.\n 'alpha': 0.005,\n\n # A comma separated list of the number of steps the\n # classifier predicts in the future. The classifier will\n # learn predictions of each order specified.\n 'steps': '1',\n\n # The specific implementation of the classifier to use\n # See SDRClassifierFactory#create for options\n 'implementation': 'py',\n\n # Diagnostic output verbosity control;\n # 0: silent; [1..6]: increasing levels of verbosity\n 'verbosity': 0}\n\n l1Classifier = network.addRegion(_L1_CLASSIFIER, \"py.SDRClassifierRegion\",\n json.dumps(classifierParams))\n l1Classifier.setParameter('inferenceMode', True)\n l1Classifier.setParameter('learningMode', True)\n network.link(_L1_TEMPORAL_MEMORY, _L1_CLASSIFIER, linkType, linkParams,\n srcOutput=\"bottomUpOut\", destInput=\"bottomUpIn\")\n network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,\n srcOutput=\"categoryOut\", destInput=\"categoryIn\")\n network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,\n srcOutput=\"bucketIdxOut\", destInput=\"bucketIdxIn\")\n network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,\n srcOutput=\"actValueOut\", destInput=\"actValueIn\")\n\n # Second Level\n l2inputWidth = l1temporalMemory.getSelf().getOutputElementCount(\"bottomUpOut\")\n createSpatialPooler(network, name=_L2_SPATIAL_POOLER, inputWidth=l2inputWidth)\n network.link(_L1_TEMPORAL_MEMORY, _L2_SPATIAL_POOLER, linkType, linkParams)\n\n createTemporalMemory(network, _L2_TEMPORAL_MEMORY)\n network.link(_L2_SPATIAL_POOLER, _L2_TEMPORAL_MEMORY, linkType, linkParams)\n\n l2Classifier = network.addRegion(_L2_CLASSIFIER, \"py.SDRClassifierRegion\",\n json.dumps(classifierParams))\n l2Classifier.setParameter('inferenceMode', True)\n l2Classifier.setParameter('learningMode', True)\n network.link(_L2_TEMPORAL_MEMORY, _L2_CLASSIFIER, linkType, linkParams,\n srcOutput=\"bottomUpOut\", destInput=\"bottomUpIn\")\n network.link(_RECORD_SENSOR, _L2_CLASSIFIER, linkType, linkParams,\n srcOutput=\"categoryOut\", destInput=\"categoryIn\")\n network.link(_RECORD_SENSOR, _L2_CLASSIFIER, linkType, linkParams,\n srcOutput=\"bucketIdxOut\", destInput=\"bucketIdxIn\")\n network.link(_RECORD_SENSOR, _L2_CLASSIFIER, linkType, linkParams,\n srcOutput=\"actValueOut\", destInput=\"actValueIn\")\n return network", "response": "Creates and returns a new Network with a sensor region reading data from\n ataSource."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrun specified Network writing the ensuing anomaly scores to writer.", "response": "def runNetwork(network, numRecords, writer):\n \"\"\"\n Runs specified Network writing the ensuing anomaly\n scores to writer.\n\n @param network: The Network instance to be run\n @param writer: A csv.writer used to write to output file.\n \"\"\"\n sensorRegion = network.regions[_RECORD_SENSOR]\n l1SpRegion = network.regions[_L1_SPATIAL_POOLER]\n l1TpRegion = network.regions[_L1_TEMPORAL_MEMORY]\n l1Classifier = network.regions[_L1_CLASSIFIER]\n\n l2SpRegion = network.regions[_L2_SPATIAL_POOLER]\n l2TpRegion = network.regions[_L2_TEMPORAL_MEMORY]\n l2Classifier = network.regions[_L2_CLASSIFIER]\n\n l1PreviousPredictedColumns = []\n l2PreviousPredictedColumns = []\n\n l1PreviousPrediction = None\n l2PreviousPrediction = None\n l1ErrorSum = 0.0\n l2ErrorSum = 0.0\n for record in xrange(numRecords):\n # Run the network for a single iteration\n network.run(1)\n\n actual = float(sensorRegion.getOutputData(\"actValueOut\")[0])\n\n l1Predictions = l1Classifier.getOutputData(\"actualValues\")\n l1Probabilities = l1Classifier.getOutputData(\"probabilities\")\n l1Prediction = l1Predictions[l1Probabilities.argmax()]\n if l1PreviousPrediction is not None:\n l1ErrorSum += math.fabs(l1PreviousPrediction - actual)\n l1PreviousPrediction = l1Prediction\n\n l2Predictions = l2Classifier.getOutputData(\"actualValues\")\n l2Probabilities = l2Classifier.getOutputData(\"probabilities\")\n l2Prediction = l2Predictions[l2Probabilities.argmax()]\n if l2PreviousPrediction is not None:\n l2ErrorSum += math.fabs(l2PreviousPrediction - actual)\n l2PreviousPrediction = l2Prediction\n\n l1AnomalyScore = l1TpRegion.getOutputData(\"anomalyScore\")[0]\n l2AnomalyScore = l2TpRegion.getOutputData(\"anomalyScore\")[0]\n\n # Write record number, actualInput, and anomaly scores\n writer.writerow((record, actual, l1PreviousPrediction, l1AnomalyScore, l2PreviousPrediction, l2AnomalyScore))\n\n # Store the predicted columns for the next timestep\n l1PredictedColumns = l1TpRegion.getOutputData(\"topDownOut\").nonzero()[0]\n l1PreviousPredictedColumns = copy.deepcopy(l1PredictedColumns)\n #\n l2PredictedColumns = l2TpRegion.getOutputData(\"topDownOut\").nonzero()[0]\n l2PreviousPredictedColumns = copy.deepcopy(l2PredictedColumns)\n\n # Output absolute average error for each level\n if numRecords > 1:\n print \"L1 ave abs class. error: %f\" % (l1ErrorSum / (numRecords - 1))\n print \"L2 ave abs class. error: %f\" % (l2ErrorSum / (numRecords - 1))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nremove trailing whitespace on each line.", "response": "def clean(s):\n \"\"\"Removes trailing whitespace on each line.\"\"\"\n lines = [l.rstrip() for l in s.split('\\n')]\n return '\\n'.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute the new metrics values given the next inference and ground - truth values.", "response": "def update(self, results):\n \"\"\"\n Compute the new metrics values, given the next inference/ground-truth values\n\n :param results: (:class:`~nupic.frameworks.opf.opf_utils.ModelResult`) \n object that was computed during the last iteration of the model.\n\n :returns: (dict) where each key is the metric-name, and the values are\n it scalar value.\n\n \"\"\"\n\n #print \"\\n\\n---------------------------------------------------------------\"\n #print \"Model results: \\nrawInput:%s \\ninferences:%s\" % \\\n # (pprint.pformat(results.rawInput), pprint.pformat(results.inferences))\n\n self._addResults(results)\n\n if not self.__metricSpecs \\\n or self.__currentInference is None:\n return {}\n\n metricResults = {}\n for metric, spec, label in zip(self.__metrics,\n self.__metricSpecs,\n self.__metricLabels):\n\n inferenceElement = spec.inferenceElement\n field = spec.field\n groundTruth = self._getGroundTruth(inferenceElement)\n inference = self._getInference(inferenceElement)\n rawRecord = self._getRawGroundTruth()\n result = self.__currentResult\n if field:\n if type(inference) in (list, tuple):\n if field in self.__fieldNameIndexMap:\n # NOTE: If the predicted field is not fed in at the bottom, we\n # won't have it in our fieldNameIndexMap\n fieldIndex = self.__fieldNameIndexMap[field]\n inference = inference[fieldIndex]\n else:\n inference = None\n if groundTruth is not None:\n if type(groundTruth) in (list, tuple):\n if field in self.__fieldNameIndexMap:\n # NOTE: If the predicted field is not fed in at the bottom, we\n # won't have it in our fieldNameIndexMap\n fieldIndex = self.__fieldNameIndexMap[field]\n groundTruth = groundTruth[fieldIndex]\n else:\n groundTruth = None\n else:\n # groundTruth could be a dict based off of field names\n groundTruth = groundTruth[field]\n\n metric.addInstance(groundTruth=groundTruth,\n prediction=inference,\n record=rawRecord,\n result=result)\n\n metricResults[label] = metric.getMetric()['value']\n\n return metricResults"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getMetrics(self):\n\n result = {}\n\n for metricObj, label in zip(self.__metrics, self.__metricLabels):\n value = metricObj.getMetric()\n result[label] = value['value']\n\n return result", "response": "Gets the current metric values\n "} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning detailed information about a given metric.", "response": "def getMetricDetails(self, metricLabel):\n \"\"\" \n Gets detailed info about a given metric, in addition to its value. This\n may including any statistics or auxilary data that are computed for a given\n metric.\n\n :param metricLabel: (string) label of the given metric (see \n :class:`~nupic.frameworks.opf.metrics.MetricSpec`)\n\n :returns: (dict) of metric information, as returned by \n :meth:`nupic.frameworks.opf.metrics.MetricsIface.getMetric`.\n \"\"\"\n try:\n metricIndex = self.__metricLabels.index(metricLabel)\n except IndexError:\n return None\n\n return self.__metrics[metricIndex].getMetric()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd the results to the internal store of the current model.", "response": "def _addResults(self, results):\n \"\"\"\n Stores the current model results in the manager's internal store\n\n Parameters:\n -----------------------------------------------------------------------\n results: A ModelResults object that contains the current timestep's\n input/inferences\n \"\"\"\n # -----------------------------------------------------------------------\n # If the model potentially has temporal inferences.\n if self.__isTemporal:\n shiftedInferences = self.__inferenceShifter.shift(results).inferences\n self.__currentResult = copy.deepcopy(results)\n self.__currentResult.inferences = shiftedInferences\n self.__currentInference = shiftedInferences\n\n # -----------------------------------------------------------------------\n # The current model has no temporal inferences.\n else:\n self.__currentResult = copy.deepcopy(results)\n self.__currentInference = copy.deepcopy(results.inferences)\n\n # -----------------------------------------------------------------------\n # Save the current ground-truth results\n self.__currentGroundTruth = copy.deepcopy(results)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _getGroundTruth(self, inferenceElement):\n sensorInputElement = InferenceElement.getInputElement(inferenceElement)\n if sensorInputElement is None:\n return None\n return getattr(self.__currentGroundTruth.sensorInput, sensorInputElement)", "response": "Returns the actual value for this metric"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __constructMetricsModules(self, metricSpecs):\n if not metricSpecs:\n return\n\n self.__metricSpecs = metricSpecs\n for spec in metricSpecs:\n if not InferenceElement.validate(spec.inferenceElement):\n raise ValueError(\"Invalid inference element for metric spec: %r\" %spec)\n\n self.__metrics.append(metrics.getModule(spec))\n self.__metricLabels.append(spec.getLabel())", "response": "Constructs the required metrics modules"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate a simple dataset.", "response": "def _generateSimple(filename=\"simple.csv\", numSequences=1, elementsPerSeq=3, \n numRepeats=10):\n \"\"\" Generate a simple dataset. This contains a bunch of non-overlapping\n sequences. \n \n At the end of the dataset, we introduce missing records so that test\n code can insure that the model didn't get confused by them. \n \n Parameters:\n ----------------------------------------------------\n filename: name of the file to produce, including extension. It will\n be created in a 'datasets' sub-directory within the \n directory containing this script. \n numSequences: how many sequences to generate\n elementsPerSeq: length of each sequence\n numRepeats: how many times to repeat each sequence in the output \n \"\"\"\n \n # Create the output file\n scriptDir = os.path.dirname(__file__)\n pathname = os.path.join(scriptDir, 'datasets', filename)\n print \"Creating %s...\" % (pathname)\n fields = [('timestamp', 'datetime', 'T'), \n ('field1', 'string', ''), \n ('field2', 'float', '')] \n outFile = FileRecordStream(pathname, write=True, fields=fields)\n \n # Create the sequences\n sequences = []\n for i in range(numSequences):\n seq = [x for x in range(i*elementsPerSeq, (i+1)*elementsPerSeq)]\n sequences.append(seq)\n \n \n # Write out the sequences in random order\n seqIdxs = []\n for i in range(numRepeats):\n seqIdxs += range(numSequences)\n random.shuffle(seqIdxs)\n \n # Put 1 hour between each record\n timestamp = datetime.datetime(year=2012, month=1, day=1, hour=0, minute=0,\n second=0)\n timeDelta = datetime.timedelta(hours=1)\n \n # Write out the sequences without missing records\n for seqIdx in seqIdxs:\n seq = sequences[seqIdx]\n for x in seq:\n outFile.appendRecord([timestamp, str(x), x])\n timestamp += timeDelta\n \n # Now, write some out with missing records\n for seqIdx in seqIdxs:\n seq = sequences[seqIdx]\n for i,x in enumerate(seq):\n if i != 1:\n outFile.appendRecord([timestamp, str(x), x])\n timestamp += timeDelta\n for seqIdx in seqIdxs:\n seq = sequences[seqIdx]\n for i,x in enumerate(seq):\n if i != 1:\n outFile.appendRecord([timestamp, str(x), x])\n timestamp += timeDelta\n\n # Write out some more of the sequences *without* missing records\n for seqIdx in seqIdxs:\n seq = sequences[seqIdx]\n for x in seq:\n outFile.appendRecord([timestamp, str(x), x])\n timestamp += timeDelta\n \n \n\n outFile.close()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nshift the model result and return the new instance.", "response": "def shift(self, modelResult):\n \"\"\"Shift the model result and return the new instance.\n\n Queues up the T(i+1) prediction value and emits a T(i)\n input/prediction pair, if possible. E.g., if the previous T(i-1)\n iteration was learn-only, then we would not have a T(i) prediction in our\n FIFO and would not be able to emit a meaningful input/prediction pair.\n\n :param modelResult: A :class:`~.nupic.frameworks.opf.opf_utils.ModelResult`\n instance to shift.\n :return: A :class:`~.nupic.frameworks.opf.opf_utils.ModelResult` instance that\n has been shifted\n \"\"\"\n inferencesToWrite = {}\n\n if self._inferenceBuffer is None:\n maxDelay = InferenceElement.getMaxDelay(modelResult.inferences)\n self._inferenceBuffer = collections.deque(maxlen=maxDelay + 1)\n\n self._inferenceBuffer.appendleft(copy.deepcopy(modelResult.inferences))\n\n for inferenceElement, inference in modelResult.inferences.iteritems():\n if isinstance(inference, dict):\n inferencesToWrite[inferenceElement] = {}\n for key, _ in inference.iteritems():\n delay = InferenceElement.getTemporalDelay(inferenceElement, key)\n if len(self._inferenceBuffer) > delay:\n prevInference = self._inferenceBuffer[delay][inferenceElement][key]\n inferencesToWrite[inferenceElement][key] = prevInference\n else:\n inferencesToWrite[inferenceElement][key] = None\n else:\n delay = InferenceElement.getTemporalDelay(inferenceElement)\n if len(self._inferenceBuffer) > delay:\n inferencesToWrite[inferenceElement] = (\n self._inferenceBuffer[delay][inferenceElement])\n else:\n if type(inference) in (list, tuple):\n inferencesToWrite[inferenceElement] = [None] * len(inference)\n else:\n inferencesToWrite[inferenceElement] = None\n\n shiftedResult = ModelResult(rawInput=modelResult.rawInput,\n sensorInput=modelResult.sensorInput,\n inferences=inferencesToWrite,\n metrics=modelResult.metrics,\n predictedFieldIdx=modelResult.predictedFieldIdx,\n predictedFieldName=modelResult.predictedFieldName)\n return shiftedResult"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngenerate statistics for each of the fields in the user input data file and returns a dictionary of dictionaries. Each dictionary contains the values for each field and the corresponding stats for each individual file.", "response": "def generateStats(filename, maxSamples = None,):\n \"\"\"\n Collect statistics for each of the fields in the user input data file and\n return a stats dict object.\n\n Parameters:\n ------------------------------------------------------------------------------\n filename: The path and name of the data file.\n maxSamples: Upper bound on the number of rows to be processed\n retval: A dictionary of dictionaries. The top level keys are the\n field names and the corresponding values are the statistics\n collected for the individual file.\n Example:\n {\n 'consumption':{'min':0,'max':90,'mean':50,...},\n 'gym':{'numDistinctCategories':10,...},\n ...\n }\n\n\n \"\"\"\n # Mapping from field type to stats collector object\n statsCollectorMapping = {'float': FloatStatsCollector,\n 'int': IntStatsCollector,\n 'string': StringStatsCollector,\n 'datetime': DateTimeStatsCollector,\n 'bool': BoolStatsCollector,\n }\n\n filename = resource_filename(\"nupic.datafiles\", filename)\n print \"*\"*40\n print \"Collecting statistics for file:'%s'\" % (filename,)\n dataFile = FileRecordStream(filename)\n\n # Initialize collector objects\n # statsCollectors list holds statsCollector objects for each field\n statsCollectors = []\n for fieldName, fieldType, fieldSpecial in dataFile.getFields():\n # Find the corresponding stats collector for each field based on field type\n # and intialize an instance\n statsCollector = \\\n statsCollectorMapping[fieldType](fieldName, fieldType, fieldSpecial)\n statsCollectors.append(statsCollector)\n\n # Now collect the stats\n if maxSamples is None:\n maxSamples = 500000\n for i in xrange(maxSamples):\n record = dataFile.getNextRecord()\n if record is None:\n break\n for i, value in enumerate(record):\n statsCollectors[i].addValue(value)\n\n # stats dict holds the statistics for each field\n stats = {}\n for statsCollector in statsCollectors:\n statsCollector.getStats(stats)\n\n # We don't want to include reset field in permutations\n # TODO: handle reset field in a clean way\n if dataFile.getResetFieldIdx() is not None:\n resetFieldName,_,_ = dataFile.getFields()[dataFile.reset]\n stats.pop(resetFieldName)\n\n if VERBOSITY > 0:\n pprint.pprint(stats)\n\n return stats"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getStats(self, stats):\n BaseStatsCollector.getStats(self, stats)\n\n sortedNumberList = sorted(self.valueList)\n listLength = len(sortedNumberList)\n min = sortedNumberList[0]\n max = sortedNumberList[-1]\n mean = numpy.mean(self.valueList)\n median = sortedNumberList[int(0.5*listLength)]\n percentile1st = sortedNumberList[int(0.01*listLength)]\n percentile99th = sortedNumberList[int(0.99*listLength)]\n\n differenceList = \\\n [(cur - prev) for prev, cur in itertools.izip(list(self.valueSet)[:-1],\n list(self.valueSet)[1:])]\n if min > max:\n print self.fieldname, min, max, '-----'\n meanResolution = numpy.mean(differenceList)\n\n\n stats[self.fieldname]['min'] = min\n stats[self.fieldname]['max'] = max\n stats[self.fieldname]['mean'] = mean\n stats[self.fieldname]['median'] = median\n stats[self.fieldname]['percentile1st'] = percentile1st\n stats[self.fieldname]['percentile99th'] = percentile99th\n stats[self.fieldname]['meanResolution'] = meanResolution\n\n # TODO: Right now, always pass the data along.\n # This is used for data-dependent encoders.\n passData = True\n if passData:\n stats[self.fieldname]['data'] = self.valueList\n\n if VERBOSITY > 2:\n print '--'\n print \"Statistics:\"\n print \"min:\", min\n print \"max:\", max\n print \"mean:\", mean\n print \"median:\", median\n print \"1st percentile :\", percentile1st\n print \"99th percentile:\", percentile99th\n\n print '--'\n print \"Resolution:\"\n print \"Mean Resolution:\", meanResolution\n\n if VERBOSITY > 3:\n print '--'\n print \"Histogram:\"\n counts, bins = numpy.histogram(self.valueList, new=True)\n print \"Counts:\", counts.tolist()\n print \"Bins:\", bins.tolist()", "response": "Override of getStats method in BaseStatsCollector to get statistics for the current base class"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef main():\n initLogging(verbose=True)\n\n # Initialize PRNGs\n initExperimentPrng()\n\n # Mock out the creation of the SDRClassifier.\n @staticmethod\n def _mockCreate(*args, **kwargs):\n kwargs.pop('implementation', None)\n return SDRClassifierDiff(*args, **kwargs)\n SDRClassifierFactory.create = _mockCreate\n\n # Run it!\n runExperiment(sys.argv[1:])", "response": "Run according to options in sys. argv and diff classifiers."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _abbreviate(text, threshold):\n if text is not None and len(text) > threshold:\n text = text[:threshold] + \"...\"\n\n return text", "response": "Abbreviate the given text to threshold chars and append an ellipsis if it is longer than threshold."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __getDBNameForVersion(cls, dbVersion):\n\n # DB Name prefix for the given version\n prefix = cls.__getDBNamePrefixForVersion(dbVersion)\n\n # DB Name suffix\n suffix = Configuration.get('nupic.cluster.database.nameSuffix')\n\n # Replace dash and dot with underscore (e.g. 'ec2-user' or ec2.user will break SQL)\n suffix = suffix.replace(\"-\", \"_\")\n suffix = suffix.replace(\".\", \"_\")\n\n # Create the name of the database for the given DB version\n dbName = '%s_%s' % (prefix, suffix)\n\n return dbName", "response": "Generates the ClientJobs database name for the given DB version"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get():\n\n # Instantiate if needed\n if ClientJobsDAO._instance is None:\n cjDAO = ClientJobsDAO()\n cjDAO.connect()\n\n ClientJobsDAO._instance = cjDAO\n\n # Return the instance to the caller\n return ClientJobsDAO._instance", "response": "Returns the instance of the ClientJobsDAO class created for this process."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _columnNameDBToPublic(self, dbName):\n\n words = dbName.split('_')\n if dbName.startswith('_'):\n words = words[1:]\n pubWords = [words[0]]\n for word in words[1:]:\n pubWords.append(word[0].upper() + word[1:])\n\n return ''.join(pubWords)", "response": "Convert a database internal column name to a public name."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef connect(self, deleteOldVersions=False, recreate=False):\n\n # Initialize tables, if needed\n with ConnectionFactory.get() as conn:\n # Initialize tables\n self._initTables(cursor=conn.cursor, deleteOldVersions=deleteOldVersions,\n recreate=recreate)\n\n # Save our connection id\n conn.cursor.execute('SELECT CONNECTION_ID()')\n self._connectionID = conn.cursor.fetchall()[0][0]\n self._logger.info(\"clientJobsConnectionID=%r\", self._connectionID)\n\n return", "response": "Establish a connection to the current version of the jobs DB or create a new one."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize tables if needed.", "response": "def _initTables(self, cursor, deleteOldVersions, recreate):\n \"\"\" Initialize tables, if needed\n\n Parameters:\n ----------------------------------------------------------------\n cursor: SQL cursor\n deleteOldVersions: if true, delete any old versions of the DB left\n on the server\n recreate: if true, recreate the database from scratch even\n if it already exists.\n \"\"\"\n\n # Delete old versions if they exist\n if deleteOldVersions:\n self._logger.info(\n \"Dropping old versions of client_jobs DB; called from: %r\",\n traceback.format_stack())\n for i in range(self._DB_VERSION):\n cursor.execute('DROP DATABASE IF EXISTS %s' %\n (self.__getDBNameForVersion(i),))\n\n # Create the database if necessary\n if recreate:\n self._logger.info(\n \"Dropping client_jobs DB %r; called from: %r\",\n self.dbName, traceback.format_stack())\n cursor.execute('DROP DATABASE IF EXISTS %s' % (self.dbName))\n\n cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % (self.dbName))\n\n\n # Get the list of tables\n cursor.execute('SHOW TABLES IN %s' % (self.dbName))\n output = cursor.fetchall()\n tableNames = [x[0] for x in output]\n\n # ------------------------------------------------------------------------\n # Create the jobs table if it doesn't exist\n # Fields that start with '_eng' are intended for private use by the engine\n # and should not be used by the UI\n if 'jobs' not in tableNames:\n self._logger.info(\"Creating table %r\", self.jobsTableName)\n fields = [\n 'job_id INT UNSIGNED NOT NULL AUTO_INCREMENT',\n # unique jobID\n 'client CHAR(%d)' % (self.CLIENT_MAX_LEN),\n # name of client (UI, StrmMgr, etc.)\n 'client_info LONGTEXT',\n # Arbitrary data defined by the client\n 'client_key varchar(255)',\n # Foreign key as defined by the client.\n 'cmd_line LONGTEXT',\n # command line to use to launch each worker process\n 'params LONGTEXT',\n # JSON encoded params for the job, for use by the worker processes\n 'job_hash BINARY(%d) DEFAULT NULL' % (self.HASH_MAX_LEN),\n # unique hash of the job, provided by the client. Used for detecting\n # identical job requests from the same client when they use the\n # jobInsertUnique() method.\n 'status VARCHAR(16) DEFAULT \"notStarted\"',\n # One of the STATUS_XXX enumerated value strings\n 'completion_reason VARCHAR(16)',\n # One of the CMPL_REASON_XXX enumerated value strings.\n # NOTE: This is the job completion reason according to the hadoop\n # job-tracker. A success here does not necessarily mean the\n # workers were \"happy\" with the job. To see if the workers\n # failed, check the worker_completion_reason\n 'completion_msg LONGTEXT',\n # Why this job completed, according to job-tracker\n 'worker_completion_reason VARCHAR(16) DEFAULT \"%s\"' % \\\n self.CMPL_REASON_SUCCESS,\n # One of the CMPL_REASON_XXX enumerated value strings. This is\n # may be changed to CMPL_REASON_ERROR if any workers encounter\n # an error while running the job.\n 'worker_completion_msg LONGTEXT',\n # Why this job completed, according to workers. If\n # worker_completion_reason is set to CMPL_REASON_ERROR, this will\n # contain the error information.\n 'cancel BOOLEAN DEFAULT FALSE',\n # set by UI, polled by engine\n 'start_time DATETIME DEFAULT NULL',\n # When job started\n 'end_time DATETIME DEFAULT NULL',\n # When job ended\n 'results LONGTEXT',\n # JSON dict with general information about the results of the job,\n # including the ID and value of the best model\n # TODO: different semantics for results field of ProductionJob\n '_eng_job_type VARCHAR(32)',\n # String used to specify the type of job that this is. Current\n # choices are hypersearch, production worker, or stream worker\n 'minimum_workers INT UNSIGNED DEFAULT 0',\n # min number of desired workers at a time. If 0, no workers will be\n # allocated in a crunch\n 'maximum_workers INT UNSIGNED DEFAULT 0',\n # max number of desired workers at a time. If 0, then use as many\n # as practical given load on the cluster.\n 'priority INT DEFAULT %d' % self.DEFAULT_JOB_PRIORITY,\n # job scheduling priority; 0 is the default priority (\n # ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are higher\n # priority (up to ClientJobsDAO.MAX_JOB_PRIORITY), and negative\n # values are lower priority (down to ClientJobsDAO.MIN_JOB_PRIORITY)\n '_eng_allocate_new_workers BOOLEAN DEFAULT TRUE',\n # Should the scheduling algorithm allocate new workers to this job?\n # If a specialized worker willingly gives up control, we set this\n # field to FALSE to avoid allocating new workers.\n '_eng_untended_dead_workers BOOLEAN DEFAULT FALSE',\n # If a specialized worker fails or is killed by the scheduler, we\n # set this feild to TRUE to indicate that the worker is dead\n 'num_failed_workers INT UNSIGNED DEFAULT 0',\n # The number of failed specialized workers for this job. If the\n # number of failures is >= max.failed.attempts, we mark the job\n # as failed\n 'last_failed_worker_error_msg LONGTEXT',\n # Error message of the most recent specialized failed worker\n '_eng_cleaning_status VARCHAR(16) DEFAULT \"%s\"' % \\\n self.CLEAN_NOT_DONE,\n # Has the job been garbage collected, this includes removing\n # unneeded # model output caches, s3 checkpoints.\n 'gen_base_description LONGTEXT',\n # The contents of the generated description.py file from hypersearch\n # requests. This is generated by the Hypersearch workers and stored\n # here for reference, debugging, and development purposes.\n 'gen_permutations LONGTEXT',\n # The contents of the generated permutations.py file from\n # hypersearch requests. This is generated by the Hypersearch workers\n # and stored here for reference, debugging, and development\n # purposes.\n '_eng_last_update_time DATETIME DEFAULT NULL',\n # time stamp of last update, used for detecting stalled jobs\n '_eng_cjm_conn_id INT UNSIGNED',\n # ID of the CJM starting up this job\n '_eng_worker_state LONGTEXT',\n # JSON encoded state of the hypersearch in progress, for private\n # use by the Hypersearch workers\n '_eng_status LONGTEXT',\n # String used for status messages sent from the engine for\n # informative purposes only. Usually printed periodically by\n # clients watching a job progress.\n '_eng_model_milestones LONGTEXT',\n # JSon encoded object with information about global model milestone\n # results\n\n 'PRIMARY KEY (job_id)',\n 'UNIQUE INDEX (client, job_hash)',\n 'INDEX (status)',\n 'INDEX (client_key)'\n ]\n options = [\n 'AUTO_INCREMENT=1000',\n ]\n\n query = 'CREATE TABLE IF NOT EXISTS %s (%s) %s' % \\\n (self.jobsTableName, ','.join(fields), ','.join(options))\n\n cursor.execute(query)\n\n\n # ------------------------------------------------------------------------\n # Create the models table if it doesn't exist\n # Fields that start with '_eng' are intended for private use by the engine\n # and should not be used by the UI\n if 'models' not in tableNames:\n self._logger.info(\"Creating table %r\", self.modelsTableName)\n fields = [\n 'model_id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT',\n # globally unique model ID\n 'job_id INT UNSIGNED NOT NULL',\n # jobID\n 'params LONGTEXT NOT NULL',\n # JSON encoded params for the model\n 'status VARCHAR(16) DEFAULT \"notStarted\"',\n # One of the STATUS_XXX enumerated value strings\n 'completion_reason VARCHAR(16)',\n # One of the CMPL_REASON_XXX enumerated value strings\n 'completion_msg LONGTEXT',\n # Why this job completed\n 'results LONGTEXT DEFAULT NULL',\n # JSON encoded structure containing metrics produced by the model\n 'optimized_metric FLOAT ',\n #Value of the particular metric we are optimizing in hypersearch\n 'update_counter INT UNSIGNED DEFAULT 0',\n # incremented by engine every time the results is updated\n 'num_records INT UNSIGNED DEFAULT 0',\n # number of records processed so far\n 'start_time DATETIME DEFAULT NULL',\n # When this model started being evaluated\n 'end_time DATETIME DEFAULT NULL',\n # When this model completed\n 'cpu_time FLOAT DEFAULT 0',\n # How much actual CPU time was spent on this model, in seconds. This\n # excludes time the process spent sleeping, or otherwise not\n # actually executing code.\n 'model_checkpoint_id LONGTEXT',\n # Checkpoint identifier for this model (after it has been saved)\n 'gen_description LONGTEXT',\n # The contents of the generated description.py file from hypersearch\n # requests. This is generated by the Hypersearch workers and stored\n # here for reference, debugging, and development purposes.\n '_eng_params_hash BINARY(%d) DEFAULT NULL' % (self.HASH_MAX_LEN),\n # MD5 hash of the params\n '_eng_particle_hash BINARY(%d) DEFAULT NULL' % (self.HASH_MAX_LEN),\n # MD5 hash of the particle info for PSO algorithm\n '_eng_last_update_time DATETIME DEFAULT NULL',\n # time stamp of last update, used for detecting stalled workers\n '_eng_task_tracker_id TINYBLOB',\n # Hadoop Task Tracker ID\n '_eng_worker_id TINYBLOB',\n # Hadoop Map Task ID\n '_eng_attempt_id TINYBLOB',\n # Hadoop Map task attempt ID\n '_eng_worker_conn_id INT DEFAULT 0',\n # database client connection ID of the worker that is running this\n # model\n '_eng_milestones LONGTEXT',\n # A JSON encoded list of metric values for the model at each\n # milestone point\n '_eng_stop VARCHAR(16) DEFAULT NULL',\n # One of the STOP_REASON_XXX enumerated value strings. Set either by\n # the swarm terminator of either the current, or another\n # Hypersearch worker.\n '_eng_matured BOOLEAN DEFAULT FALSE',\n # Set by the model maturity-checker when it decides that this model\n # has \"matured\". This means that it has reached the point of\n # not getting better results with more data.\n 'PRIMARY KEY (model_id)',\n 'UNIQUE INDEX (job_id, _eng_params_hash)',\n 'UNIQUE INDEX (job_id, _eng_particle_hash)',\n ]\n options = [\n 'AUTO_INCREMENT=1000',\n ]\n\n query = 'CREATE TABLE IF NOT EXISTS %s (%s) %s' % \\\n (self.modelsTableName, ','.join(fields), ','.join(options))\n\n cursor.execute(query)\n\n\n # ---------------------------------------------------------------------\n # Get the field names for each table\n cursor.execute('DESCRIBE %s' % (self.jobsTableName))\n fields = cursor.fetchall()\n self._jobs.dbFieldNames = [str(field[0]) for field in fields]\n\n cursor.execute('DESCRIBE %s' % (self.modelsTableName))\n fields = cursor.fetchall()\n self._models.dbFieldNames = [str(field[0]) for field in fields]\n\n\n # ---------------------------------------------------------------------\n # Generate the public names\n self._jobs.publicFieldNames = [self._columnNameDBToPublic(x)\n for x in self._jobs.dbFieldNames]\n self._models.publicFieldNames = [self._columnNameDBToPublic(x)\n for x in self._models.dbFieldNames]\n\n\n # ---------------------------------------------------------------------\n # Generate the name conversion dicts\n self._jobs.pubToDBNameDict = dict(\n zip(self._jobs.publicFieldNames, self._jobs.dbFieldNames))\n self._jobs.dbToPubNameDict = dict(\n zip(self._jobs.dbFieldNames, self._jobs.publicFieldNames))\n self._models.pubToDBNameDict = dict(\n zip(self._models.publicFieldNames, self._models.dbFieldNames))\n self._models.dbToPubNameDict = dict(\n zip(self._models.dbFieldNames, self._models.publicFieldNames))\n\n\n # ---------------------------------------------------------------------\n # Generate the dynamic namedtuple classes we use\n self._models.modelInfoNamedTuple = collections.namedtuple(\n '_modelInfoNamedTuple', self._models.publicFieldNames)\n\n self._jobs.jobInfoNamedTuple = collections.namedtuple(\n '_jobInfoNamedTuple', self._jobs.publicFieldNames)\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _getMatchingRowsNoRetries(self, tableInfo, conn, fieldsToMatch,\n selectFieldNames, maxRows=None):\n \"\"\" Return a sequence of matching rows with the requested field values from\n a table or empty sequence if nothing matched.\n\n tableInfo: Table information: a ClientJobsDAO._TableInfoBase instance\n conn: Owned connection acquired from ConnectionFactory.get()\n fieldsToMatch: Dictionary of internal fieldName/value mappings that\n identify the desired rows. If a value is an instance of\n ClientJobsDAO._SEQUENCE_TYPES (list/set/tuple), then the\n operator 'IN' will be used in the corresponding SQL\n predicate; if the value is bool: \"IS TRUE/FALSE\"; if the\n value is None: \"IS NULL\"; '=' will be used for all other\n cases.\n selectFieldNames:\n list of fields to return, using internal field names\n maxRows: maximum number of rows to return; unlimited if maxRows\n is None\n\n retval: A sequence of matching rows, each row consisting of field\n values in the order of the requested field names. Empty\n sequence is returned when not match exists.\n \"\"\"\n\n assert fieldsToMatch, repr(fieldsToMatch)\n assert all(k in tableInfo.dbFieldNames\n for k in fieldsToMatch.iterkeys()), repr(fieldsToMatch)\n\n assert selectFieldNames, repr(selectFieldNames)\n assert all(f in tableInfo.dbFieldNames for f in selectFieldNames), repr(\n selectFieldNames)\n\n # NOTE: make sure match expressions and values are in the same order\n matchPairs = fieldsToMatch.items()\n matchExpressionGen = (\n p[0] +\n (' IS ' + {True:'TRUE', False:'FALSE'}[p[1]] if isinstance(p[1], bool)\n else ' IS NULL' if p[1] is None\n else ' IN %s' if isinstance(p[1], self._SEQUENCE_TYPES)\n else '=%s')\n for p in matchPairs)\n matchFieldValues = [p[1] for p in matchPairs\n if (not isinstance(p[1], (bool)) and p[1] is not None)]\n\n query = 'SELECT %s FROM %s WHERE (%s)' % (\n ','.join(selectFieldNames), tableInfo.tableName,\n ' AND '.join(matchExpressionGen))\n sqlParams = matchFieldValues\n if maxRows is not None:\n query += ' LIMIT %s'\n sqlParams.append(maxRows)\n\n conn.cursor.execute(query, sqlParams)\n rows = conn.cursor.fetchall()\n\n if rows:\n assert maxRows is None or len(rows) <= maxRows, \"%d !<= %d\" % (\n len(rows), maxRows)\n assert len(rows[0]) == len(selectFieldNames), \"%d != %d\" % (\n len(rows[0]), len(selectFieldNames))\n else:\n rows = tuple()\n\n return rows", "response": "Returns a sequence of matching rows with the requested field values from the specified table."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nliking _getMatchingRowsNoRetries but with retries on transient MySQL failures", "response": "def _getMatchingRowsWithRetries(self, tableInfo, fieldsToMatch,\n selectFieldNames, maxRows=None):\n \"\"\" Like _getMatchingRowsNoRetries(), but with retries on transient MySQL\n failures\n \"\"\"\n with ConnectionFactory.get() as conn:\n return self._getMatchingRowsNoRetries(tableInfo, conn, fieldsToMatch,\n selectFieldNames, maxRows)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a single matching row with the requested field values from the specified table.", "response": "def _getOneMatchingRowNoRetries(self, tableInfo, conn, fieldsToMatch,\n selectFieldNames):\n \"\"\" Return a single matching row with the requested field values from the\n the requested table or None if nothing matched.\n\n tableInfo: Table information: a ClientJobsDAO._TableInfoBase instance\n conn: Owned connection acquired from ConnectionFactory.get()\n fieldsToMatch: Dictionary of internal fieldName/value mappings that\n identify the desired rows. If a value is an instance of\n ClientJobsDAO._SEQUENCE_TYPES (list/set/tuple), then the\n operator 'IN' will be used in the corresponding SQL\n predicate; if the value is bool: \"IS TRUE/FALSE\"; if the\n value is None: \"IS NULL\"; '=' will be used for all other\n cases.\n selectFieldNames:\n list of fields to return, using internal field names\n\n retval: A sequence of field values of the matching row in the order\n of the given field names; or None if there was no match.\n \"\"\"\n rows = self._getMatchingRowsNoRetries(tableInfo, conn, fieldsToMatch,\n selectFieldNames, maxRows=1)\n if rows:\n assert len(rows) == 1, repr(len(rows))\n result = rows[0]\n else:\n result = None\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _getOneMatchingRowWithRetries(self, tableInfo, fieldsToMatch,\n selectFieldNames):\n \"\"\" Like _getOneMatchingRowNoRetries(), but with retries on transient MySQL\n failures\n \"\"\"\n with ConnectionFactory.get() as conn:\n return self._getOneMatchingRowNoRetries(tableInfo, conn, fieldsToMatch,\n selectFieldNames)", "response": "Like _getOneMatchingRowNoRetries but with retries on transient MySQL\n failures"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _insertOrGetUniqueJobNoRetries(\n self, conn, client, cmdLine, jobHash, clientInfo, clientKey, params,\n minimumWorkers, maximumWorkers, jobType, priority, alreadyRunning):\n \"\"\" Attempt to insert a row with the given parameters into the jobs table.\n Return jobID of the inserted row, or of an existing row with matching\n client/jobHash key.\n\n The combination of client and jobHash are expected to be unique (enforced\n by a unique index on the two columns).\n\n NOTE: It's possibe that this or another process (on this or another machine)\n already inserted a row with matching client/jobHash key (e.g.,\n StreamMgr). This may also happen undetected by this function due to a\n partially-successful insert operation (e.g., row inserted, but then\n connection was lost while reading response) followed by retries either of\n this function or in SteadyDB module.\n\n Parameters:\n ----------------------------------------------------------------\n conn: Owned connection acquired from ConnectionFactory.get()\n client: Name of the client submitting the job\n cmdLine: Command line to use to launch each worker process; must be\n a non-empty string\n jobHash: unique hash of this job. The caller must insure that this,\n together with client, uniquely identifies this job request\n for the purposes of detecting duplicates.\n clientInfo: JSON encoded dict of client specific information.\n clientKey: Foreign key.\n params: JSON encoded dict of the parameters for the job. This\n can be fetched out of the database by the worker processes\n based on the jobID.\n minimumWorkers: minimum number of workers design at a time.\n maximumWorkers: maximum number of workers desired at a time.\n priority: Job scheduling priority; 0 is the default priority (\n ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are\n higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),\n and negative values are lower priority (down to\n ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will\n be scheduled to run at the expense of the lower-priority\n jobs, and higher-priority job tasks will preempt those\n with lower priority if there is inadequate supply of\n scheduling slots. Excess lower priority job tasks will\n starve as long as slot demand exceeds supply. Most jobs\n should be scheduled with DEFAULT_JOB_PRIORITY. System jobs\n that must run at all cost, such as Multi-Model-Master,\n should be scheduled with MAX_JOB_PRIORITY.\n alreadyRunning: Used for unit test purposes only. This inserts the job\n in the running state. It is used when running a worker\n in standalone mode without hadoop- it gives it a job\n record to work with.\n\n retval: jobID of the inserted jobs row, or of an existing jobs row\n with matching client/jobHash key\n \"\"\"\n\n assert len(client) <= self.CLIENT_MAX_LEN, \"client too long:\" + repr(client)\n assert cmdLine, \"Unexpected empty or None command-line: \" + repr(cmdLine)\n assert len(jobHash) == self.HASH_MAX_LEN, \"wrong hash len=%d\" % len(jobHash)\n\n # Initial status\n if alreadyRunning:\n # STATUS_TESTMODE, so that scheduler won't pick it up (for in-proc tests)\n initStatus = self.STATUS_TESTMODE\n else:\n initStatus = self.STATUS_NOTSTARTED\n\n # Create a new job entry\n query = 'INSERT IGNORE INTO %s (status, client, client_info, client_key,' \\\n 'cmd_line, params, job_hash, _eng_last_update_time, ' \\\n 'minimum_workers, maximum_workers, priority, _eng_job_type) ' \\\n ' VALUES (%%s, %%s, %%s, %%s, %%s, %%s, %%s, ' \\\n ' UTC_TIMESTAMP(), %%s, %%s, %%s, %%s) ' \\\n % (self.jobsTableName,)\n sqlParams = (initStatus, client, clientInfo, clientKey, cmdLine, params,\n jobHash, minimumWorkers, maximumWorkers, priority, jobType)\n numRowsInserted = conn.cursor.execute(query, sqlParams)\n\n jobID = 0\n\n if numRowsInserted == 1:\n # Get the chosen job id\n # NOTE: LAST_INSERT_ID() returns 0 after intermittent connection failure\n conn.cursor.execute('SELECT LAST_INSERT_ID()')\n jobID = conn.cursor.fetchall()[0][0]\n if jobID == 0:\n self._logger.warn(\n '_insertOrGetUniqueJobNoRetries: SELECT LAST_INSERT_ID() returned 0; '\n 'likely due to reconnection in SteadyDB following INSERT. '\n 'jobType=%r; client=%r; clientInfo=%r; clientKey=%s; jobHash=%r; '\n 'cmdLine=%r',\n jobType, client, _abbreviate(clientInfo, 32), clientKey, jobHash,\n cmdLine)\n else:\n # Assumption: nothing was inserted because this is a retry and the row\n # with this client/hash already exists from our prior\n # partially-successful attempt; or row with matching client/jobHash was\n # inserted already by some process on some machine.\n assert numRowsInserted == 0, repr(numRowsInserted)\n\n if jobID == 0:\n # Recover from intermittent failure in a partially-successful attempt;\n # or row with matching client/jobHash was already in table\n row = self._getOneMatchingRowNoRetries(\n self._jobs, conn, dict(client=client, job_hash=jobHash), ['job_id'])\n assert row is not None\n assert len(row) == 1, 'Unexpected num fields: ' + repr(len(row))\n jobID = row[0]\n\n # ---------------------------------------------------------------------\n # If asked to enter the job in the running state, set the connection id\n # and start time as well\n if alreadyRunning:\n query = 'UPDATE %s SET _eng_cjm_conn_id=%%s, ' \\\n ' start_time=UTC_TIMESTAMP(), ' \\\n ' _eng_last_update_time=UTC_TIMESTAMP() ' \\\n ' WHERE job_id=%%s' \\\n % (self.jobsTableName,)\n conn.cursor.execute(query, (self._connectionID, jobID))\n\n return jobID", "response": "Insert or get a unique job into the jobs table."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nresumes processing of an existing job that is not presently in the status of STATUS_COMPLETED state.", "response": "def _resumeJobNoRetries(self, conn, jobID, alreadyRunning):\n \"\"\" Resumes processing of an existing job that is presently in the\n STATUS_COMPLETED state.\n\n NOTE: this is primarily for resuming suspended Production and Stream Jobs; DO\n NOT use it on Hypersearch jobs.\n\n This prepares an existing job entry to resume processing. The CJM is always\n periodically sweeping the jobs table and when it finds a job that is ready\n to run, it will proceed to start it up on Hadoop.\n\n Parameters:\n ----------------------------------------------------------------\n conn: Owned connection acquired from ConnectionFactory.get()\n jobID: jobID of the job to resume\n alreadyRunning: Used for unit test purposes only. This inserts the job\n in the running state. It is used when running a worker\n in standalone mode without hadoop.\n\n raises: Throws a RuntimeError if no rows are affected. This could\n either be because:\n 1) Because there was not matching jobID\n 2) or if the status of the job was not STATUS_COMPLETED.\n\n retval: nothing\n \"\"\"\n\n # Initial status\n if alreadyRunning:\n # Use STATUS_TESTMODE so scheduler will leave our row alone\n initStatus = self.STATUS_TESTMODE\n else:\n initStatus = self.STATUS_NOTSTARTED\n\n # NOTE: some of our clients (e.g., StreamMgr) may call us (directly or\n # indirectly) for the same job from different processes (even different\n # machines), so we should be prepared for the update to fail; same holds\n # if the UPDATE succeeds, but connection fails while reading result\n assignments = [\n 'status=%s',\n 'completion_reason=DEFAULT',\n 'completion_msg=DEFAULT',\n 'worker_completion_reason=DEFAULT',\n 'worker_completion_msg=DEFAULT',\n 'end_time=DEFAULT',\n 'cancel=DEFAULT',\n '_eng_last_update_time=UTC_TIMESTAMP()',\n '_eng_allocate_new_workers=DEFAULT',\n '_eng_untended_dead_workers=DEFAULT',\n 'num_failed_workers=DEFAULT',\n 'last_failed_worker_error_msg=DEFAULT',\n '_eng_cleaning_status=DEFAULT',\n ]\n assignmentValues = [initStatus]\n\n if alreadyRunning:\n assignments += ['_eng_cjm_conn_id=%s', 'start_time=UTC_TIMESTAMP()',\n '_eng_last_update_time=UTC_TIMESTAMP()']\n assignmentValues.append(self._connectionID)\n else:\n assignments += ['_eng_cjm_conn_id=DEFAULT', 'start_time=DEFAULT']\n\n assignments = ', '.join(assignments)\n\n query = 'UPDATE %s SET %s ' \\\n ' WHERE job_id=%%s AND status=%%s' \\\n % (self.jobsTableName, assignments)\n sqlParams = assignmentValues + [jobID, self.STATUS_COMPLETED]\n\n numRowsAffected = conn.cursor.execute(query, sqlParams)\n\n assert numRowsAffected <= 1, repr(numRowsAffected)\n\n if numRowsAffected == 0:\n self._logger.info(\n \"_resumeJobNoRetries: Redundant job-resume UPDATE: job was not \"\n \"suspended or was resumed by another process or operation was retried \"\n \"after connection failure; jobID=%s\", jobID)\n\n return"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nresume processing of an existing job.", "response": "def jobResume(self, jobID, alreadyRunning=False):\n \"\"\" Resumes processing of an existing job that is presently in the\n STATUS_COMPLETED state.\n\n NOTE: this is primarily for resuming suspended Production Jobs; DO NOT use\n it on Hypersearch jobs.\n\n NOTE: The job MUST be in the STATUS_COMPLETED state at the time of this\n call, otherwise an exception will be raised.\n\n This prepares an existing job entry to resume processing. The CJM is always\n periodically sweeping the jobs table and when it finds a job that is ready\n to run, will proceed to start it up on Hadoop.\n\n Parameters:\n ----------------------------------------------------------------\n job: jobID of the job to resume\n alreadyRunning: Used for unit test purposes only. This inserts the job\n in the running state. It is used when running a worker\n in standalone mode without hadoop.\n\n raises: Throws a RuntimeError if no rows are affected. This could\n either be because:\n 1) Because there was not matching jobID\n 2) or if the status of the job was not STATUS_COMPLETED.\n\n retval: nothing\n \"\"\"\n\n row = self.jobGetFields(jobID, ['status'])\n (jobStatus,) = row\n if jobStatus != self.STATUS_COMPLETED:\n raise RuntimeError((\"Failed to resume job: job was not suspended; \"\n \"jobID=%s; job status=%r\") % (jobID, jobStatus))\n\n # NOTE: on MySQL failures, we need to retry ConnectionFactory.get() as well\n # in order to recover from lost connections\n @g_retrySQL\n def resumeWithRetries():\n with ConnectionFactory.get() as conn:\n self._resumeJobNoRetries(conn, jobID, alreadyRunning)\n\n resumeWithRetries()\n return"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd an entry to the jobs table for a new job request. This is called by clients that wish to startup a new job, like a Hypersearch, stream job, or specific model evaluation from the engine. This puts a new entry into the jobs table. The CJM is always periodically sweeping the jobs table and when it finds a new job, will proceed to start it up on Hadoop. Parameters: ---------------------------------------------------------------- client: Name of the client submitting the job cmdLine: Command line to use to launch each worker process; must be a non-empty string clientInfo: JSON encoded dict of client specific information. clientKey: Foreign key. params: JSON encoded dict of the parameters for the job. This can be fetched out of the database by the worker processes based on the jobID. alreadyRunning: Used for unit test purposes only. This inserts the job in the running state. It is used when running a worker in standalone mode without hadoop - it gives it a job record to work with. minimumWorkers: minimum number of workers design at a time. maximumWorkers: maximum number of workers desired at a time. jobType: The type of job that this is. This should be one of the JOB_TYPE_XXXX enums. This is needed to allow a standard way of recognizing a job's function and capabilities. priority: Job scheduling priority; 0 is the default priority ( ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY), and negative values are lower priority (down to ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will be scheduled to run at the expense of the lower-priority jobs, and higher-priority job tasks will preempt those with lower priority if there is inadequate supply of scheduling slots. Excess lower priority job tasks will starve as long as slot demand exceeds supply. Most jobs should be scheduled with DEFAULT_JOB_PRIORITY. System jobs that must run at all cost, such as Multi-Model-Master, should be scheduled with MAX_JOB_PRIORITY. retval: jobID - unique ID assigned to this job", "response": "def jobInsert(self, client, cmdLine, clientInfo='', clientKey='', params='',\n alreadyRunning=False, minimumWorkers=0, maximumWorkers=0,\n jobType='', priority=DEFAULT_JOB_PRIORITY):\n \"\"\" Add an entry to the jobs table for a new job request. This is called by\n clients that wish to startup a new job, like a Hypersearch, stream job, or\n specific model evaluation from the engine.\n\n This puts a new entry into the jobs table. The CJM is always periodically\n sweeping the jobs table and when it finds a new job, will proceed to start it\n up on Hadoop.\n\n Parameters:\n ----------------------------------------------------------------\n client: Name of the client submitting the job\n cmdLine: Command line to use to launch each worker process; must be\n a non-empty string\n clientInfo: JSON encoded dict of client specific information.\n clientKey: Foreign key.\n params: JSON encoded dict of the parameters for the job. This\n can be fetched out of the database by the worker processes\n based on the jobID.\n alreadyRunning: Used for unit test purposes only. This inserts the job\n in the running state. It is used when running a worker\n in standalone mode without hadoop - it gives it a job\n record to work with.\n minimumWorkers: minimum number of workers design at a time.\n maximumWorkers: maximum number of workers desired at a time.\n jobType: The type of job that this is. This should be one of the\n JOB_TYPE_XXXX enums. This is needed to allow a standard\n way of recognizing a job's function and capabilities.\n priority: Job scheduling priority; 0 is the default priority (\n ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are\n higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),\n and negative values are lower priority (down to\n ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will\n be scheduled to run at the expense of the lower-priority\n jobs, and higher-priority job tasks will preempt those\n with lower priority if there is inadequate supply of\n scheduling slots. Excess lower priority job tasks will\n starve as long as slot demand exceeds supply. Most jobs\n should be scheduled with DEFAULT_JOB_PRIORITY. System jobs\n that must run at all cost, such as Multi-Model-Master,\n should be scheduled with MAX_JOB_PRIORITY.\n\n retval: jobID - unique ID assigned to this job\n \"\"\"\n\n jobHash = self._normalizeHash(uuid.uuid1().bytes)\n\n @g_retrySQL\n def insertWithRetries():\n with ConnectionFactory.get() as conn:\n return self._insertOrGetUniqueJobNoRetries(\n conn, client=client, cmdLine=cmdLine, jobHash=jobHash,\n clientInfo=clientInfo, clientKey=clientKey, params=params,\n minimumWorkers=minimumWorkers, maximumWorkers=maximumWorkers,\n jobType=jobType, priority=priority, alreadyRunning=alreadyRunning)\n\n try:\n jobID = insertWithRetries()\n except:\n self._logger.exception(\n 'jobInsert FAILED: jobType=%r; client=%r; clientInfo=%r; clientKey=%r;'\n 'jobHash=%r; cmdLine=%r',\n jobType, client, _abbreviate(clientInfo, 48), clientKey, jobHash,\n cmdLine)\n raise\n else:\n self._logger.info(\n 'jobInsert: returning jobID=%s. jobType=%r; client=%r; clientInfo=%r; '\n 'clientKey=%r; jobHash=%r; cmdLine=%r',\n jobID, jobType, client, _abbreviate(clientInfo, 48), clientKey,\n jobHash, cmdLine)\n\n return jobID"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding an entry to the jobs table for a new job request, but only if the same job, by the same client is not already running. If the job is already running, or queued up to run, this call does nothing. If the job does not exist in the jobs table or has completed, it will be inserted and/or started up again. This method is called by clients, like StreamMgr, that wish to only start up a job if it hasn't already been started up. Parameters: ---------------------------------------------------------------- client: Name of the client submitting the job cmdLine: Command line to use to launch each worker process; must be a non-empty string jobHash: unique hash of this job. The client must insure that this uniquely identifies this job request for the purposes of detecting duplicates. clientInfo: JSON encoded dict of client specific information. clientKey: Foreign key. params: JSON encoded dict of the parameters for the job. This can be fetched out of the database by the worker processes based on the jobID. minimumWorkers: minimum number of workers design at a time. maximumWorkers: maximum number of workers desired at a time. jobType: The type of job that this is. This should be one of the JOB_TYPE_XXXX enums. This is needed to allow a standard way of recognizing a job's function and capabilities. priority: Job scheduling priority; 0 is the default priority ( ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY), and negative values are lower priority (down to ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will be scheduled to run at the expense of the lower-priority jobs, and higher-priority job tasks will preempt those with lower priority if there is inadequate supply of scheduling slots. Excess lower priority job tasks will starve as long as slot demand exceeds supply. Most jobs should be scheduled with DEFAULT_JOB_PRIORITY. System jobs that must run at all cost, such as Multi-Model-Master, should be scheduled with MAX_JOB_PRIORITY. retval: jobID of the newly inserted or existing job.", "response": "def jobInsertUnique(self, client, cmdLine, jobHash, clientInfo='',\n clientKey='', params='', minimumWorkers=0,\n maximumWorkers=0, jobType='',\n priority=DEFAULT_JOB_PRIORITY):\n \"\"\" Add an entry to the jobs table for a new job request, but only if the\n same job, by the same client is not already running. If the job is already\n running, or queued up to run, this call does nothing. If the job does not\n exist in the jobs table or has completed, it will be inserted and/or started\n up again.\n\n This method is called by clients, like StreamMgr, that wish to only start up\n a job if it hasn't already been started up.\n\n Parameters:\n ----------------------------------------------------------------\n client: Name of the client submitting the job\n cmdLine: Command line to use to launch each worker process; must be\n a non-empty string\n jobHash: unique hash of this job. The client must insure that this\n uniquely identifies this job request for the purposes\n of detecting duplicates.\n clientInfo: JSON encoded dict of client specific information.\n clientKey: Foreign key.\n params: JSON encoded dict of the parameters for the job. This\n can be fetched out of the database by the worker processes\n based on the jobID.\n minimumWorkers: minimum number of workers design at a time.\n maximumWorkers: maximum number of workers desired at a time.\n jobType: The type of job that this is. This should be one of the\n JOB_TYPE_XXXX enums. This is needed to allow a standard\n way of recognizing a job's function and capabilities.\n priority: Job scheduling priority; 0 is the default priority (\n ClientJobsDAO.DEFAULT_JOB_PRIORITY); positive values are\n higher priority (up to ClientJobsDAO.MAX_JOB_PRIORITY),\n and negative values are lower priority (down to\n ClientJobsDAO.MIN_JOB_PRIORITY). Higher-priority jobs will\n be scheduled to run at the expense of the lower-priority\n jobs, and higher-priority job tasks will preempt those\n with lower priority if there is inadequate supply of\n scheduling slots. Excess lower priority job tasks will\n starve as long as slot demand exceeds supply. Most jobs\n should be scheduled with DEFAULT_JOB_PRIORITY. System jobs\n that must run at all cost, such as Multi-Model-Master,\n should be scheduled with MAX_JOB_PRIORITY.\n\n retval: jobID of the newly inserted or existing job.\n \"\"\"\n\n assert cmdLine, \"Unexpected empty or None command-line: \" + repr(cmdLine)\n\n @g_retrySQL\n def insertUniqueWithRetries():\n jobHashValue = self._normalizeHash(jobHash)\n\n jobID = None\n with ConnectionFactory.get() as conn:\n row = self._getOneMatchingRowNoRetries(\n self._jobs, conn, dict(client=client, job_hash=jobHashValue),\n ['job_id', 'status'])\n\n if row is not None:\n (jobID, status) = row\n\n if status == self.STATUS_COMPLETED:\n # Restart existing job that had completed\n query = 'UPDATE %s SET client_info=%%s, ' \\\n ' client_key=%%s, ' \\\n ' cmd_line=%%s, ' \\\n ' params=%%s, ' \\\n ' minimum_workers=%%s, ' \\\n ' maximum_workers=%%s, ' \\\n ' priority=%%s, '\\\n ' _eng_job_type=%%s ' \\\n ' WHERE (job_id=%%s AND status=%%s)' \\\n % (self.jobsTableName,)\n sqlParams = (clientInfo, clientKey, cmdLine, params,\n minimumWorkers, maximumWorkers, priority,\n jobType, jobID, self.STATUS_COMPLETED)\n\n numRowsUpdated = conn.cursor.execute(query, sqlParams)\n assert numRowsUpdated <= 1, repr(numRowsUpdated)\n\n if numRowsUpdated == 0:\n self._logger.info(\n \"jobInsertUnique: Redundant job-reuse UPDATE: job restarted by \"\n \"another process, values were unchanged, or operation was \"\n \"retried after connection failure; jobID=%s\", jobID)\n\n # Restart the job, unless another process beats us to it\n self._resumeJobNoRetries(conn, jobID, alreadyRunning=False)\n else:\n # There was no job row with matching client/jobHash, so insert one\n jobID = self._insertOrGetUniqueJobNoRetries(\n conn, client=client, cmdLine=cmdLine, jobHash=jobHashValue,\n clientInfo=clientInfo, clientKey=clientKey, params=params,\n minimumWorkers=minimumWorkers, maximumWorkers=maximumWorkers,\n jobType=jobType, priority=priority, alreadyRunning=False)\n\n return jobID\n\n try:\n jobID = insertUniqueWithRetries()\n except:\n self._logger.exception(\n 'jobInsertUnique FAILED: jobType=%r; client=%r; '\n 'clientInfo=%r; clientKey=%r; jobHash=%r; cmdLine=%r',\n jobType, client, _abbreviate(clientInfo, 48), clientKey, jobHash,\n cmdLine)\n raise\n else:\n self._logger.info(\n 'jobInsertUnique: returning jobID=%s. jobType=%r; client=%r; '\n 'clientInfo=%r; clientKey=%r; jobHash=%r; cmdLine=%r',\n jobID, jobType, client, _abbreviate(clientInfo, 48), clientKey,\n jobHash, cmdLine)\n\n return jobID"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nplacing the given job in STATUS_RUNNING mode and retry the job if it has failed.", "response": "def _startJobWithRetries(self, jobID):\n \"\"\" Place the given job in STATUS_RUNNING mode; the job is expected to be\n STATUS_NOTSTARTED.\n\n NOTE: this function was factored out of jobStartNext because it's also\n needed for testing (e.g., test_client_jobs_dao.py)\n \"\"\"\n with ConnectionFactory.get() as conn:\n query = 'UPDATE %s SET status=%%s, ' \\\n ' _eng_cjm_conn_id=%%s, ' \\\n ' start_time=UTC_TIMESTAMP(), ' \\\n ' _eng_last_update_time=UTC_TIMESTAMP() ' \\\n ' WHERE (job_id=%%s AND status=%%s)' \\\n % (self.jobsTableName,)\n sqlParams = [self.STATUS_RUNNING, self._connectionID,\n jobID, self.STATUS_NOTSTARTED]\n numRowsUpdated = conn.cursor.execute(query, sqlParams)\n if numRowsUpdated != 1:\n self._logger.warn('jobStartNext: numRowsUpdated=%r instead of 1; '\n 'likely side-effect of transient connection '\n 'failure', numRowsUpdated)\n return"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef jobStartNext(self):\n\n # NOTE: cursor.execute('SELECT @update_id') trick is unreliable: if a\n # connection loss occurs during cursor.execute, then the server-cached\n # information is lost, and we cannot get the updated job ID; so, we use\n # this select instead\n row = self._getOneMatchingRowWithRetries(\n self._jobs, dict(status=self.STATUS_NOTSTARTED), ['job_id'])\n if row is None:\n return None\n\n (jobID,) = row\n\n self._startJobWithRetries(jobID)\n\n return jobID", "response": "This method is used by ClientJobManager to start the next job in the sequence."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef jobReactivateRunningJobs(self):\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n\n query = 'UPDATE %s SET _eng_cjm_conn_id=%%s, ' \\\n ' _eng_allocate_new_workers=TRUE ' \\\n ' WHERE status=%%s ' \\\n % (self.jobsTableName,)\n conn.cursor.execute(query, [self._connectionID, self.STATUS_RUNNING])\n\n return", "response": "Reactivate all running jobs in the jobs table."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef jobGetDemand(self,):\n rows = self._getMatchingRowsWithRetries(\n self._jobs, dict(status=self.STATUS_RUNNING),\n [self._jobs.pubToDBNameDict[f]\n for f in self._jobs.jobDemandNamedTuple._fields])\n\n return [self._jobs.jobDemandNamedTuple._make(r) for r in rows]", "response": "Returns a list of job demand entries."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncanceling all currently - running jobs.", "response": "def jobCancelAllRunningJobs(self):\n \"\"\" Set cancel field of all currently-running jobs to true.\n \"\"\"\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n\n query = 'UPDATE %s SET cancel=TRUE WHERE status<>%%s ' \\\n % (self.jobsTableName,)\n conn.cursor.execute(query, [self.STATUS_COMPLETED])\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the number of jobs that are in the cancel field set to True.", "response": "def jobCountCancellingJobs(self,):\n \"\"\" Look through the jobs table and count the running jobs whose\n cancel field is true.\n\n Parameters:\n ----------------------------------------------------------------\n retval: A count of running jobs with the cancel field set to true.\n \"\"\"\n with ConnectionFactory.get() as conn:\n query = 'SELECT COUNT(job_id) '\\\n 'FROM %s ' \\\n 'WHERE (status<>%%s AND cancel is TRUE)' \\\n % (self.jobsTableName,)\n\n conn.cursor.execute(query, [self.STATUS_COMPLETED])\n rows = conn.cursor.fetchall()\n\n return rows[0][0]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef jobGetCancellingJobs(self,):\n with ConnectionFactory.get() as conn:\n query = 'SELECT job_id '\\\n 'FROM %s ' \\\n 'WHERE (status<>%%s AND cancel is TRUE)' \\\n % (self.jobsTableName,)\n conn.cursor.execute(query, [self.STATUS_COMPLETED])\n rows = conn.cursor.fetchall()\n\n return tuple(r[0] for r in rows)", "response": "Returns a tuple of all running jobs whose cancel field is true."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef partitionAtIntervals(data, intervals):\n assert sum(intervals) <= len(data)\n\n start = 0\n for interval in intervals:\n end = start + interval\n yield data[start:end]\n start = end\n\n raise StopIteration", "response": "Generator that yields data at a set of intervals."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncombines a single database result into a single list of namedtuples.", "response": "def _combineResults(result, *namedTuples):\n \"\"\" Return a list of namedtuples from the result of a join query. A\n single database result is partitioned at intervals corresponding to the\n fields in namedTuples. The return value is the result of applying\n namedtuple._make() to each of the partitions, for each of the namedTuples.\n\n Parameters:\n ----------------------------------------------------------------\n result: Tuple representing a single result from a database query\n *namedTuples: List of named tuples.\n\n \"\"\"\n results = ClientJobsDAO.partitionAtIntervals(\n result, [len(nt._fields) for nt in namedTuples])\n return [nt._make(result) for nt, result in zip(namedTuples, results)]"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef jobInfoWithModels(self, jobID):\n\n # Get a database connection and cursor\n combinedResults = None\n\n with ConnectionFactory.get() as conn:\n # NOTE: Since we're using a LEFT JOIN on the models table, there need not\n # be a matching row in the models table, but the matching row from the\n # jobs table will still be returned (along with all fields from the models\n # table with values of None in case there were no matchings models)\n query = ' '.join([\n 'SELECT %s.*, %s.*' % (self.jobsTableName, self.modelsTableName),\n 'FROM %s' % self.jobsTableName,\n 'LEFT JOIN %s USING(job_id)' % self.modelsTableName,\n 'WHERE job_id=%s'])\n\n conn.cursor.execute(query, (jobID,))\n\n if conn.cursor.rowcount > 0:\n combinedResults = [\n ClientJobsDAO._combineResults(\n result, self._jobs.jobInfoNamedTuple,\n self._models.modelInfoNamedTuple\n ) for result in conn.cursor.fetchall()]\n\n if combinedResults is not None:\n return combinedResults\n\n raise RuntimeError(\"jobID=%s not found within the jobs table\" % (jobID))", "response": "Get all info about a job with models."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef jobInfo(self, jobID):\n row = self._getOneMatchingRowWithRetries(\n self._jobs, dict(job_id=jobID),\n [self._jobs.pubToDBNameDict[n]\n for n in self._jobs.jobInfoNamedTuple._fields])\n\n if row is None:\n raise RuntimeError(\"jobID=%s not found within the jobs table\" % (jobID))\n\n # Create a namedtuple with the names to values\n return self._jobs.jobInfoNamedTuple._make(row)", "response": "Returns all info about a job in the jobInfoTable"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef jobSetStatus(self, jobID, status, useConnectionID=True,):\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n query = 'UPDATE %s SET status=%%s, ' \\\n ' _eng_last_update_time=UTC_TIMESTAMP() ' \\\n ' WHERE job_id=%%s' \\\n % (self.jobsTableName,)\n sqlParams = [status, jobID]\n\n if useConnectionID:\n query += ' AND _eng_cjm_conn_id=%s'\n sqlParams.append(self._connectionID)\n\n result = conn.cursor.execute(query, sqlParams)\n\n if result != 1:\n raise RuntimeError(\"Tried to change the status of job %d to %s, but \"\n \"this job belongs to some other CJM\" % (\n jobID, status))", "response": "Change the status of a given job"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef jobSetCompleted(self, jobID, completionReason, completionMsg,\n useConnectionID = True):\n \"\"\" Change the status on the given job to completed\n\n Parameters:\n ----------------------------------------------------------------\n job: jobID of the job to mark as completed\n completionReason: completionReason string\n completionMsg: completionMsg string\n\n useConnectionID: True if the connection id of the calling function\n must be the same as the connection that created the job. Set\n to False for hypersearch workers\n \"\"\"\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n query = 'UPDATE %s SET status=%%s, ' \\\n ' completion_reason=%%s, ' \\\n ' completion_msg=%%s, ' \\\n ' end_time=UTC_TIMESTAMP(), ' \\\n ' _eng_last_update_time=UTC_TIMESTAMP() ' \\\n ' WHERE job_id=%%s' \\\n % (self.jobsTableName,)\n sqlParams = [self.STATUS_COMPLETED, completionReason, completionMsg,\n jobID]\n\n if useConnectionID:\n query += ' AND _eng_cjm_conn_id=%s'\n sqlParams.append(self._connectionID)\n\n result = conn.cursor.execute(query, sqlParams)\n\n if result != 1:\n raise RuntimeError(\"Tried to change the status of jobID=%s to \"\n \"completed, but this job could not be found or \"\n \"belongs to some other CJM\" % (jobID))", "response": "Change the status of a given job to completed."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncancels the given job.", "response": "def jobCancel(self, jobID):\n \"\"\" Cancel the given job. This will update the cancel field in the\n jobs table and will result in the job being cancelled.\n\n Parameters:\n ----------------------------------------------------------------\n jobID: jobID of the job to mark as completed\n\n to False for hypersearch workers\n \"\"\"\n self._logger.info('Canceling jobID=%s', jobID)\n # NOTE: jobSetFields does retries on transient mysql failures\n self.jobSetFields(jobID, {\"cancel\" : True}, useConnectionID=False)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfetch all the modelIDs that correspond to a given jobID ; empty sequence if none", "response": "def jobGetModelIDs(self, jobID):\n \"\"\"Fetch all the modelIDs that correspond to a given jobID; empty sequence\n if none\"\"\"\n\n rows = self._getMatchingRowsWithRetries(self._models, dict(job_id=jobID),\n ['model_id'])\n return [r[0] for r in rows]"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the number of jobs that are active for the given clientInfo and a status that is not completed.", "response": "def getActiveJobCountForClientInfo(self, clientInfo):\n \"\"\" Return the number of jobs for the given clientInfo and a status that is\n not completed.\n \"\"\"\n with ConnectionFactory.get() as conn:\n query = 'SELECT count(job_id) ' \\\n 'FROM %s ' \\\n 'WHERE client_info = %%s ' \\\n ' AND status != %%s' % self.jobsTableName\n conn.cursor.execute(query, [clientInfo, self.STATUS_COMPLETED])\n activeJobCount = conn.cursor.fetchone()[0]\n\n return activeJobCount"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the number of jobs that are not completed for the given clientKey and a status that is not completed.", "response": "def getActiveJobCountForClientKey(self, clientKey):\n \"\"\" Return the number of jobs for the given clientKey and a status that is\n not completed.\n \"\"\"\n with ConnectionFactory.get() as conn:\n query = 'SELECT count(job_id) ' \\\n 'FROM %s ' \\\n 'WHERE client_key = %%s ' \\\n ' AND status != %%s' % self.jobsTableName\n conn.cursor.execute(query, [clientKey, self.STATUS_COMPLETED])\n activeJobCount = conn.cursor.fetchone()[0]\n\n return activeJobCount"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getActiveJobsForClientInfo(self, clientInfo, fields=[]):\n\n # Form the sequence of field name strings that will go into the\n # request\n dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]\n dbFieldsStr = ','.join(['job_id'] + dbFields)\n\n with ConnectionFactory.get() as conn:\n query = 'SELECT %s FROM %s ' \\\n 'WHERE client_info = %%s ' \\\n ' AND status != %%s' % (dbFieldsStr, self.jobsTableName)\n conn.cursor.execute(query, [clientInfo, self.STATUS_COMPLETED])\n rows = conn.cursor.fetchall()\n\n return rows", "response": "Fetch the jobIDs for jobs that have a specific clientInfo."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the list of fields that are available for the given jobType.", "response": "def getFieldsForActiveJobsOfType(self, jobType, fields=[]):\n \"\"\" Helper function for querying the models table including relevant job\n info where the job type matches the specified jobType. Only records for\n which there is a matching jobId in both tables is returned, and only the\n requested fields are returned in each result, assuming that there is not\n a conflict. This function is useful, for example, in querying a cluster\n for a list of actively running production models (according to the state\n of the client jobs database). jobType must be one of the JOB_TYPE_XXXX\n enumerations.\n\n Parameters:\n ----------------------------------------------------------------\n jobType: jobType enum\n fields: list of fields to return\n\n Returns: List of tuples containing the jobId and requested field values\n \"\"\"\n dbFields = [self._jobs.pubToDBNameDict[x] for x in fields]\n dbFieldsStr = ','.join(['job_id'] + dbFields)\n with ConnectionFactory.get() as conn:\n query = \\\n 'SELECT DISTINCT %s ' \\\n 'FROM %s j ' \\\n 'LEFT JOIN %s m USING(job_id) '\\\n 'WHERE j.status != %%s ' \\\n 'AND _eng_job_type = %%s' % (dbFieldsStr, self.jobsTableName,\n self.modelsTableName)\n\n conn.cursor.execute(query, [self.STATUS_COMPLETED, jobType])\n return conn.cursor.fetchall()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the values of one or more fields from a job record.", "response": "def jobGetFields(self, jobID, fields):\n \"\"\" Fetch the values of 1 or more fields from a job record. Here, 'fields'\n is a list with the names of the fields to fetch. The names are the public\n names of the fields (camelBack, not the lower_case_only form as stored in\n the DB).\n\n Parameters:\n ----------------------------------------------------------------\n jobID: jobID of the job record\n fields: list of fields to return\n\n Returns: A sequence of field values in the same order as the requested\n field list -> [field1, field2, ...]\n \"\"\"\n # NOTE: jobsGetFields retries on transient mysql failures\n return self.jobsGetFields([jobID], fields, requireAll=True)[0][1]"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef jobsGetFields(self, jobIDs, fields, requireAll=True):\n assert isinstance(jobIDs, self._SEQUENCE_TYPES)\n assert len(jobIDs) >=1\n\n rows = self._getMatchingRowsWithRetries(\n self._jobs, dict(job_id=jobIDs),\n ['job_id'] + [self._jobs.pubToDBNameDict[x] for x in fields])\n\n\n if requireAll and len(rows) < len(jobIDs):\n # NOTE: this will also trigger if the jobIDs list included duplicates\n raise RuntimeError(\"jobIDs %s not found within the jobs table\" % (\n (set(jobIDs) - set(r[0] for r in rows)),))\n\n\n return [(r[0], list(r[1:])) for r in rows]", "response": "Fetch the values of 1 or more fields from a sequence of job records."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchanging the values of one or more fields in a job.", "response": "def jobSetFields(self, jobID, fields, useConnectionID=True,\n ignoreUnchanged=False):\n \"\"\" Change the values of 1 or more fields in a job. Here, 'fields' is a\n dict with the name/value pairs to change. The names are the public names of\n the fields (camelBack, not the lower_case_only form as stored in the DB).\n This method is for private use by the ClientJobManager only.\n\n Parameters:\n ----------------------------------------------------------------\n jobID: jobID of the job record\n\n fields: dictionary of fields to change\n\n useConnectionID: True if the connection id of the calling function\n must be the same as the connection that created the job. Set\n to False for hypersearch workers\n\n ignoreUnchanged: The default behavior is to throw a\n RuntimeError if no rows are affected. This could either be\n because:\n 1) Because there was not matching jobID\n 2) or if the data to update matched the data in the DB exactly.\n\n Set this parameter to True if you expect case 2 and wish to\n supress the error.\n \"\"\"\n\n # Form the sequecce of key=value strings that will go into the\n # request\n assignmentExpressions = ','.join(\n [\"%s=%%s\" % (self._jobs.pubToDBNameDict[f],) for f in fields.iterkeys()])\n assignmentValues = fields.values()\n\n query = 'UPDATE %s SET %s ' \\\n ' WHERE job_id=%%s' \\\n % (self.jobsTableName, assignmentExpressions,)\n sqlParams = assignmentValues + [jobID]\n\n if useConnectionID:\n query += ' AND _eng_cjm_conn_id=%s'\n sqlParams.append(self._connectionID)\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n result = conn.cursor.execute(query, sqlParams)\n\n if result != 1 and not ignoreUnchanged:\n raise RuntimeError(\n \"Tried to change fields (%r) of jobID=%s conn_id=%r), but an error \" \\\n \"occurred. result=%r; query=%r\" % (\n assignmentExpressions, jobID, self._connectionID, result, query))"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchanging the value of 1 field in a job to 'newValue', but only if the current value matches 'curValue'. The 'fieldName' is the public name of the field (camelBack, not the lower_case_only form as stored in the DB). This method is used for example by HypersearcWorkers to update the engWorkerState field periodically. By qualifying on curValue, it insures that only 1 worker at a time is elected to perform the next scheduled periodic sweep of the models. Parameters: ---------------------------------------------------------------- jobID: jobID of the job record to modify fieldName: public field name of the field newValue: new value of the field to set curValue: current value to qualify against retval: True if we successfully modified the field False if curValue did not match", "response": "def jobSetFieldIfEqual(self, jobID, fieldName, newValue, curValue):\n \"\"\" Change the value of 1 field in a job to 'newValue', but only if the\n current value matches 'curValue'. The 'fieldName' is the public name of\n the field (camelBack, not the lower_case_only form as stored in the DB).\n\n This method is used for example by HypersearcWorkers to update the\n engWorkerState field periodically. By qualifying on curValue, it insures\n that only 1 worker at a time is elected to perform the next scheduled\n periodic sweep of the models.\n\n Parameters:\n ----------------------------------------------------------------\n jobID: jobID of the job record to modify\n fieldName: public field name of the field\n newValue: new value of the field to set\n curValue: current value to qualify against\n\n retval: True if we successfully modified the field\n False if curValue did not match\n \"\"\"\n\n # Get the private field name and string form of the value\n dbFieldName = self._jobs.pubToDBNameDict[fieldName]\n\n conditionValue = []\n if isinstance(curValue, bool):\n conditionExpression = '%s IS %s' % (\n dbFieldName, {True:'TRUE', False:'FALSE'}[curValue])\n elif curValue is None:\n conditionExpression = '%s is NULL' % (dbFieldName,)\n else:\n conditionExpression = '%s=%%s' % (dbFieldName,)\n conditionValue.append(curValue)\n\n query = 'UPDATE %s SET _eng_last_update_time=UTC_TIMESTAMP(), %s=%%s ' \\\n ' WHERE job_id=%%s AND %s' \\\n % (self.jobsTableName, dbFieldName, conditionExpression)\n sqlParams = [newValue, jobID] + conditionValue\n\n with ConnectionFactory.get() as conn:\n result = conn.cursor.execute(query, sqlParams)\n\n return (result == 1)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef jobIncrementIntField(self, jobID, fieldName, increment=1,\n useConnectionID=False):\n \"\"\" Incremet the value of 1 field in a job by increment. The 'fieldName' is\n the public name of the field (camelBack, not the lower_case_only form as\n stored in the DB).\n\n This method is used for example by HypersearcWorkers to update the\n engWorkerState field periodically. By qualifying on curValue, it insures\n that only 1 worker at a time is elected to perform the next scheduled\n periodic sweep of the models.\n\n Parameters:\n ----------------------------------------------------------------\n jobID: jobID of the job record to modify\n fieldName: public field name of the field\n increment: increment is added to the current value of the field\n \"\"\"\n # Get the private field name and string form of the value\n dbFieldName = self._jobs.pubToDBNameDict[fieldName]\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n query = 'UPDATE %s SET %s=%s+%%s ' \\\n ' WHERE job_id=%%s' \\\n % (self.jobsTableName, dbFieldName, dbFieldName)\n sqlParams = [increment, jobID]\n\n if useConnectionID:\n query += ' AND _eng_cjm_conn_id=%s'\n sqlParams.append(self._connectionID)\n\n result = conn.cursor.execute(query, sqlParams)\n\n if result != 1:\n raise RuntimeError(\n \"Tried to increment the field (%r) of jobID=%s (conn_id=%r), but an \" \\\n \"error occurred. result=%r; query=%r\" % (\n dbFieldName, jobID, self._connectionID, result, query))", "response": "Increment the value of a field in a job record by increment."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef jobUpdateResults(self, jobID, results):\n with ConnectionFactory.get() as conn:\n query = 'UPDATE %s SET _eng_last_update_time=UTC_TIMESTAMP(), ' \\\n ' results=%%s ' \\\n ' WHERE job_id=%%s' % (self.jobsTableName,)\n conn.cursor.execute(query, [results, jobID])", "response": "Update the results string and last - update - time fields of a single job."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef modelsClearAll(self):\n self._logger.info('Deleting all rows from models table %r',\n self.modelsTableName)\n with ConnectionFactory.get() as conn:\n query = 'DELETE FROM %s' % (self.modelsTableName)\n conn.cursor.execute(query)", "response": "Delete all models from the models table"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef modelInsertAndStart(self, jobID, params, paramsHash, particleHash=None):\n # Fill in default particleHash\n if particleHash is None:\n particleHash = paramsHash\n\n # Normalize hashes\n paramsHash = self._normalizeHash(paramsHash)\n particleHash = self._normalizeHash(particleHash)\n\n def findExactMatchNoRetries(conn):\n return self._getOneMatchingRowNoRetries(\n self._models, conn,\n {'job_id':jobID, '_eng_params_hash':paramsHash,\n '_eng_particle_hash':particleHash},\n ['model_id', '_eng_worker_conn_id'])\n\n @g_retrySQL\n def findExactMatchWithRetries():\n with ConnectionFactory.get() as conn:\n return findExactMatchNoRetries(conn)\n\n # Check if the model is already in the models table\n #\n # NOTE: with retries of mysql transient failures, we can't always tell\n # whether the row was already inserted (e.g., comms failure could occur\n # after insertion into table, but before arrival or response), so the\n # need to check before attempting to insert a new row\n #\n # TODO: if we could be assured that the caller already verified the\n # model's absence before calling us, we could skip this check here\n row = findExactMatchWithRetries()\n if row is not None:\n return (row[0], False)\n\n @g_retrySQL\n def insertModelWithRetries():\n \"\"\" NOTE: it's possible that another process on some machine is attempting\n to insert the same model at the same time as the caller \"\"\"\n with ConnectionFactory.get() as conn:\n # Create a new job entry\n query = 'INSERT INTO %s (job_id, params, status, _eng_params_hash, ' \\\n ' _eng_particle_hash, start_time, _eng_last_update_time, ' \\\n ' _eng_worker_conn_id) ' \\\n ' VALUES (%%s, %%s, %%s, %%s, %%s, UTC_TIMESTAMP(), ' \\\n ' UTC_TIMESTAMP(), %%s) ' \\\n % (self.modelsTableName,)\n sqlParams = (jobID, params, self.STATUS_RUNNING, paramsHash,\n particleHash, self._connectionID)\n try:\n numRowsAffected = conn.cursor.execute(query, sqlParams)\n except Exception, e:\n # NOTE: We have seen instances where some package in the calling\n # chain tries to interpret the exception message using unicode.\n # Since the exception message contains binary data (the hashes), this\n # can in turn generate a Unicode translation exception. So, we catch\n # ALL exceptions here and look for the string \"Duplicate entry\" in\n # the exception args just in case this happens. For example, the\n # Unicode exception we might get is:\n # (, UnicodeDecodeError('utf8', \"Duplicate entry '1000-?.\\x18\\xb1\\xd3\\xe0CO\\x05\\x8b\\xf80\\xd7E5\\xbb' for key 'job_id'\", 25, 26, 'invalid start byte'))\n #\n # If it weren't for this possible Unicode translation error, we\n # could watch for only the exceptions we want, like this:\n # except pymysql.IntegrityError, e:\n # if e.args[0] != mysqlerrors.DUP_ENTRY:\n # raise\n if \"Duplicate entry\" not in str(e):\n raise\n\n # NOTE: duplicate entry scenario: however, we can't discern\n # whether it was inserted by another process or this one, because an\n # intermittent failure may have caused us to retry\n self._logger.info('Model insert attempt failed with DUP_ENTRY: '\n 'jobID=%s; paramsHash=%s OR particleHash=%s; %r',\n jobID, paramsHash.encode('hex'),\n particleHash.encode('hex'), e)\n else:\n if numRowsAffected == 1:\n # NOTE: SELECT LAST_INSERT_ID() returns 0 after re-connection\n conn.cursor.execute('SELECT LAST_INSERT_ID()')\n modelID = conn.cursor.fetchall()[0][0]\n if modelID != 0:\n return (modelID, True)\n else:\n self._logger.warn(\n 'SELECT LAST_INSERT_ID for model returned 0, implying loss of '\n 'connection: jobID=%s; paramsHash=%r; particleHash=%r',\n jobID, paramsHash, particleHash)\n else:\n self._logger.error(\n 'Attempt to insert model resulted in unexpected numRowsAffected: '\n 'expected 1, but got %r; jobID=%s; paramsHash=%r; '\n 'particleHash=%r',\n numRowsAffected, jobID, paramsHash, particleHash)\n\n # Look up the model and discern whether it is tagged with our conn id\n row = findExactMatchNoRetries(conn)\n if row is not None:\n (modelID, connectionID) = row\n return (modelID, connectionID == self._connectionID)\n\n # This set of params is already in the table, just get the modelID\n query = 'SELECT (model_id) FROM %s ' \\\n ' WHERE job_id=%%s AND ' \\\n ' (_eng_params_hash=%%s ' \\\n ' OR _eng_particle_hash=%%s) ' \\\n ' LIMIT 1 ' \\\n % (self.modelsTableName,)\n sqlParams = [jobID, paramsHash, particleHash]\n numRowsFound = conn.cursor.execute(query, sqlParams)\n assert numRowsFound == 1, (\n 'Model not found: jobID=%s AND (paramsHash=%r OR particleHash=%r); '\n 'numRowsFound=%r') % (jobID, paramsHash, particleHash, numRowsFound)\n (modelID,) = conn.cursor.fetchall()[0]\n return (modelID, False)\n\n\n return insertModelWithRetries()", "response": "Insert a new model into the model table and return the modelID of the new model."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget ALL info for a set of models", "response": "def modelsInfo(self, modelIDs):\n \"\"\" Get ALL info for a set of models\n\n WARNING!!!: The order of the results are NOT necessarily in the same order as\n the order of the model IDs passed in!!!\n\n Parameters:\n ----------------------------------------------------------------\n modelIDs: list of model IDs\n retval: list of nametuples containing all the fields stored for each\n model.\n \"\"\"\n assert isinstance(modelIDs, self._SEQUENCE_TYPES), (\n \"wrong modelIDs type: %s\") % (type(modelIDs),)\n assert modelIDs, \"modelIDs is empty\"\n\n rows = self._getMatchingRowsWithRetries(\n self._models, dict(model_id=modelIDs),\n [self._models.pubToDBNameDict[f]\n for f in self._models.modelInfoNamedTuple._fields])\n\n results = [self._models.modelInfoNamedTuple._make(r) for r in rows]\n\n # NOTE: assetion will also fail if modelIDs contains duplicates\n assert len(results) == len(modelIDs), \"modelIDs not found: %s\" % (\n set(modelIDs) - set(r.modelId for r in results))\n\n return results"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the values of 1 or more fields from a sequence of models.", "response": "def modelsGetFields(self, modelIDs, fields):\n \"\"\" Fetch the values of 1 or more fields from a sequence of model records.\n Here, 'fields' is a list with the names of the fields to fetch. The names\n are the public names of the fields (camelBack, not the lower_case_only form\n as stored in the DB).\n\n WARNING!!!: The order of the results are NOT necessarily in the same order\n as the order of the model IDs passed in!!!\n\n\n Parameters:\n ----------------------------------------------------------------\n modelIDs: A single modelID or sequence of modelIDs\n fields: A list of fields to return\n\n Returns: If modelIDs is a sequence:\n a list of tuples->(modelID, [field1, field2,...])\n If modelIDs is a single modelID:\n a list of field values->[field1, field2,...]\n \"\"\"\n assert len(fields) >= 1, 'fields is empty'\n\n # Form the sequence of field name strings that will go into the\n # request\n isSequence = isinstance(modelIDs, self._SEQUENCE_TYPES)\n\n if isSequence:\n assert len(modelIDs) >=1, 'modelIDs is empty'\n else:\n modelIDs = [modelIDs]\n\n rows = self._getMatchingRowsWithRetries(\n self._models, dict(model_id=modelIDs),\n ['model_id'] + [self._models.pubToDBNameDict[f] for f in fields])\n\n if len(rows) < len(modelIDs):\n raise RuntimeError(\"modelIDs not found within the models table: %s\" % (\n (set(modelIDs) - set(r[0] for r in rows)),))\n\n if not isSequence:\n return list(rows[0][1:])\n\n return [(r[0], list(r[1:])) for r in rows]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef modelsGetFieldsForJob(self, jobID, fields, ignoreKilled=False):\n\n assert len(fields) >= 1, 'fields is empty'\n\n # Form the sequence of field name strings that will go into the\n # request\n dbFields = [self._models.pubToDBNameDict[x] for x in fields]\n dbFieldsStr = ','.join(dbFields)\n\n query = 'SELECT model_id, %s FROM %s ' \\\n ' WHERE job_id=%%s ' \\\n % (dbFieldsStr, self.modelsTableName)\n sqlParams = [jobID]\n\n if ignoreKilled:\n query += ' AND (completion_reason IS NULL OR completion_reason != %s)'\n sqlParams.append(self.CMPL_REASON_KILLED)\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n conn.cursor.execute(query, sqlParams)\n rows = conn.cursor.fetchall()\n\n if rows is None:\n # fetchall is defined to return a (possibly-empty) sequence of\n # sequences; however, we occasionally see None returned and don't know\n # why...\n self._logger.error(\"Unexpected None result from cursor.fetchall; \"\n \"query=%r; Traceback=%r\",\n query, traceback.format_exc())\n\n return [(r[0], list(r[1:])) for r in rows]", "response": "Returns the specified fields for all the models for a single job."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef modelsGetFieldsForCheckpointed(self, jobID, fields):\n\n assert len(fields) >= 1, \"fields is empty\"\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n dbFields = [self._models.pubToDBNameDict[f] for f in fields]\n dbFieldStr = \", \".join(dbFields)\n\n query = 'SELECT model_id, {fields} from {models}' \\\n ' WHERE job_id=%s AND model_checkpoint_id IS NOT NULL'.format(\n fields=dbFieldStr, models=self.modelsTableName)\n\n conn.cursor.execute(query, [jobID])\n rows = conn.cursor.fetchall()\n\n return [(r[0], list(r[1:])) for r in rows]", "response": "Returns a list of tuples that are used to figure out whether or not a new model should be checkpointed."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nchange the values of one or more fields in a model.", "response": "def modelSetFields(self, modelID, fields, ignoreUnchanged = False):\n \"\"\" Change the values of 1 or more fields in a model. Here, 'fields' is a\n dict with the name/value pairs to change. The names are the public names of\n the fields (camelBack, not the lower_case_only form as stored in the DB).\n\n Parameters:\n ----------------------------------------------------------------\n jobID: jobID of the job record\n\n fields: dictionary of fields to change\n\n ignoreUnchanged: The default behavior is to throw a\n RuntimeError if no rows are affected. This could either be\n because:\n 1) Because there was no matching modelID\n 2) or if the data to update matched the data in the DB exactly.\n\n Set this parameter to True if you expect case 2 and wish to\n supress the error.\n \"\"\"\n\n # Form the sequence of key=value strings that will go into the\n # request\n assignmentExpressions = ','.join(\n '%s=%%s' % (self._models.pubToDBNameDict[f],) for f in fields.iterkeys())\n assignmentValues = fields.values()\n\n query = 'UPDATE %s SET %s, update_counter = update_counter+1 ' \\\n ' WHERE model_id=%%s' \\\n % (self.modelsTableName, assignmentExpressions)\n sqlParams = assignmentValues + [modelID]\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n numAffectedRows = conn.cursor.execute(query, sqlParams)\n self._logger.debug(\"Executed: numAffectedRows=%r, query=%r, sqlParams=%r\",\n numAffectedRows, query, sqlParams)\n\n if numAffectedRows != 1 and not ignoreUnchanged:\n raise RuntimeError(\n (\"Tried to change fields (%r) of model %r (conn_id=%r), but an error \"\n \"occurred. numAffectedRows=%r; query=%r; sqlParams=%r\") % (\n fields, modelID, self._connectionID, numAffectedRows, query,\n sqlParams,))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef modelsGetParams(self, modelIDs):\n assert isinstance(modelIDs, self._SEQUENCE_TYPES), (\n \"Wrong modelIDs type: %r\") % (type(modelIDs),)\n assert len(modelIDs) >= 1, \"modelIDs is empty\"\n\n rows = self._getMatchingRowsWithRetries(\n self._models, {'model_id' : modelIDs},\n [self._models.pubToDBNameDict[f]\n for f in self._models.getParamsNamedTuple._fields])\n\n # NOTE: assertion will also fail when modelIDs contains duplicates\n assert len(rows) == len(modelIDs), \"Didn't find modelIDs: %r\" % (\n (set(modelIDs) - set(r[0] for r in rows)),)\n\n # Return the params and params hashes as a namedtuple\n return [self._models.getParamsNamedTuple._make(r) for r in rows]", "response": "Returns the params and paramsHash for a set of models."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the results string and other status fields for a set of models.", "response": "def modelsGetResultAndStatus(self, modelIDs):\n \"\"\" Get the results string and other status fields for a set of models.\n\n WARNING!!!: The order of the results are NOT necessarily in the same order\n as the order of the model IDs passed in!!!\n\n For each model, this returns a tuple containing:\n (modelID, results, status, updateCounter, numRecords, completionReason,\n completionMsg, engParamsHash\n\n Parameters:\n ----------------------------------------------------------------\n modelIDs: list of model IDs\n retval: list of result tuples. Each tuple contains:\n (modelID, results, status, updateCounter, numRecords,\n completionReason, completionMsg, engParamsHash)\n \"\"\"\n assert isinstance(modelIDs, self._SEQUENCE_TYPES), (\n \"Wrong modelIDs type: %r\") % type(modelIDs)\n assert len(modelIDs) >= 1, \"modelIDs is empty\"\n\n rows = self._getMatchingRowsWithRetries(\n self._models, {'model_id' : modelIDs},\n [self._models.pubToDBNameDict[f]\n for f in self._models.getResultAndStatusNamedTuple._fields])\n\n # NOTE: assertion will also fail when modelIDs contains duplicates\n assert len(rows) == len(modelIDs), \"Didn't find modelIDs: %r\" % (\n (set(modelIDs) - set(r[0] for r in rows)),)\n\n # Return the results as a list of namedtuples\n return [self._models.getResultAndStatusNamedTuple._make(r) for r in rows]"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef modelsGetUpdateCounters(self, jobID):\n rows = self._getMatchingRowsWithRetries(\n self._models, {'job_id' : jobID},\n [self._models.pubToDBNameDict[f]\n for f in self._models.getUpdateCountersNamedTuple._fields])\n\n # Return the results as a list of namedtuples\n return [self._models.getUpdateCountersNamedTuple._make(r) for r in rows]", "response": "Returns info on all of the models that are in already in the models\n table for a given job."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nupdating the results string and numRecords fields of a model.", "response": "def modelUpdateResults(self, modelID, results=None, metricValue =None,\n numRecords=None):\n \"\"\" Update the results string, and/or num_records fields of\n a model. This will fail if the model does not currently belong to this\n client (connection_id doesn't match).\n\n Parameters:\n ----------------------------------------------------------------\n modelID: model ID of model to modify\n results: new results, or None to ignore\n metricValue: the value of the metric being optimized, or None to ignore\n numRecords: new numRecords, or None to ignore\n \"\"\"\n\n assignmentExpressions = ['_eng_last_update_time=UTC_TIMESTAMP()',\n 'update_counter=update_counter+1']\n assignmentValues = []\n\n if results is not None:\n assignmentExpressions.append('results=%s')\n assignmentValues.append(results)\n\n if numRecords is not None:\n assignmentExpressions.append('num_records=%s')\n assignmentValues.append(numRecords)\n\n # NOTE1: (metricValue==metricValue) tests for Nan\n # NOTE2: metricValue is being passed as numpy.float64\n if metricValue is not None and (metricValue==metricValue):\n assignmentExpressions.append('optimized_metric=%s')\n assignmentValues.append(float(metricValue))\n\n query = 'UPDATE %s SET %s ' \\\n ' WHERE model_id=%%s and _eng_worker_conn_id=%%s' \\\n % (self.modelsTableName, ','.join(assignmentExpressions))\n sqlParams = assignmentValues + [modelID, self._connectionID]\n\n # Get a database connection and cursor\n with ConnectionFactory.get() as conn:\n numRowsAffected = conn.cursor.execute(query, sqlParams)\n\n if numRowsAffected != 1:\n raise InvalidConnectionException(\n (\"Tried to update the info of modelID=%r using connectionID=%r, but \"\n \"this model belongs to some other worker or modelID not found; \"\n \"numRowsAffected=%r\") % (modelID,self._connectionID, numRowsAffected,))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef modelSetCompleted(self, modelID, completionReason, completionMsg,\n cpuTime=0, useConnectionID=True):\n \"\"\" Mark a model as completed, with the given completionReason and\n completionMsg. This will fail if the model does not currently belong to this\n client (connection_id doesn't match).\n\n Parameters:\n ----------------------------------------------------------------\n modelID: model ID of model to modify\n completionReason: completionReason string\n completionMsg: completionMsg string\n cpuTime: amount of CPU time spent on this model\n useConnectionID: True if the connection id of the calling function\n must be the same as the connection that created the\n job. Set to True for hypersearch workers, which use\n this mechanism for orphaned model detection.\n \"\"\"\n if completionMsg is None:\n completionMsg = ''\n\n query = 'UPDATE %s SET status=%%s, ' \\\n ' completion_reason=%%s, ' \\\n ' completion_msg=%%s, ' \\\n ' end_time=UTC_TIMESTAMP(), ' \\\n ' cpu_time=%%s, ' \\\n ' _eng_last_update_time=UTC_TIMESTAMP(), ' \\\n ' update_counter=update_counter+1 ' \\\n ' WHERE model_id=%%s' \\\n % (self.modelsTableName,)\n sqlParams = [self.STATUS_COMPLETED, completionReason, completionMsg,\n cpuTime, modelID]\n\n if useConnectionID:\n query += \" AND _eng_worker_conn_id=%s\"\n sqlParams.append(self._connectionID)\n\n with ConnectionFactory.get() as conn:\n numRowsAffected = conn.cursor.execute(query, sqlParams)\n\n if numRowsAffected != 1:\n raise InvalidConnectionException(\n (\"Tried to set modelID=%r using connectionID=%r, but this model \"\n \"belongs to some other worker or modelID not found; \"\n \"numRowsAffected=%r\") % (modelID, self._connectionID, numRowsAffected))", "response": "Mark a model as completed."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadopt next orphaned model in the models table.", "response": "def modelAdoptNextOrphan(self, jobId, maxUpdateInterval):\n \"\"\" Look through the models table for an orphaned model, which is a model\n that is not completed yet, whose _eng_last_update_time is more than\n maxUpdateInterval seconds ago.\n\n If one is found, change its _eng_worker_conn_id to the current worker's\n and return the model id.\n\n Parameters:\n ----------------------------------------------------------------\n retval: modelId of the model we adopted, or None if none found\n \"\"\"\n\n @g_retrySQL\n def findCandidateModelWithRetries():\n modelID = None\n with ConnectionFactory.get() as conn:\n # TODO: may need a table index on job_id/status for speed\n query = 'SELECT model_id FROM %s ' \\\n ' WHERE status=%%s ' \\\n ' AND job_id=%%s ' \\\n ' AND TIMESTAMPDIFF(SECOND, ' \\\n ' _eng_last_update_time, ' \\\n ' UTC_TIMESTAMP()) > %%s ' \\\n ' LIMIT 1 ' \\\n % (self.modelsTableName,)\n sqlParams = [self.STATUS_RUNNING, jobId, maxUpdateInterval]\n numRows = conn.cursor.execute(query, sqlParams)\n rows = conn.cursor.fetchall()\n\n assert numRows <= 1, \"Unexpected numRows: %r\" % numRows\n if numRows == 1:\n (modelID,) = rows[0]\n\n return modelID\n\n @g_retrySQL\n def adoptModelWithRetries(modelID):\n adopted = False\n with ConnectionFactory.get() as conn:\n query = 'UPDATE %s SET _eng_worker_conn_id=%%s, ' \\\n ' _eng_last_update_time=UTC_TIMESTAMP() ' \\\n ' WHERE model_id=%%s ' \\\n ' AND status=%%s' \\\n ' AND TIMESTAMPDIFF(SECOND, ' \\\n ' _eng_last_update_time, ' \\\n ' UTC_TIMESTAMP()) > %%s ' \\\n ' LIMIT 1 ' \\\n % (self.modelsTableName,)\n sqlParams = [self._connectionID, modelID, self.STATUS_RUNNING,\n maxUpdateInterval]\n numRowsAffected = conn.cursor.execute(query, sqlParams)\n\n assert numRowsAffected <= 1, 'Unexpected numRowsAffected=%r' % (\n numRowsAffected,)\n\n if numRowsAffected == 1:\n adopted = True\n else:\n # Discern between transient failure during update and someone else\n # claiming this model\n (status, connectionID) = self._getOneMatchingRowNoRetries(\n self._models, conn, {'model_id':modelID},\n ['status', '_eng_worker_conn_id'])\n adopted = (status == self.STATUS_RUNNING and\n connectionID == self._connectionID)\n return adopted\n\n\n adoptedModelID = None\n while True:\n modelID = findCandidateModelWithRetries()\n if modelID is None:\n break\n if adoptModelWithRetries(modelID):\n adoptedModelID = modelID\n break\n\n return adoptedModelID"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprofiles the performance of a SpatialPooler", "response": "def profileSP(spClass, spDim, nRuns):\n \"\"\"\n profiling performance of SpatialPooler (SP)\n using the python cProfile module and ordered by cumulative time,\n see how to run on command-line above.\n\n @param spClass implementation of SP (cpp, py, ..)\n @param spDim number of columns in SP (in 1D, for 2D see colDim in code)\n @param nRuns number of calls of the profiled code (epochs)\n \"\"\"\n # you can change dimensionality here, eg to 2D\n inDim = [10000, 1, 1]\n colDim = [spDim, 1, 1]\n\n\n # create SP instance to measure\n # changing the params here affects the performance\n sp = spClass(\n inputDimensions=inDim,\n columnDimensions=colDim,\n potentialRadius=3,\n potentialPct=0.5,\n globalInhibition=False,\n localAreaDensity=-1.0,\n numActiveColumnsPerInhArea=3,\n stimulusThreshold=1,\n synPermInactiveDec=0.01,\n synPermActiveInc=0.1,\n synPermConnected=0.10,\n minPctOverlapDutyCycle=0.1,\n dutyCyclePeriod=10,\n boostStrength=10.0,\n seed=42,\n spVerbosity=0)\n\n\n # generate input data\n dataDim = inDim\n dataDim.append(nRuns)\n data = numpy.random.randint(0, 2, dataDim).astype('float32')\n\n for i in xrange(nRuns):\n # new data every time, this is the worst case performance\n # real performance would be better, as the input data would not be completely random\n d = data[:,:,:,i]\n activeArray = numpy.zeros(colDim)\n\n # the actual function to profile!\n sp.compute(d, True, activeArray)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getSpec(cls):\n ns = dict(\n description=KNNClassifierRegion.__doc__,\n singleNodeOnly=True,\n inputs=dict(\n categoryIn=dict(\n description='Vector of zero or more category indices for this input'\n 'sample. -1 implies no category.',\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n bottomUpIn=dict(\n description='Belief values over children\\'s groups',\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=False,\n isDefaultInput=True,\n requireSplitterMap=False),\n\n partitionIn=dict(\n description='Partition ID of the input sample',\n dataType='Real32',\n count=0,\n required=True,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False),\n\n auxDataIn=dict(\n description='Auxiliary data from the sensor',\n dataType='Real32',\n count=0,\n required=False,\n regionLevel=True,\n isDefaultInput=False,\n requireSplitterMap=False)\n ),\n\n\n outputs=dict(\n categoriesOut=dict(\n description='A vector representing, for each category '\n 'index, the likelihood that the input to the node belongs '\n 'to that category based on the number of neighbors of '\n 'that category that are among the nearest K.',\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=True),\n\n bestPrototypeIndices=dict(\n description='A vector that lists, in descending order of '\n 'the match, the positions of the prototypes '\n 'that best match the input pattern.',\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=False),\n\n categoryProbabilitiesOut=dict(\n description='A vector representing, for each category '\n 'index, the probability that the input to the node belongs '\n 'to that category based on the distance to the nearest '\n 'neighbor of each category.',\n dataType='Real32',\n count=0,\n regionLevel=True,\n isDefaultOutput=True),\n\n ),\n\n parameters=dict(\n learningMode=dict(\n description='Boolean (0/1) indicating whether or not a region '\n 'is in learning mode.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=1,\n accessMode='ReadWrite'),\n\n inferenceMode=dict(\n description='Boolean (0/1) indicating whether or not a region '\n 'is in inference mode.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='ReadWrite'),\n\n acceptanceProbability=dict(\n description='During learning, inputs are learned with '\n 'probability equal to this parameter. '\n 'If set to 1.0, the default, '\n 'all inputs will be considered '\n '(subject to other tests).',\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=1.0,\n #accessMode='Create'),\n accessMode='ReadWrite'), # and Create too\n\n confusion=dict(\n description='Confusion matrix accumulated during inference. '\n 'Reset with reset(). This is available to Python '\n 'client code only.',\n dataType='Handle',\n count=2,\n constraints='',\n defaultValue=None,\n accessMode='Read'),\n\n activeOutputCount=dict(\n description='The number of active elements in the '\n '\"categoriesOut\" output.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Read'),\n\n categoryCount=dict(\n description='An integer indicating the number of '\n 'categories that have been learned',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=None,\n accessMode='Read'),\n\n patternCount=dict(\n description='Number of patterns learned by the classifier.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=None,\n accessMode='Read'),\n\n patternMatrix=dict(\n description='The actual patterns learned by the classifier, '\n 'returned as a matrix.',\n dataType='Handle',\n count=1,\n constraints='',\n defaultValue=None,\n accessMode='Read'),\n\n k=dict(\n description='The number of nearest neighbors to use '\n 'during inference.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=1,\n accessMode='Create'),\n\n maxCategoryCount=dict(\n description='The maximal number of categories the '\n 'classifier will distinguish between.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=2,\n accessMode='Create'),\n\n distanceNorm=dict(\n description='The norm to use for a distance metric (i.e., '\n 'the \"p\" in Lp-norm)',\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=2.0,\n accessMode='ReadWrite'),\n #accessMode='Create'),\n\n distanceMethod=dict(\n description='Method used to compute distances between inputs and'\n 'prototypes. Possible options are norm, rawOverlap, '\n 'pctOverlapOfLarger, and pctOverlapOfProto',\n dataType=\"Byte\",\n count=0,\n constraints='enum: norm, rawOverlap, pctOverlapOfLarger, '\n 'pctOverlapOfProto, pctOverlapOfInput',\n defaultValue='norm',\n accessMode='ReadWrite'),\n\n outputProbabilitiesByDist=dict(\n description='If True, categoryProbabilitiesOut is the probability of '\n 'each category based on the distance to the nearest neighbor of '\n 'each category. If False, categoryProbabilitiesOut is the '\n 'percentage of neighbors among the top K that are of each category.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='Create'),\n\n distThreshold=dict(\n description='Distance Threshold. If a pattern that '\n 'is less than distThreshold apart from '\n 'the input pattern already exists in the '\n 'KNN memory, then the input pattern is '\n 'not added to KNN memory.',\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=0.0,\n accessMode='ReadWrite'),\n\n inputThresh=dict(\n description='Input binarization threshold, used if '\n '\"doBinarization\" is True.',\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=0.5,\n accessMode='Create'),\n\n doBinarization=dict(\n description='Whether or not to binarize the input vectors.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='Create'),\n\n useSparseMemory=dict(\n description='A boolean flag that determines whether or '\n 'not the KNNClassifier will use sparse Memory',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=1,\n accessMode='Create'),\n\n minSparsity=dict(\n description=\"If useSparseMemory is set, only vectors with sparsity\"\n \" >= minSparsity will be stored during learning. A value\"\n \" of 0.0 implies all vectors will be stored. A value of\"\n \" 0.1 implies only vectors with at least 10% sparsity\"\n \" will be stored\",\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=0.0,\n accessMode='ReadWrite'),\n\n sparseThreshold=dict(\n description='If sparse memory is used, input variables '\n 'whose absolute value is less than this '\n 'threshold will be stored as zero',\n dataType='Real32',\n count=1,\n constraints='',\n defaultValue=0.0,\n accessMode='Create'),\n\n relativeThreshold=dict(\n description='Whether to multiply sparseThreshold by max value '\n ' in input',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='Create'),\n\n winnerCount=dict(\n description='Only this many elements of the input are '\n 'stored. All elements are stored if 0.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Create'),\n\n doSphering=dict(\n description='A boolean indicating whether or not data should'\n 'be \"sphered\" (i.e. each dimension should be normalized such'\n 'that its mean and variance are zero and one, respectively.) This'\n ' sphering normalization would be performed after all training '\n 'samples had been received but before inference was performed. '\n 'The dimension-specific normalization constants would then '\n ' be applied to all future incoming vectors prior to performing '\n ' conventional NN inference.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='Create'),\n\n SVDSampleCount=dict(\n description='If not 0, carries out SVD transformation after '\n 'that many samples have been seen.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Create'),\n\n SVDDimCount=dict(\n description='Number of dimensions to keep after SVD if greater '\n 'than 0. If set to -1 it is considered unspecified. '\n 'If set to 0 it is consider \"adaptive\" and the number '\n 'is chosen automatically.',\n dataType='Int32',\n count=1,\n constraints='',\n defaultValue=-1,\n accessMode='Create'),\n\n fractionOfMax=dict(\n description='The smallest singular value which is retained '\n 'as a fraction of the largest singular value. This is '\n 'used only if SVDDimCount==0 (\"adaptive\").',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Create'),\n\n useAuxiliary=dict(\n description='Whether or not the classifier should use auxiliary '\n 'input data.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='Create'),\n\n justUseAuxiliary=dict(\n description='Whether or not the classifier should ONLUY use the '\n 'auxiliary input data.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=0,\n accessMode='Create'),\n\n verbosity=dict(\n description='An integer that controls the verbosity level, '\n '0 means no verbose output, increasing integers '\n 'provide more verbosity.',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0 ,\n accessMode='ReadWrite'),\n\n keepAllDistances=dict(\n description='Whether to store all the protoScores in an array, '\n 'rather than just the ones for the last inference. '\n 'When this parameter is changed from True to False, '\n 'all the scores are discarded except for the most '\n 'recent one.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=None,\n accessMode='ReadWrite'),\n\n replaceDuplicates=dict(\n description='A boolean flag that determines whether or'\n 'not the KNNClassifier should replace duplicates'\n 'during learning. This should be on when online'\n 'learning.',\n dataType='UInt32',\n count=1,\n constraints='bool',\n defaultValue=None,\n accessMode='ReadWrite'),\n\n cellsPerCol=dict(\n description='If >= 1, we assume the input is organized into columns, '\n 'in the same manner as the temporal memory AND '\n 'whenever we store a new prototype, we only store the '\n 'start cell (first cell) in any column which is bursting.'\n 'colum ',\n dataType='UInt32',\n count=1,\n constraints='',\n defaultValue=0,\n accessMode='Create'),\n\n maxStoredPatterns=dict(\n description='Limits the maximum number of the training patterns '\n 'stored. When KNN learns in a fixed capacity mode, '\n 'the unused patterns are deleted once the number '\n 'of stored patterns is greater than maxStoredPatterns'\n 'columns. [-1 is no limit] ',\n dataType='Int32',\n count=1,\n constraints='',\n defaultValue=-1,\n accessMode='Create'),\n ),\n commands=dict()\n )\n\n return ns", "response": "Returns a dictionary of all the possible KNN classifiers for this region."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing the ephemerals of the current object.", "response": "def _initEphemerals(self):\n \"\"\"\n Initialize attributes that are not saved with the checkpoint.\n \"\"\"\n self._firstComputeCall = True\n self._accuracy = None\n self._protoScores = None\n self._categoryDistances = None\n\n self._knn = knn_classifier.KNNClassifier(**self.knnParams)\n\n for x in ('_partitions', '_useAuxiliary', '_doSphering',\n '_scanInfo', '_protoScores'):\n if not hasattr(self, x):\n setattr(self, x, None)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getParameter(self, name, index=-1):\n if name == \"patternCount\":\n return self._knn._numPatterns\n elif name == \"patternMatrix\":\n return self._getPatternMatrix()\n elif name == \"k\":\n return self._knn.k\n elif name == \"distanceNorm\":\n return self._knn.distanceNorm\n elif name == \"distanceMethod\":\n return self._knn.distanceMethod\n elif name == \"distThreshold\":\n return self._knn.distThreshold\n elif name == \"inputThresh\":\n return self._knn.binarizationThreshold\n elif name == \"doBinarization\":\n return self._knn.doBinarization\n elif name == \"useSparseMemory\":\n return self._knn.useSparseMemory\n elif name == \"sparseThreshold\":\n return self._knn.sparseThreshold\n elif name == \"winnerCount\":\n return self._knn.numWinners\n elif name == \"relativeThreshold\":\n return self._knn.relativeThreshold\n elif name == \"SVDSampleCount\":\n v = self._knn.numSVDSamples\n return v if v is not None else 0\n elif name == \"SVDDimCount\":\n v = self._knn.numSVDDims\n return v if v is not None else 0\n elif name == \"fractionOfMax\":\n v = self._knn.fractionOfMax\n return v if v is not None else 0\n elif name == \"useAuxiliary\":\n return self._useAuxiliary\n elif name == \"justUseAuxiliary\":\n return self._justUseAuxiliary\n elif name == \"doSphering\":\n return self._doSphering\n elif name == \"cellsPerCol\":\n return self._knn.cellsPerCol\n elif name == \"maxStoredPatterns\":\n return self.maxStoredPatterns\n elif name == 'categoryRecencyList':\n return self._knn._categoryRecencyList\n else:\n # If any spec parameter name is the same as an attribute, this call\n # will get it automatically, e.g. self.learningMode\n return PyRegion.getParameter(self, name, index)", "response": "Returns the value of the parameter with the given name."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nenable the tap file for the specified path.", "response": "def enableTap(self, tapPath):\n \"\"\"\n Begin writing output tap files.\n\n :param tapPath: (string) base name of the output tap files to write.\n \"\"\"\n\n self._tapFileIn = open(tapPath + '.in', 'w')\n self._tapFileOut = open(tapPath + '.out', 'w')"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndisable writing of input tap files.", "response": "def disableTap(self):\n \"\"\"\n Disable writing of output tap files.\n \"\"\"\n\n if self._tapFileIn is not None:\n self._tapFileIn.close()\n self._tapFileIn = None\n if self._tapFileOut is not None:\n self._tapFileOut.close()\n self._tapFileOut = None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef handleLogInput(self, inputs):\n\n if self._tapFileIn is not None:\n for input in inputs:\n for k in range(len(input)):\n print >> self._tapFileIn, input[k],\n print >> self._tapFileIn", "response": "Write input to output tap file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting outputs to output tap file.", "response": "def handleLogOutput(self, output):\n \"\"\"\n Write outputs to output tap file.\n\n :param outputs: (iter) some outputs.\n \"\"\"\n #raise Exception('MULTI-LINE DUMMY\\nMULTI-LINE DUMMY')\n if self._tapFileOut is not None:\n for k in range(len(output)):\n print >> self._tapFileOut, output[k],\n print >> self._tapFileOut"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nstore a training sample vector and associated category label.", "response": "def _storeSample(self, inputVector, trueCatIndex, partition=0):\n \"\"\"\n Store a training sample and associated category label\n \"\"\"\n\n # If this is the first sample, then allocate a numpy array\n # of the appropriate size in which to store all samples.\n if self._samples is None:\n self._samples = numpy.zeros((0, len(inputVector)), dtype=RealNumpyDType)\n assert self._labels is None\n self._labels = []\n\n # Add the sample vector and category lable\n self._samples = numpy.concatenate((self._samples, numpy.atleast_2d(inputVector)), axis=0)\n self._labels += [trueCatIndex]\n\n # Add the partition ID\n if self._partitions is None:\n self._partitions = []\n if partition is None:\n partition = 0\n self._partitions += [partition]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute(self, inputs, outputs):\n\n #raise Exception('MULTI-LINE DUMMY\\nMULTI-LINE DUMMY')\n #For backward compatibility\n if self._useAuxiliary is None:\n self._useAuxiliary = False\n\n # If the first time being called, then print potential warning messsages\n if self._firstComputeCall:\n self._firstComputeCall = False\n if self._useAuxiliary:\n #print \"\\n Auxiliary input stream from Image Sensor enabled.\"\n if self._justUseAuxiliary == True:\n print \" Warning: You have chosen to ignore the image data and instead just use the auxiliary data stream.\"\n\n\n # Format inputs\n #childInputs = [x.wvector(0) for x in inputs[\"bottomUpIn\"]]\n #inputVector = numpy.concatenate([x.array() for x in childInputs])\n inputVector = inputs['bottomUpIn']\n\n # Look for auxiliary input\n if self._useAuxiliary==True:\n #auxVector = inputs['auxDataIn'][0].wvector(0).array()\n auxVector = inputs['auxDataIn']\n if auxVector.dtype != numpy.float32:\n raise RuntimeError, \"KNNClassifierRegion expects numpy.float32 for the auxiliary data vector\"\n if self._justUseAuxiliary == True:\n #inputVector = inputs['auxDataIn'][0].wvector(0).array()\n inputVector = inputs['auxDataIn']\n else:\n #inputVector = numpy.concatenate([inputVector, inputs['auxDataIn'][0].wvector(0).array()])\n inputVector = numpy.concatenate([inputVector, inputs['auxDataIn']])\n\n # Logging\n #self.handleLogInput(childInputs)\n self.handleLogInput([inputVector])\n\n # Read the category.\n assert \"categoryIn\" in inputs, \"No linked category input.\"\n categories = inputs['categoryIn']\n\n # Read the partition ID.\n if \"partitionIn\" in inputs:\n assert len(inputs[\"partitionIn\"]) == 1, \"Must have exactly one link to partition input.\"\n partInput = inputs['partitionIn']\n assert len(partInput) == 1, \"Partition input element count must be exactly 1.\"\n partition = int(partInput[0])\n else:\n partition = None\n\n\n # ---------------------------------------------------------------------\n # Inference (can be done simultaneously with learning)\n if self.inferenceMode:\n categoriesOut = outputs['categoriesOut']\n probabilitiesOut = outputs['categoryProbabilitiesOut']\n\n # If we are sphering, then apply normalization\n if self._doSphering:\n inputVector = (inputVector + self._normOffset) * self._normScale\n\n nPrototypes = 0\n if \"bestPrototypeIndices\" in outputs:\n #bestPrototypeIndicesOut = outputs[\"bestPrototypeIndices\"].wvector()\n bestPrototypeIndicesOut = outputs[\"bestPrototypeIndices\"]\n nPrototypes = len(bestPrototypeIndicesOut)\n\n winner, inference, protoScores, categoryDistances = \\\n self._knn.infer(inputVector, partitionId=partition)\n\n\n\n if not self.keepAllDistances:\n self._protoScores = protoScores\n else:\n # Keep all prototype scores in an array\n if self._protoScores is None:\n self._protoScores = numpy.zeros((1, protoScores.shape[0]),\n protoScores.dtype)\n self._protoScores[0,:] = protoScores#.reshape(1, protoScores.shape[0])\n self._protoScoreCount = 1\n else:\n if self._protoScoreCount == self._protoScores.shape[0]:\n # Double the size of the array\n newProtoScores = numpy.zeros((self._protoScores.shape[0] * 2,\n self._protoScores.shape[1]),\n self._protoScores.dtype)\n newProtoScores[:self._protoScores.shape[0],:] = self._protoScores\n self._protoScores = newProtoScores\n # Store the new prototype score\n self._protoScores[self._protoScoreCount,:] = protoScores\n self._protoScoreCount += 1\n self._categoryDistances = categoryDistances\n\n\n # --------------------------------------------------------------------\n # Compute the probability of each category\n if self.outputProbabilitiesByDist:\n scores = 1.0 - self._categoryDistances\n else:\n scores = inference\n\n # Probability is simply the scores/scores.sum()\n total = scores.sum()\n if total == 0:\n numScores = len(scores)\n probabilities = numpy.ones(numScores) / numScores\n else:\n probabilities = scores / total\n\n\n # -------------------------------------------------------------------\n # Fill the output vectors with our results\n nout = min(len(categoriesOut), len(inference))\n categoriesOut.fill(0)\n categoriesOut[0:nout] = inference[0:nout]\n\n probabilitiesOut.fill(0)\n probabilitiesOut[0:nout] = probabilities[0:nout]\n\n if self.verbosity >= 1:\n print \"KNNRegion: categoriesOut: \", categoriesOut[0:nout]\n print \"KNNRegion: probabilitiesOut: \", probabilitiesOut[0:nout]\n\n if self._scanInfo is not None:\n self._scanResults = [tuple(inference[:nout])]\n\n # Update the stored confusion matrix.\n for category in categories:\n if category >= 0:\n dims = max(int(category)+1, len(inference))\n oldDims = len(self.confusion)\n if oldDims < dims:\n confusion = numpy.zeros((dims, dims))\n confusion[0:oldDims, 0:oldDims] = self.confusion\n self.confusion = confusion\n self.confusion[inference.argmax(), int(category)] += 1\n\n # Calculate the best prototype indices\n if nPrototypes > 1:\n bestPrototypeIndicesOut.fill(0)\n if categoryDistances is not None:\n indices = categoryDistances.argsort()\n nout = min(len(indices), nPrototypes)\n bestPrototypeIndicesOut[0:nout] = indices[0:nout]\n elif nPrototypes == 1:\n if (categoryDistances is not None) and len(categoryDistances):\n bestPrototypeIndicesOut[0] = categoryDistances.argmin()\n else:\n bestPrototypeIndicesOut[0] = 0\n\n # Logging\n self.handleLogOutput(inference)\n\n # ---------------------------------------------------------------------\n # Learning mode\n if self.learningMode:\n if (self.acceptanceProbability < 1.0) and \\\n (self._rgen.getReal64() > self.acceptanceProbability):\n pass\n\n else:\n # Accept the input\n for category in categories:\n if category >= 0:\n # category values of -1 are to be skipped (they are non-categories)\n if self._doSphering:\n # If we are sphering, then we can't provide the data to the KNN\n # library until we have computed per-dimension normalization\n # constants. So instead, we'll just store each training sample.\n self._storeSample(inputVector, category, partition)\n else:\n # Pass the raw training sample directly to the KNN library.\n self._knn.learn(inputVector, category, partition)\n\n self._epoch += 1", "response": "This method computes the KNN classifier for one input sample."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _finishLearning(self):\n if self._doSphering:\n self._finishSphering()\n\n self._knn.finishLearning()\n\n # Compute leave-one-out validation accuracy if\n # we actually received non-trivial partition info\n self._accuracy = None", "response": "Finishes learning and returns None."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _finishSphering(self):\n\n # If we are sphering our data, we need to compute the\n # per-dimension normalization constants\n # First normalize the means (to zero)\n self._normOffset = self._samples.mean(axis=0) * -1.0\n self._samples += self._normOffset\n # Now normalize the variances (to one). However, we need to be\n # careful because the variance could conceivably be zero for one\n # or more dimensions.\n variance = self._samples.var(axis=0)\n variance[numpy.where(variance == 0.0)] = 1.0\n self._normScale = 1.0 / numpy.sqrt(variance)\n self._samples *= self._normScale\n\n # Now feed each \"sphered\" sample into the SVM library\n for sampleIndex in range(len(self._labels)):\n self._knn.learn(self._samples[sampleIndex],\n self._labels[sampleIndex],\n self._partitions[sampleIndex])", "response": "Finish sphering the SVM model."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getOutputElementCount(self, name):\n if name == 'categoriesOut':\n return self.maxCategoryCount\n elif name == 'categoryProbabilitiesOut':\n return self.maxCategoryCount\n elif name == 'bestPrototypeIndices':\n return self._bestPrototypeIndexCount if self._bestPrototypeIndexCount else 0\n else:\n raise Exception('Unknown output: ' + name)", "response": "Returns the number of elements in the output sequence."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef generateStats(filename, statsInfo, maxSamples = None, filters=[], cache=True):\n\n # Sanity checking\n if not isinstance(statsInfo, dict):\n raise RuntimeError(\"statsInfo must be a dict -- \"\n \"found '%s' instead\" % type(statsInfo))\n\n filename = resource_filename(\"nupic.datafiles\", filename)\n\n if cache:\n statsFilename = getStatsFilename(filename, statsInfo, filters)\n # Use cached stats if found AND if it has the right data\n if os.path.exists(statsFilename):\n try:\n r = pickle.load(open(statsFilename, \"rb\"))\n except:\n # Ok to ignore errors -- we will just re-generate the file\n print \"Warning: unable to load stats for %s -- \" \\\n \"will regenerate\" % filename\n r = dict()\n requestedKeys = set([s for s in statsInfo])\n availableKeys = set(r.keys())\n unavailableKeys = requestedKeys.difference(availableKeys)\n if len(unavailableKeys ) == 0:\n return r\n else:\n print \"generateStats: re-generating stats file %s because \" \\\n \"keys %s are not available\" % \\\n (filename, str(unavailableKeys))\n os.remove(filename)\n\n print \"Generating statistics for file '%s' with filters '%s'\" % (filename, filters)\n sensor = RecordSensor()\n sensor.dataSource = FileRecordStream(filename)\n sensor.preEncodingFilters = filters\n\n # Convert collector description to collector object\n stats = []\n for field in statsInfo:\n # field = key from statsInfo\n if statsInfo[field] == \"number\":\n # This wants a field name e.g. consumption and the field type as the value\n statsInfo[field] = NumberStatsCollector()\n elif statsInfo[field] == \"category\":\n statsInfo[field] = CategoryStatsCollector()\n else:\n raise RuntimeError(\"Unknown stats type '%s' for field '%s'\" % (statsInfo[field], field))\n\n # Now collect the stats\n if maxSamples is None:\n maxSamples = 500000\n for i in xrange(maxSamples):\n try:\n record = sensor.getNextRecord()\n except StopIteration:\n break\n for (name, collector) in statsInfo.items():\n collector.add(record[name])\n\n del sensor\n\n # Assemble the results and return\n r = dict()\n for (field, collector) in statsInfo.items():\n stats = collector.getStats()\n if field not in r:\n r[field] = stats\n else:\n r[field].update(stats)\n\n if cache:\n f = open(statsFilename, \"wb\")\n pickle.dump(r, f)\n f.close()\n # caller may need to know name of cached file\n r[\"_filename\"] = statsFilename\n\n return r", "response": "Generate requested statistics for a dataset."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getScalarMetricWithTimeOfDayAnomalyParams(metricData,\n minVal=None,\n maxVal=None,\n minResolution=None,\n tmImplementation = \"cpp\"):\n \"\"\"\n Return a dict that can be used to create an anomaly model via\n :meth:`nupic.frameworks.opf.model_factory.ModelFactory.create`.\n\n Example:\n\n .. code-block:: python\n\n from nupic.frameworks.opf.model_factory import ModelFactory\n from nupic.frameworks.opf.common_models.cluster_params import (\n getScalarMetricWithTimeOfDayAnomalyParams)\n\n params = getScalarMetricWithTimeOfDayAnomalyParams(\n metricData=[0],\n tmImplementation=\"cpp\",\n minVal=0.0,\n maxVal=100.0)\n\n model = ModelFactory.create(modelConfig=params[\"modelConfig\"])\n model.enableLearning()\n model.enableInference(params[\"inferenceArgs\"])\n\n\n :param metricData: numpy array of metric data. Used to calculate ``minVal``\n and ``maxVal`` if either is unspecified\n\n :param minVal: minimum value of metric. Used to set up encoders. If ``None``\n will be derived from ``metricData``.\n\n :param maxVal: maximum value of metric. Used to set up input encoders. If\n ``None`` will be derived from ``metricData``\n\n :param minResolution: minimum resolution of metric. Used to set up\n encoders. If ``None``, will use default value of ``0.001``.\n\n :param tmImplementation: (string) specifying type of temporal memory\n implementation. Valid strings : ``[\"cpp\", \"tm_cpp\"]``\n\n :returns: (dict) containing ``modelConfig`` and ``inferenceArgs`` top-level\n properties. The value of the ``modelConfig`` property is for passing to\n :meth:`~nupic.frameworks.opf.model_factory.ModelFactory.create` method as\n the ``modelConfig`` parameter. The ``inferenceArgs`` property is for passing\n to the resulting model's\n :meth:`~nupic.frameworks.opf.model.Model.enableInference` method as the\n ``inferenceArgs`` parameter.\n\n .. note:: The timestamp field corresponds to input ``c0``; the predicted\n field corresponds to input ``c1``.\n\n \"\"\"\n # Default values\n if minResolution is None:\n minResolution = 0.001\n\n # Compute min and/or max from the data if not specified\n if minVal is None or maxVal is None:\n compMinVal, compMaxVal = _rangeGen(metricData)\n if minVal is None:\n minVal = compMinVal\n if maxVal is None:\n maxVal = compMaxVal\n\n # Handle the corner case where the incoming min and max are the same\n if minVal == maxVal:\n maxVal = minVal + 1\n\n # Load model parameters and update encoder params\n if (tmImplementation is \"cpp\"):\n paramFileRelativePath = os.path.join(\n \"anomaly_params_random_encoder\",\n \"best_single_metric_anomaly_params_cpp.json\")\n elif (tmImplementation is \"tm_cpp\"):\n paramFileRelativePath = os.path.join(\n \"anomaly_params_random_encoder\",\n \"best_single_metric_anomaly_params_tm_cpp.json\")\n else:\n raise ValueError(\"Invalid string for tmImplementation. Try cpp or tm_cpp\")\n\n with resource_stream(__name__, paramFileRelativePath) as infile:\n paramSet = json.load(infile)\n\n _fixupRandomEncoderParams(paramSet, minVal, maxVal, minResolution)\n\n return paramSet", "response": "Returns a dict that can be used to create an anomaly model for a set of time - of - day metric data."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning reasonable min and max values to use given the data.", "response": "def _rangeGen(data, std=1):\n \"\"\"\n Return reasonable min/max values to use given the data.\n \"\"\"\n dataStd = np.std(data)\n if dataStd == 0:\n dataStd = 1\n minval = np.min(data) - std * dataStd\n maxval = np.max(data) + std * dataStd\n return minval, maxval"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _fixupRandomEncoderParams(params, minVal, maxVal, minResolution):\n encodersDict = (\n params[\"modelConfig\"][\"modelParams\"][\"sensorParams\"][\"encoders\"]\n )\n\n for encoder in encodersDict.itervalues():\n if encoder is not None:\n if encoder[\"type\"] == \"RandomDistributedScalarEncoder\":\n resolution = max(minResolution,\n (maxVal - minVal) / encoder.pop(\"numBuckets\")\n )\n encodersDict[\"c1\"][\"resolution\"] = resolution", "response": "Given model params figure out the correct parameters for RandomDistributed encoder. Modifies params in place."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing the TemporalMemory state and returns it.", "response": "def read(cls, proto):\n \"\"\"\n Intercepts TemporalMemory deserialization request in order to initialize\n `TemporalMemoryMonitorMixin` state\n\n @param proto (DynamicStructBuilder) Proto object\n\n @return (TemporalMemory) TemporalMemory shim instance\n \"\"\"\n tm = super(TemporalMemoryMonitorMixin, cls).read(proto)\n\n # initialize `TemporalMemoryMonitorMixin` attributes\n tm.mmName = None\n tm._mmTraces = None\n tm._mmData = None\n tm.mmClearHistory()\n tm._mmResetActive = True\n return tm"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read(cls, proto):\n tm = super(TMShimMixin, cls).read(proto)\n tm.infActiveState = {\"t\": None}\n return tm", "response": "Deserializes the object into a TemporalMemory shim instance."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef topDownCompute(self, topDownIn=None):\n output = numpy.zeros(self.numberOfColumns())\n columns = [self.columnForCell(idx) for idx in self.getPredictiveCells()]\n output[columns] = 1\n return output", "response": "Top - down compute - generate expected input given output of the TM"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef read(cls, proto):\n tm = super(MonitoredTMShim, cls).read(proto)\n tm.infActiveState = {\"t\": None}\n return tm", "response": "Deserializes the object into a MonitoredTMShim instance."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute(self, bottomUpInput, enableLearn, computeInfOutput=None):\n super(MonitoredTMShim, self).compute(set(bottomUpInput.nonzero()[0]),\n learn=enableLearn)\n numberOfCells = self.numberOfCells()\n\n activeState = numpy.zeros(numberOfCells)\n activeState[self.getActiveCells()] = 1\n self.infActiveState[\"t\"] = activeState\n\n output = numpy.zeros(numberOfCells)\n output[self.getPredictiveCells() + self.getActiveCells()] = 1\n return output", "response": "This method computes the current state of the MonitoredTMShim."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pickByDistribution(distribution, r=None):\n\n if r is None:\n r = random\n\n x = r.uniform(0, sum(distribution))\n for i, d in enumerate(distribution):\n if x <= d:\n return i\n x -= d", "response": "Pick a value according to the provided distribution."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning an array of length size and type dtype that is everywhere 0, except in the index in pos. :param pos: (int) specifies the position of the one entry that will be set. :param size: (int) The total size of the array to be returned. :param dtype: The element type (compatible with NumPy array()) of the array to be returned. :returns: (list) of length ``size`` and element type ``dtype``.", "response": "def Indicator(pos, size, dtype):\n \"\"\"\n Returns an array of length size and type dtype that is everywhere 0,\n except in the index in pos.\n\n :param pos: (int) specifies the position of the one entry that will be set.\n :param size: (int) The total size of the array to be returned.\n :param dtype: The element type (compatible with NumPy array())\n of the array to be returned.\n :returns: (list) of length ``size`` and element type ``dtype``.\n \"\"\"\n x = numpy.zeros(size, dtype=dtype)\n x[pos] = 1\n return x"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget tuple (actually a generator) of indices where the max value of array x occurs. Requires that x have a max() method, as x.max() (in the case of NumPy) is much faster than max(x). For a simpler, faster argmax when there is only a single maximum entry, or when knowing only the first index where the maximum occurs, call argmax() on a NumPy array. :param x: Any sequence that has a max() method. :returns: Generator with the indices where the max value occurs.", "response": "def MultiArgMax(x):\n \"\"\"\n Get tuple (actually a generator) of indices where the max value of\n array x occurs. Requires that x have a max() method, as x.max()\n (in the case of NumPy) is much faster than max(x).\n For a simpler, faster argmax when there is only a single maximum entry,\n or when knowing only the first index where the maximum occurs,\n call argmax() on a NumPy array.\n\n :param x: Any sequence that has a max() method.\n :returns: Generator with the indices where the max value occurs.\n \"\"\"\n m = x.max()\n return (i for i, v in enumerate(x) if v == m)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Any(sequence):\n return bool(reduce(lambda x, y: x or y, sequence, False))", "response": "Tests much faster than bool. \n\n"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef All(sequence):\n return bool(reduce(lambda x, y: x and y, sequence, True))", "response": "Returns True if all elements of the given sequence satisfy True and False otherwise."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an array of length size and type dtype that is everywhere 0, except in the indices listed in sequence pos. :param pos: A single integer or sequence of integers that specify the position of ones to be set. :param size: The total size of the array to be returned. :param dtype: The element type (compatible with NumPy array()) of the array to be returned. :returns: An array of length size and element type dtype.", "response": "def MultiIndicator(pos, size, dtype):\n \"\"\"\n Returns an array of length size and type dtype that is everywhere 0,\n except in the indices listed in sequence pos.\n\n :param pos: A single integer or sequence of integers that specify\n the position of ones to be set.\n :param size: The total size of the array to be returned.\n :param dtype: The element type (compatible with NumPy array())\n of the array to be returned.\n :returns: An array of length size and element type dtype.\n \"\"\"\n x = numpy.zeros(size, dtype=dtype)\n if hasattr(pos, '__iter__'):\n for i in pos: x[i] = 1\n else: x[pos] = 1\n return x"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Distribution(pos, size, counts, dtype):\n x = numpy.zeros(size, dtype=dtype)\n if hasattr(pos, '__iter__'):\n # calculate normalization constant\n total = 0\n for i in pos:\n total += counts[i]\n total = float(total)\n # set included positions to normalized probability\n for i in pos:\n x[i] = counts[i]/total\n # If we don't have a set of positions, assume there's only one position\n else: x[pos] = 1\n return x", "response": "Returns an array of length size and type dtype that is everywhere 0."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef grow(self, rows, cols):\n if not self.hist_:\n self.hist_ = SparseMatrix(rows, cols)\n self.rowSums_ = numpy.zeros(rows, dtype=dtype)\n self.colSums_ = numpy.zeros(cols, dtype=dtype)\n self.hack_ = None\n else:\n oldRows = self.hist_.nRows()\n oldCols = self.hist_.nCols()\n nextRows = max(oldRows, rows)\n nextCols = max(oldCols, cols)\n if (oldRows < nextRows) or (oldCols < nextCols):\n self.hist_.resize(nextRows, nextCols)\n if oldRows < nextRows:\n oldSums = self.rowSums_\n self.rowSums_ = numpy.zeros(nextRows, dtype=dtype)\n self.rowSums_[0:len(oldSums)] = oldSums\n self.hack_ = None\n if oldCols < nextCols:\n oldSums = self.colSums_\n self.colSums_ = numpy.zeros(nextCols, dtype=dtype)\n self.colSums_[0:len(oldSums)] = oldSums\n self.hack_ = None", "response": "Grow the histogram to have rows and cols columns."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding a distribution to a row.", "response": "def updateRow(self, row, distribution):\n \"\"\"\n Add distribution to row row.\n Distribution should be an array of probabilities or counts.\n\n :param row: Integer index of the row to add to.\n May be larger than the current number of rows, in which case\n the histogram grows.\n :param distribution: Array of length equal to the number of columns.\n \"\"\"\n self.grow(row+1, len(distribution))\n self.hist_.axby(row, 1, 1, distribution)\n self.rowSums_[row] += distribution.sum()\n self.colSums_ += distribution\n self.hack_ = None"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef inferRowCompat(self, distribution):\n if self.hack_ is None:\n self.clean_outcpd()\n return self.hack_.vecMaxProd(distribution)", "response": "Equivalent to the category inference of zeta1. TopLevel."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef clean_outcpd(self):\n m = self.hist_.toDense()\n for j in xrange(m.shape[1]): # For each column.\n cmax = m[:,j].max()\n if cmax:\n m[:,j] = numpy.array(m[:,j] == cmax, dtype=dtype)\n self.hack_ = SparseMatrix(0, self.hist_.nCols())\n for i in xrange(m.shape[0]):\n self.hack_.addRow(m[i,:])", "response": "This function is used to clean outcpd on zeta1. TopLevelNode."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef importAndRunFunction(\n path,\n moduleName,\n funcName,\n **keywords\n ):\n \"\"\"\n Run a named function specified by a filesystem path, module name\n and function name.\n\n Returns the value returned by the imported function.\n\n Use this when access is needed to code that has\n not been added to a package accessible from the ordinary Python\n path. Encapsulates the multiple lines usually needed to\n safely manipulate and restore the Python path.\n\n Parameters\n ----------\n path: filesystem path\n Path to the directory where the desired module is stored.\n This will be used to temporarily augment the Python path.\n\n moduleName: basestring\n Name of the module, without trailing extension, where the desired\n function is stored. This module should be in the directory specified\n with path.\n\n funcName: basestring\n Name of the function to import and call.\n\n keywords:\n Keyword arguments to be passed to the imported function.\n \"\"\"\n import sys\n originalPath = sys.path\n try:\n augmentedPath = [path] + sys.path\n sys.path = augmentedPath\n func = getattr(__import__(moduleName, fromlist=[funcName]), funcName)\n sys.path = originalPath\n except:\n # Restore the original path in case of an exception.\n sys.path = originalPath\n raise\n return func(**keywords)", "response": "Imports a named function from the specified module and runs it."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntransferring coincidences from one element to another element.", "response": "def transferCoincidences(network, fromElementName, toElementName):\n \"\"\"\n Gets the coincidence matrix from one element and sets it on\n another element\n (using locked handles, a la nupic.bindings.research.lockHandle).\n\n TODO: Generalize to more node types, parameter name pairs, etc.\n\n Does not work across processes.\n \"\"\"\n coincidenceHandle = getLockedHandle(\n runtimeElement=network.getElement(fromElementName),\n # TODO: Re-purpose for use with nodes other than PMXClassifierNode.\n expression=\"self._cd._W\"\n )\n\n network.getElement(toElementName).setParameter(\"coincidencesAbove\",\n coincidenceHandle)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninstancing method wrapper around compute.", "response": "def next(self, newValue):\n \"\"\"Instance method wrapper around compute.\"\"\"\n newAverage, self.slidingWindow, self.total = self.compute(\n self.slidingWindow, self.total, newValue, self.windowSize)\n return newAverage"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning an appropriate metric module for the given metricSpec.", "response": "def getModule(metricSpec):\n \"\"\"\n Factory method to return an appropriate :class:`MetricsIface` module.\n \n - ``rmse``: :class:`MetricRMSE`\n - ``nrmse``: :class:`MetricNRMSE`\n - ``aae``: :class:`MetricAAE`\n - ``acc``: :class:`MetricAccuracy`\n - ``avg_err``: :class:`MetricAveError`\n - ``trivial``: :class:`MetricTrivial`\n - ``two_gram``: :class:`MetricTwoGram`\n - ``moving_mean``: :class:`MetricMovingMean`\n - ``moving_mode``: :class:`MetricMovingMode`\n - ``neg_auc``: :class:`MetricNegAUC`\n - ``custom_error_metric``: :class:`CustomErrorMetric`\n - ``multiStep``: :class:`MetricMultiStep`\n - ``ms_aae``: :class:`MetricMultiStepAAE`\n - ``ms_avg_err``: :class:`MetricMultiStepAveError`\n - ``passThruPrediction``: :class:`MetricPassThruPrediction`\n - ``altMAPE``: :class:`MetricAltMAPE`\n - ``MAPE``: :class:`MetricMAPE`\n - ``multi``: :class:`MetricMulti`\n - ``negativeLogLikelihood``: :class:`MetricNegativeLogLikelihood`\n \n :param metricSpec: (:class:`MetricSpec`) metric to find module for. \n ``metricSpec.metric`` must be in the list above.\n \n :returns: (:class:`AggregateMetric`) an appropriate metric module\n \"\"\"\n\n metricName = metricSpec.metric\n\n if metricName == 'rmse':\n return MetricRMSE(metricSpec)\n if metricName == 'nrmse':\n return MetricNRMSE(metricSpec)\n elif metricName == 'aae':\n return MetricAAE(metricSpec)\n elif metricName == 'acc':\n return MetricAccuracy(metricSpec)\n elif metricName == 'avg_err':\n return MetricAveError(metricSpec)\n elif metricName == 'trivial':\n return MetricTrivial(metricSpec)\n elif metricName == 'two_gram':\n return MetricTwoGram(metricSpec)\n elif metricName == 'moving_mean':\n return MetricMovingMean(metricSpec)\n elif metricName == 'moving_mode':\n return MetricMovingMode(metricSpec)\n elif metricName == 'neg_auc':\n return MetricNegAUC(metricSpec)\n elif metricName == 'custom_error_metric':\n return CustomErrorMetric(metricSpec)\n elif metricName == 'multiStep':\n return MetricMultiStep(metricSpec)\n elif metricName == 'multiStepProbability':\n return MetricMultiStepProbability(metricSpec)\n elif metricName == 'ms_aae':\n return MetricMultiStepAAE(metricSpec)\n elif metricName == 'ms_avg_err':\n return MetricMultiStepAveError(metricSpec)\n elif metricName == 'passThruPrediction':\n return MetricPassThruPrediction(metricSpec)\n elif metricName == 'altMAPE':\n return MetricAltMAPE(metricSpec)\n elif metricName == 'MAPE':\n return MetricMAPE(metricSpec)\n elif metricName == 'multi':\n return MetricMulti(metricSpec)\n elif metricName == 'negativeLogLikelihood':\n return MetricNegativeLogLikelihood(metricSpec)\n else:\n raise Exception(\"Unsupported metric type: %s\" % metricName)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getLabel(self, inferenceType=None):\n result = []\n if inferenceType is not None:\n result.append(InferenceType.getLabel(inferenceType))\n result.append(self.inferenceElement)\n result.append(self.metric)\n\n params = self.params\n if params is not None:\n\n sortedParams= params.keys()\n sortedParams.sort()\n for param in sortedParams:\n # Don't include the customFuncSource - it is too long an unwieldy\n if param in ('customFuncSource', 'customFuncDef', 'customExpr'):\n continue\n value = params[param]\n if isinstance(value, str):\n result.extend([\"%s='%s'\"% (param, value)])\n else:\n result.extend([\"%s=%s\"% (param, value)])\n\n if self.field:\n result.append(\"field=%s\"% (self.field) )\n\n return self._LABEL_SEPARATOR.join(result)", "response": "Returns a unique label for a metricSpec."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getInferenceTypeFromLabel(cls, label):\n infType, _, _= label.partition(cls._LABEL_SEPARATOR)\n\n if not InferenceType.validate(infType):\n return None\n\n return infType", "response": "Extracts the PredictionKind from the given label."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef addInstance(self, groundTruth, prediction, record = None, result = None):\n self.value = self.avg(prediction)", "response": "Compute and store metric value"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef mostLikely(self, pred):\n if len(pred) == 1:\n return pred.keys()[0]\n\n mostLikelyOutcome = None\n maxProbability = 0\n\n for prediction, probability in pred.items():\n if probability > maxProbability:\n mostLikelyOutcome = prediction\n maxProbability = probability\n\n return mostLikelyOutcome", "response": "Helper function to return a scalar value representing the most likely outcome given a probability distribution\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef expValue(self, pred):\n if len(pred) == 1:\n return pred.keys()[0]\n\n return sum([x*p for x,p in pred.items()])", "response": "Helper function to return a scalar value representing the expected\n value of a probability distribution."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getScalarNames(self, parentFieldName=''):\n names = []\n\n if self.encoders is not None:\n for (name, encoder, offset) in self.encoders:\n subNames = encoder.getScalarNames(parentFieldName=name)\n if parentFieldName != '':\n subNames = ['%s.%s' % (parentFieldName, name) for name in subNames]\n names.extend(subNames)\n else:\n if parentFieldName != '':\n names.append(parentFieldName)\n else:\n names.append(self.name)\n\n return names", "response": "Returns the field names for each of the scalar values returned by this object."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getDecoderOutputFieldTypes(self):\n if hasattr(self, '_flattenedFieldTypeList') and \\\n self._flattenedFieldTypeList is not None:\n return self._flattenedFieldTypeList\n\n fieldTypes = []\n\n # NOTE: we take care of the composites, but leaf encoders must override\n # this method and return a list of one field_meta.FieldMetaType.XXXX\n # element corresponding to the encoder's decoder output field type\n for (name, encoder, offset) in self.encoders:\n subTypes = encoder.getDecoderOutputFieldTypes()\n fieldTypes.extend(subTypes)\n\n self._flattenedFieldTypeList = fieldTypes\n return fieldTypes", "response": "Returns a list of field types corresponding to the elements in the decoder output field array."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _getInputValue(self, obj, fieldName):\n if isinstance(obj, dict):\n if not fieldName in obj:\n knownFields = \", \".join(\n key for key in obj.keys() if not key.startswith(\"_\")\n )\n raise ValueError(\n \"Unknown field name '%s' in input record. Known fields are '%s'.\\n\"\n \"This could be because input headers are mislabeled, or because \"\n \"input data rows do not contain a value for '%s'.\" % (\n fieldName, knownFields, fieldName\n )\n )\n return obj[fieldName]\n else:\n return getattr(obj, fieldName)", "response": "Gets the value of a given field from the input record"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a list of all sub - encoders in this encoder", "response": "def getEncoderList(self):\n \"\"\"\n :return: a reference to each sub-encoder in this encoder. They are\n returned in the same order as they are for :meth:`.getScalarNames`\n and :meth:`.getScalars`.\n\n \"\"\"\n if hasattr(self, '_flattenedEncoderList') and \\\n self._flattenedEncoderList is not None:\n\n return self._flattenedEncoderList\n\n encoders = []\n\n if self.encoders is not None:\n for (name, encoder, offset) in self.encoders:\n subEncoders = encoder.getEncoderList()\n encoders.extend(subEncoders)\n else:\n encoders.append(self)\n\n self._flattenedEncoderList = encoders\n return encoders"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the scalar values for each sub - field of the inputData.", "response": "def getScalars(self, inputData):\n \"\"\"\n Returns a numpy array containing the sub-field scalar value(s) for\n each sub-field of the ``inputData``. To get the associated field names for\n each of the scalar values, call :meth:`.getScalarNames()`.\n\n For a simple scalar encoder, the scalar value is simply the input unmodified.\n For category encoders, it is the scalar representing the category string\n that is passed in. For the datetime encoder, the scalar value is the\n the number of seconds since epoch.\n\n The intent of the scalar representation of a sub-field is to provide a\n baseline for measuring error differences. You can compare the scalar value\n of the inputData with the scalar value returned from :meth:`.topDownCompute`\n on a top-down representation to evaluate prediction accuracy, for example.\n\n :param inputData: The data from the source. This is typically an object with\n members\n :return: array of scalar values\n \"\"\"\n\n retVals = numpy.array([])\n\n if self.encoders is not None:\n for (name, encoder, offset) in self.encoders:\n values = encoder.getScalars(self._getInputValue(inputData, name))\n retVals = numpy.hstack((retVals, values))\n else:\n retVals = numpy.hstack((retVals, inputData))\n\n return retVals"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getEncodedValues(self, inputData):\n\n retVals = []\n\n if self.encoders is not None:\n for name, encoders, offset in self.encoders:\n values = encoders.getEncodedValues(self._getInputValue(inputData, name))\n\n if _isSequence(values):\n retVals.extend(values)\n else:\n retVals.append(values)\n else:\n if _isSequence(inputData):\n retVals.extend(inputData)\n else:\n retVals.append(inputData)\n\n return tuple(retVals)", "response": "Returns the input data in the same format as is returned by the topDownCompute method."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the indices of each sub - field bucket of the inputData.", "response": "def getBucketIndices(self, inputData):\n \"\"\"\n Returns an array containing the sub-field bucket indices for each sub-field\n of the inputData. To get the associated field names for each of the buckets,\n call :meth:`.getScalarNames`.\n\n :param inputData: The data from the source. This is typically an object with\n members.\n :return: array of bucket indices\n \"\"\"\n\n retVals = []\n\n if self.encoders is not None:\n for (name, encoder, offset) in self.encoders:\n values = encoder.getBucketIndices(self._getInputValue(inputData, name))\n retVals.extend(values)\n else:\n assert False, \"Should be implemented in base classes that are not \" \\\n \"containers for other encoders\"\n\n return retVals"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef scalarsToStr(self, scalarValues, scalarNames=None):\n\n if scalarNames is None:\n scalarNames = self.getScalarNames()\n\n desc = ''\n for (name, value) in zip(scalarNames, scalarValues):\n if len(desc) > 0:\n desc += \", %s:%.2f\" % (name, value)\n else:\n desc += \"%s:%.2f\" % (name, value)\n\n return desc", "response": "Returns a pretty print string representing the return values from the given scalar values."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getFieldDescription(self, fieldName):\n\n # Find which field it's in\n description = self.getDescription() + [(\"end\", self.getWidth())]\n for i in xrange(len(description)):\n (name, offset) = description[i]\n if (name == fieldName):\n break\n\n if i >= len(description)-1:\n raise RuntimeError(\"Field name %s not found in this encoder\" % fieldName)\n\n # Return the offset and width\n return (offset, description[i+1][1] - offset)", "response": "Returns the offset and length of a given field within the encoded output."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef encodedBitDescription(self, bitOffset, formatted=False):\n\n # Find which field it's in\n (prevFieldName, prevFieldOffset) = (None, None)\n description = self.getDescription()\n for i in xrange(len(description)):\n (name, offset) = description[i]\n if formatted:\n offset = offset + i\n if bitOffset == offset-1:\n prevFieldName = \"separator\"\n prevFieldOffset = bitOffset\n break\n\n if bitOffset < offset:\n break\n (prevFieldName, prevFieldOffset) = (name, offset)\n\n # Return the field name and offset within the field\n # return (fieldName, bitOffset - fieldOffset)\n width = self.getDisplayWidth() if formatted else self.getWidth()\n\n if prevFieldOffset is None or bitOffset > self.getWidth():\n raise IndexError(\"Bit is outside of allowable range: [0 - %d]\" % width)\n\n return (prevFieldName, bitOffset - prevFieldOffset)", "response": "Return a description of the given bit in the encoded output."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef decode(self, encoded, parentFieldName=''):\n\n fieldsDict = dict()\n fieldsOrder = []\n\n # What is the effective parent name?\n if parentFieldName == '':\n parentName = self.name\n else:\n parentName = \"%s.%s\" % (parentFieldName, self.name)\n\n if self.encoders is not None:\n # Merge decodings of all child encoders together\n for i in xrange(len(self.encoders)):\n\n # Get the encoder and the encoded output\n (name, encoder, offset) = self.encoders[i]\n if i < len(self.encoders)-1:\n nextOffset = self.encoders[i+1][2]\n else:\n nextOffset = self.width\n fieldOutput = encoded[offset:nextOffset]\n (subFieldsDict, subFieldsOrder) = encoder.decode(fieldOutput,\n parentFieldName=parentName)\n\n fieldsDict.update(subFieldsDict)\n fieldsOrder.extend(subFieldsOrder)\n\n\n return (fieldsDict, fieldsOrder)", "response": "This method decodes the encoded output into a tuple of the name of the key and the value of each entry in the dictionary."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef decodedToStr(self, decodeResults):\n\n (fieldsDict, fieldsOrder) = decodeResults\n\n desc = ''\n for fieldName in fieldsOrder:\n (ranges, rangesStr) = fieldsDict[fieldName]\n if len(desc) > 0:\n desc += \", %s:\" % (fieldName)\n else:\n desc += \"%s:\" % (fieldName)\n\n desc += \"[%s]\" % (rangesStr)\n\n return desc", "response": "Return a pretty print string representing the return value from\n ."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a list of namedtuples describing the inputs for each sub - field that correspond to the bucket indices passed in buckets.", "response": "def getBucketInfo(self, buckets):\n \"\"\"\n Returns a list of :class:`.EncoderResult` namedtuples describing the inputs\n for each sub-field that correspond to the bucket indices passed in\n ``buckets``. To get the associated field names for each of the values, call\n :meth:`.getScalarNames`.\n\n :param buckets: The list of bucket indices, one for each sub-field encoder.\n These bucket indices for example may have been retrieved\n from the :meth:`.getBucketIndices` call.\n :return: A list of :class:`.EncoderResult`.\n \"\"\"\n # Fall back topdown compute\n if self.encoders is None:\n raise RuntimeError(\"Must be implemented in sub-class\")\n\n # Concatenate the results from bucketInfo on each child encoder\n retVals = []\n bucketOffset = 0\n for i in xrange(len(self.encoders)):\n (name, encoder, offset) = self.encoders[i]\n\n if encoder.encoders is not None:\n nextBucketOffset = bucketOffset + len(encoder.encoders)\n else:\n nextBucketOffset = bucketOffset + 1\n bucketIndices = buckets[bucketOffset:nextBucketOffset]\n values = encoder.getBucketInfo(bucketIndices)\n\n retVals.extend(values)\n\n bucketOffset = nextBucketOffset\n\n return retVals"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncomputing the top - down best guess inputs for each sub - field given the encoded output.", "response": "def topDownCompute(self, encoded):\n \"\"\"\n Returns a list of :class:`.EncoderResult` namedtuples describing the\n top-down best guess inputs for each sub-field given the encoded output.\n These are the values which are most likely to generate the given encoded\n output. To get the associated field names for each of the values, call\n :meth:`.getScalarNames`.\n\n :param encoded: The encoded output. Typically received from the topDown\n outputs from the spatial pooler just above us.\n\n :return: A list of :class:`.EncoderResult`\n \"\"\"\n # Fallback topdown compute\n if self.encoders is None:\n raise RuntimeError(\"Must be implemented in sub-class\")\n\n # Concatenate the results from topDownCompute on each child encoder\n retVals = []\n for i in xrange(len(self.encoders)):\n (name, encoder, offset) = self.encoders[i]\n\n if i < len(self.encoders)-1:\n nextOffset = self.encoders[i+1][2]\n else:\n nextOffset = self.width\n\n fieldOutput = encoded[offset:nextOffset]\n values = encoder.topDownCompute(fieldOutput)\n\n if _isSequence(values):\n retVals.extend(values)\n else:\n retVals.append(values)\n\n return retVals"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef closenessScores(self, expValues, actValues, fractional=True):\n # Fallback closenss is a percentage match\n if self.encoders is None:\n err = abs(expValues[0] - actValues[0])\n if fractional:\n denom = max(expValues[0], actValues[0])\n if denom == 0:\n denom = 1.0\n closeness = 1.0 - float(err)/denom\n if closeness < 0:\n closeness = 0\n else:\n closeness = err\n\n return numpy.array([closeness])\n\n # Concatenate the results from closeness scores on each child encoder\n scalarIdx = 0\n retVals = numpy.array([])\n for (name, encoder, offset) in self.encoders:\n values = encoder.closenessScores(expValues[scalarIdx:], actValues[scalarIdx:],\n fractional=fractional)\n scalarIdx += len(values)\n retVals = numpy.hstack((retVals, values))\n\n return retVals", "response": "Compute closeness scores between the expected and actual scalar values."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nposts method for storing model in the model registry", "response": "def POST(self, name):\n \"\"\"\n /models/{name}\n\n schema:\n {\n \"modelParams\": dict containing model parameters\n \"predictedFieldName\": str\n }\n\n returns:\n {\"success\":name}\n \"\"\"\n global g_models\n\n data = json.loads(web.data())\n modelParams = data[\"modelParams\"]\n predictedFieldName = data[\"predictedFieldName\"]\n\n if name in g_models.keys():\n raise web.badrequest(\"Model with name <%s> already exists\" % name)\n\n model = ModelFactory.create(modelParams)\n model.enableInference({'predictedField': predictedFieldName})\n g_models[name] = model\n\n return json.dumps({\"success\": name})"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef POST(self, name):\n global g_models\n\n data = json.loads(web.data())\n data[\"timestamp\"] = datetime.datetime.strptime(\n data[\"timestamp\"], \"%m/%d/%y %H:%M\")\n\n if name not in g_models.keys():\n raise web.notfound(\"Model with name <%s> does not exist.\" % name)\n\n modelResult = g_models[name].run(data)\n predictionNumber = modelResult.predictionNumber\n anomalyScore = modelResult.inferences[\"anomalyScore\"]\n\n return json.dumps({\"predictionNumber\": predictionNumber,\n \"anomalyScore\": anomalyScore})", "response": "POST method for training a new resource in the\neer model with the given name."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmirrors Image Visualization: Shows the encoding space juxtaposed against the coincidence space. The encoding space is the bottom-up sensory encoding and the coincidence space depicts the corresponding activation of coincidences in the SP. Hence, the mirror image visualization is a visual depiction of the mapping of SP cells to the input representations. Note: * The files spBUOut and sensorBUOut are assumed to be in the output format used for LPF experiment outputs. * BU outputs for some sample datasets are provided. Specify the name of the dataset as an option while running this script.", "response": "def analyzeOverlaps(activeCoincsFile, encodingsFile, dataset):\n '''Mirror Image Visualization: Shows the encoding space juxtaposed against the\n coincidence space. The encoding space is the bottom-up sensory encoding and\n the coincidence space depicts the corresponding activation of coincidences in\n the SP. Hence, the mirror image visualization is a visual depiction of the\n mapping of SP cells to the input representations.\n \n Note:\n * The files spBUOut and sensorBUOut are assumed to be in the output format\n used for LPF experiment outputs.\n * BU outputs for some sample datasets are provided. Specify the name of the\n dataset as an option while running this script. \n '''\n \n lines = activeCoincsFile.readlines()\n inputs = encodingsFile.readlines()\n w = len(inputs[0].split(' '))-1\n\n patterns = set([])\n encodings = set([])\n coincs = [] #The set of all coincidences that have won at least once\n reUsedCoincs = []\n \n firstLine = inputs[0].split(' ')\n size = int(firstLine.pop(0))\n spOutput = np.zeros((len(lines),40))\n inputBits = np.zeros((len(lines),w))\n print 'Total n:', size\n print 'Total number of records in the file:', len(lines), '\\n'\n print 'w:', w\n \n count = 0\n for x in xrange(len(lines)):\n inputSpace = [] #Encoded representation for each input \n \n spBUout = [int(z) for z in lines[x].split(' ')] \n spBUout.pop(0) #The first element of each row of spBUOut is the size of the SP \n temp = set(spBUout)\n spOutput[x]=spBUout\n \n input = [int(z) for z in inputs[x].split(' ')] \n input.pop(0) #The first element of each row of sensorBUout is the size of the encoding space\n tempInput = set(input)\n inputBits[x]=input\n \n #Creating the encoding space \n for m in xrange(size):\n if m in tempInput:\n inputSpace.append(m)\n else:\n inputSpace.append('|') #A non-active bit\n \n repeatedBits = tempInput.intersection(encodings) #Storing the bits that have been previously active\n reUsed = temp.intersection(patterns) #Checking if any of the active cells have been previously active \n \n #Dividing the coincidences into two difference categories. \n if len(reUsed)==0:\n coincs.append((count,temp,repeatedBits,inputSpace, tempInput)) #Pattern no, active cells, repeated bits, encoding (full), encoding (summary)\n else:\n reUsedCoincs.append((count,temp,repeatedBits,inputSpace, tempInput))\n patterns=patterns.union(temp) #Adding the active cells to the set of coincs that have been active at least once\n \n encodings = encodings.union(tempInput)\n count +=1\n \n overlap = {}\n overlapVal = 0\n\n seen = []\n seen = (printOverlaps(coincs, coincs, seen))\n print len(seen), 'sets of 40 cells'\n seen = printOverlaps(reUsedCoincs, coincs, seen)\n \n Summ=[]\n for z in coincs:\n c=0\n for y in reUsedCoincs:\n c += len(z[1].intersection(y[1]))\n Summ.append(c)\n print 'Sum: ', Summ\n \n for m in xrange(3):\n displayLimit = min(51, len(spOutput[m*200:]))\n if displayLimit>0:\n drawFile(dataset, np.zeros([len(inputBits[:(m+1)*displayLimit]),len(inputBits[:(m+1)*displayLimit])]), inputBits[:(m+1)*displayLimit], spOutput[:(m+1)*displayLimit], w, m+1)\n else: \n print 'No more records to display'\n pyl.show()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndraw the bit - encoding space.", "response": "def drawFile(dataset, matrix, patterns, cells, w, fnum):\n '''The similarity of two patterns in the bit-encoding space is displayed alongside\n their similarity in the sp-coinc space.'''\n score=0\n count = 0\n assert len(patterns)==len(cells)\n for p in xrange(len(patterns)-1):\n matrix[p+1:,p] = [len(set(patterns[p]).intersection(set(q)))*100/w for q in patterns[p+1:]]\n matrix[p,p+1:] = [len(set(cells[p]).intersection(set(r)))*5/2 for r in cells[p+1:]]\n \n score += sum(abs(np.array(matrix[p+1:,p])-np.array(matrix[p,p+1:])))\n count += len(matrix[p+1:,p])\n \n print 'Score', score/count\n \n fig = pyl.figure(figsize = (10,10), num = fnum)\n pyl.matshow(matrix, fignum = fnum)\n pyl.colorbar()\n pyl.title('Coincidence Space', verticalalignment='top', fontsize=12)\n pyl.xlabel('The Mirror Image Visualization for '+dataset, fontsize=17)\n pyl.ylabel('Encoding space', fontsize=12)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef printOverlaps(comparedTo, coincs, seen):\n inputOverlap = 0\n cellOverlap = 0\n for y in comparedTo:\n closestInputs = []\n closestCells = []\n if len(seen)>0:\n inputOverlap = max([len(seen[m][1].intersection(y[4])) for m in xrange(len(seen))])\n cellOverlap = max([len(seen[m][0].intersection(y[1])) for m in xrange(len(seen))])\n for m in xrange( len(seen) ):\n if len(seen[m][1].intersection(y[4]))==inputOverlap:\n closestInputs.append(seen[m][2])\n if len(seen[m][0].intersection(y[1]))==cellOverlap:\n closestCells.append(seen[m][2])\n seen.append((y[1], y[4], y[0]))\n \n print 'Pattern',y[0]+1,':',' '.join(str(len(z[1].intersection(y[1]))).rjust(2) for z in coincs),'input overlap:', inputOverlap, ';', len(closestInputs), 'closest encodings:',','.join(str(m+1) for m in closestInputs).ljust(15), \\\n 'cell overlap:', cellOverlap, ';', len(closestCells), 'closest set(s):',','.join(str(m+1) for m in closestCells)\n \n return seen", "response": "Print the overlap between two sets of cells."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef createInput(self):\n\n print \"-\" * 70 + \"Creating a random input vector\" + \"-\" * 70\n\n #clear the inputArray to zero before creating a new input vector\n self.inputArray[0:] = 0\n\n for i in range(self.inputSize):\n #randrange returns 0 or 1\n self.inputArray[i] = random.randrange(2)", "response": "create a random input vector"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns the spatial pooler with the input vector", "response": "def run(self):\n \"\"\"Run the spatial pooler with the input vector\"\"\"\n\n print \"-\" * 80 + \"Computing the SDR\" + \"-\" * 80\n\n #activeArray[column]=1 if column is active after spatial pooling\n self.sp.compute(self.inputArray, True, self.activeArray)\n\n print self.activeArray.nonzero()"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds noise to the base image", "response": "def addNoise(self, noiseLevel):\n \"\"\"Flip the value of 10% of input bits (add noise)\n\n :param noiseLevel: The percentage of total input bits that should be flipped\n \"\"\"\n\n for _ in range(int(noiseLevel * self.inputSize)):\n # 0.1*self.inputSize represents 10% of the total input bits\n # random.random() returns a float between 0 and 1\n randomPosition = int(random.random() * self.inputSize)\n\n # Flipping the bit at the randomly picked position\n if self.inputArray[randomPosition] == 1:\n self.inputArray[randomPosition] = 0\n\n else:\n self.inputArray[randomPosition] = 1"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _labeledInput(activeInputs, cellsPerCol=32):\n if cellsPerCol == 0:\n cellsPerCol = 1\n cols = activeInputs.size / cellsPerCol\n activeInputs = activeInputs.reshape(cols, cellsPerCol)\n (cols, cellIdxs) = activeInputs.nonzero()\n\n if len(cols) == 0:\n return \"NONE\"\n\n items = [\"(%d): \" % (len(cols))]\n prevCol = -1\n for (col,cellIdx) in zip(cols, cellIdxs):\n if col != prevCol:\n if prevCol != -1:\n items.append(\"] \")\n items.append(\"Col %d: [\" % col)\n prevCol = col\n\n items.append(\"%d,\" % cellIdx)\n\n items.append(\"]\")\n return \" \".join(items)", "response": "Print the list of column and cell indices for each of the active\n cells in activeInputs."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nclearing the state of the KNNClassifier.", "response": "def clear(self):\n \"\"\"Clears the state of the KNNClassifier.\"\"\"\n self._Memory = None\n self._numPatterns = 0\n self._M = None\n self._categoryList = []\n self._partitionIdList = []\n self._partitionIdMap = {}\n self._finishedLearning = False\n self._iterationIdx = -1\n\n # Fixed capacity KNN\n if self.maxStoredPatterns > 0:\n assert self.useSparseMemory, (\"Fixed capacity KNN is implemented only \"\n \"in the sparse memory mode\")\n self.fixedCapacity = True\n self._categoryRecencyList = []\n else:\n self.fixedCapacity = False\n\n # Cached value of the store prototype sizes\n self._protoSizes = None\n\n # Used by PCA\n self._s = None\n self._vt = None\n self._nc = None\n self._mean = None\n\n # Used by Network Builder\n self._specificIndexTraining = False\n self._nextTrainingIndices = None"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nallows ids to be assigned a category and subsequently enables users to use: - :meth:`~.KNNClassifier.KNNClassifier.removeCategory` - :meth:`~.KNNClassifier.KNNClassifier.closestTrainingPattern` - :meth:`~.KNNClassifier.KNNClassifier.closestOtherTrainingPattern`", "response": "def prototypeSetCategory(self, idToCategorize, newCategory):\n \"\"\"\n Allows ids to be assigned a category and subsequently enables users to use:\n\n - :meth:`~.KNNClassifier.KNNClassifier.removeCategory`\n - :meth:`~.KNNClassifier.KNNClassifier.closestTrainingPattern`\n - :meth:`~.KNNClassifier.KNNClassifier.closestOtherTrainingPattern`\n \"\"\"\n if idToCategorize not in self._categoryRecencyList:\n return\n\n recordIndex = self._categoryRecencyList.index(idToCategorize)\n self._categoryList[recordIndex] = newCategory"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef removeIds(self, idsToRemove):\n # Form a list of all categories to remove\n rowsToRemove = [k for k, rowID in enumerate(self._categoryRecencyList) \\\n if rowID in idsToRemove]\n\n # Remove rows from the classifier\n self._removeRows(rowsToRemove)", "response": "Removes the rows from the classifier with the given ids."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef removeCategory(self, categoryToRemove):\n removedRows = 0\n if self._Memory is None:\n return removedRows\n\n # The internal category indices are stored in float\n # format, so we should compare with a float\n catToRemove = float(categoryToRemove)\n\n # Form a list of all categories to remove\n rowsToRemove = [k for k, catID in enumerate(self._categoryList) \\\n if catID == catToRemove]\n\n # Remove rows from the classifier\n self._removeRows(rowsToRemove)\n\n assert catToRemove not in self._categoryList", "response": "Removes a category from the classifier."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _removeRows(self, rowsToRemove):\n # Form a numpy array of row indices to be removed\n removalArray = numpy.array(rowsToRemove)\n\n # Remove categories\n self._categoryList = numpy.delete(numpy.array(self._categoryList),\n removalArray).tolist()\n\n if self.fixedCapacity:\n self._categoryRecencyList = numpy.delete(\n numpy.array(self._categoryRecencyList), removalArray).tolist()\n\n # Remove the partition ID, if any for these rows and rebuild the id map.\n for row in reversed(rowsToRemove): # Go backwards\n # Remove these patterns from partitionList\n self._partitionIdList.pop(row)\n self._rebuildPartitionIdMap(self._partitionIdList)\n\n\n # Remove actual patterns\n if self.useSparseMemory:\n # Delete backwards\n for rowIndex in rowsToRemove[::-1]:\n self._Memory.deleteRow(rowIndex)\n else:\n self._M = numpy.delete(self._M, removalArray, 0)\n\n numRemoved = len(rowsToRemove)\n\n # Sanity checks\n numRowsExpected = self._numPatterns - numRemoved\n if self.useSparseMemory:\n if self._Memory is not None:\n assert self._Memory.nRows() == numRowsExpected\n else:\n assert self._M.shape[0] == numRowsExpected\n assert len(self._categoryList) == numRowsExpected\n\n self._numPatterns -= numRemoved\n return numRemoved", "response": "Removes the rows from the internal list of rows in the internal list."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef learn(self, inputPattern, inputCategory, partitionId=None, isSparse=0,\n rowID=None):\n \"\"\"\n Train the classifier to associate specified input pattern with a\n particular category.\n\n :param inputPattern: (list) The pattern to be assigned a category. If\n isSparse is 0, this should be a dense array (both ON and OFF bits\n present). Otherwise, if isSparse > 0, this should be a list of the\n indices of the non-zero bits in sorted order\n\n :param inputCategory: (int) The category to be associated to the training\n pattern\n\n :param partitionId: (int) partitionID allows you to associate an id with each\n input vector. It can be used to associate input patterns stored in the\n classifier with an external id. This can be useful for debugging or\n visualizing. Another use case is to ignore vectors with a specific id\n during inference (see description of infer() for details). There can be\n at most one partitionId per stored pattern (i.e. if two patterns are\n within distThreshold, only the first partitionId will be stored). This\n is an optional parameter.\n\n :param isSparse: (int) 0 if the input pattern is a dense representation.\n When the input pattern is a list of non-zero indices, then isSparse\n is the number of total bits (n). E.g. for the dense array\n [0, 1, 1, 0, 0, 1], isSparse should be `0`. For the equivalent sparse\n representation [1, 2, 5] (which specifies the indices of active bits),\n isSparse should be `6`, which is the total number of bits in the input\n space.\n\n :param rowID: (int) UNKNOWN\n\n :returns: The number of patterns currently stored in the classifier\n \"\"\"\n if self.verbosity >= 1:\n print \"%s learn:\" % g_debugPrefix\n print \" category:\", int(inputCategory)\n print \" active inputs:\", _labeledInput(inputPattern,\n cellsPerCol=self.cellsPerCol)\n\n if isSparse > 0:\n assert all(inputPattern[i] <= inputPattern[i+1]\n for i in xrange(len(inputPattern)-1)), \\\n \"Sparse inputPattern must be sorted.\"\n assert all(bit < isSparse for bit in inputPattern), \\\n (\"Sparse inputPattern must not index outside the dense \"\n \"representation's bounds.\")\n\n if rowID is None:\n rowID = self._iterationIdx\n\n # Dense vectors\n if not self.useSparseMemory:\n\n # Not supported\n assert self.cellsPerCol == 0, \"not implemented for dense vectors\"\n\n # If the input was given in sparse form, convert it to dense\n if isSparse > 0:\n denseInput = numpy.zeros(isSparse)\n denseInput[inputPattern] = 1.0\n inputPattern = denseInput\n\n if self._specificIndexTraining and not self._nextTrainingIndices:\n # Specific index mode without any index provided - skip training\n return self._numPatterns\n\n if self._Memory is None:\n # Initialize memory with 100 rows and numPatterns = 0\n inputWidth = len(inputPattern)\n self._Memory = numpy.zeros((100,inputWidth))\n self._numPatterns = 0\n self._M = self._Memory[:self._numPatterns]\n\n addRow = True\n\n if self._vt is not None:\n # Compute projection\n inputPattern = numpy.dot(self._vt, inputPattern - self._mean)\n\n if self.distThreshold > 0:\n # Check if input is too close to an existing input to be accepted\n dist = self._calcDistance(inputPattern)\n minDist = dist.min()\n addRow = (minDist >= self.distThreshold)\n\n if addRow:\n self._protoSizes = None # need to re-compute\n if self._numPatterns == self._Memory.shape[0]:\n # Double the size of the memory\n self._doubleMemoryNumRows()\n\n if not self._specificIndexTraining:\n # Normal learning - append the new input vector\n self._Memory[self._numPatterns] = inputPattern\n self._numPatterns += 1\n self._categoryList.append(int(inputCategory))\n else:\n # Specific index training mode - insert vector in specified slot\n vectorIndex = self._nextTrainingIndices.pop(0)\n while vectorIndex >= self._Memory.shape[0]:\n self._doubleMemoryNumRows()\n self._Memory[vectorIndex] = inputPattern\n self._numPatterns = max(self._numPatterns, vectorIndex + 1)\n if vectorIndex >= len(self._categoryList):\n self._categoryList += [-1] * (vectorIndex -\n len(self._categoryList) + 1)\n self._categoryList[vectorIndex] = int(inputCategory)\n\n # Set _M to the \"active\" part of _Memory\n self._M = self._Memory[0:self._numPatterns]\n\n self._addPartitionId(self._numPatterns-1, partitionId)\n\n # Sparse vectors\n else:\n\n # If the input was given in sparse form, convert it to dense if necessary\n if isSparse > 0 and (self._vt is not None or self.distThreshold > 0 \\\n or self.numSVDDims is not None or self.numSVDSamples > 0 \\\n or self.numWinners > 0):\n denseInput = numpy.zeros(isSparse)\n denseInput[inputPattern] = 1.0\n inputPattern = denseInput\n isSparse = 0\n\n # Get the input width\n if isSparse > 0:\n inputWidth = isSparse\n else:\n inputWidth = len(inputPattern)\n\n # Allocate storage if this is the first training vector\n if self._Memory is None:\n self._Memory = NearestNeighbor(0, inputWidth)\n\n # Support SVD if it is on\n if self._vt is not None:\n inputPattern = numpy.dot(self._vt, inputPattern - self._mean)\n\n # Threshold the input, zeroing out entries that are too close to 0.\n # This is only done if we are given a dense input.\n if isSparse == 0:\n thresholdedInput = self._sparsifyVector(inputPattern, True)\n addRow = True\n\n # If given the layout of the cells, then turn on the logic that stores\n # only the start cell for bursting columns.\n if self.cellsPerCol >= 1:\n burstingCols = thresholdedInput.reshape(-1,\n self.cellsPerCol).min(axis=1).nonzero()[0]\n for col in burstingCols:\n thresholdedInput[(col * self.cellsPerCol) + 1 :\n (col * self.cellsPerCol) + self.cellsPerCol] = 0\n\n\n # Don't learn entries that are too close to existing entries.\n if self._Memory.nRows() > 0:\n dist = None\n # if this vector is a perfect match for one we already learned, then\n # replace the category - it may have changed with online learning on.\n if self.replaceDuplicates:\n dist = self._calcDistance(thresholdedInput, distanceNorm=1)\n if dist.min() == 0:\n rowIdx = dist.argmin()\n self._categoryList[rowIdx] = int(inputCategory)\n if self.fixedCapacity:\n self._categoryRecencyList[rowIdx] = rowID\n addRow = False\n\n # Don't add this vector if it matches closely with another we already\n # added\n if self.distThreshold > 0:\n if dist is None or self.distanceNorm != 1:\n dist = self._calcDistance(thresholdedInput)\n minDist = dist.min()\n addRow = (minDist >= self.distThreshold)\n if not addRow:\n if self.fixedCapacity:\n rowIdx = dist.argmin()\n self._categoryRecencyList[rowIdx] = rowID\n\n\n # If sparsity is too low, we do not want to add this vector\n if addRow and self.minSparsity > 0.0:\n if isSparse==0:\n sparsity = ( float(len(thresholdedInput.nonzero()[0])) /\n len(thresholdedInput) )\n else:\n sparsity = float(len(inputPattern)) / isSparse\n if sparsity < self.minSparsity:\n addRow = False\n\n # Add the new sparse vector to our storage\n if addRow:\n self._protoSizes = None # need to re-compute\n if isSparse == 0:\n self._Memory.addRow(thresholdedInput)\n else:\n self._Memory.addRowNZ(inputPattern, [1]*len(inputPattern))\n self._numPatterns += 1\n self._categoryList.append(int(inputCategory))\n self._addPartitionId(self._numPatterns-1, partitionId)\n if self.fixedCapacity:\n self._categoryRecencyList.append(rowID)\n if self._numPatterns > self.maxStoredPatterns and \\\n self.maxStoredPatterns > 0:\n leastRecentlyUsedPattern = numpy.argmin(self._categoryRecencyList)\n self._Memory.deleteRow(leastRecentlyUsedPattern)\n self._categoryList.pop(leastRecentlyUsedPattern)\n self._categoryRecencyList.pop(leastRecentlyUsedPattern)\n self._numPatterns -= 1\n\n if self.numSVDDims is not None and self.numSVDSamples > 0 \\\n and self._numPatterns == self.numSVDSamples:\n self.computeSVD()\n\n return self._numPatterns", "response": "Train the classifier to associate specified input pattern with specified id."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the degree of overlap between an input pattern and each category", "response": "def getOverlaps(self, inputPattern):\n \"\"\"\n Return the degree of overlap between an input pattern and each category\n stored in the classifier. The overlap is computed by computing:\n\n .. code-block:: python\n\n logical_and(inputPattern != 0, trainingPattern != 0).sum()\n\n :param inputPattern: pattern to check overlap of\n\n :returns: (overlaps, categories) Two numpy arrays of the same length, where:\n \n * overlaps: an integer overlap amount for each category\n * categories: category index for each element of overlaps\n \"\"\"\n assert self.useSparseMemory, \"Not implemented yet for dense storage\"\n\n overlaps = self._Memory.rightVecSumAtNZ(inputPattern)\n return (overlaps, self._categoryList)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getDistances(self, inputPattern):\n dist = self._getDistances(inputPattern)\n return (dist, self._categoryList)", "response": "Returns the distances between the inputPattern and all other available patterns."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef infer(self, inputPattern, computeScores=True, overCategories=True,\n partitionId=None):\n \"\"\"Finds the category that best matches the input pattern. Returns the\n winning category index as well as a distribution over all categories.\n\n :param inputPattern: (list or array) The pattern to be classified. This\n must be a dense representation of the array (e.g. [0, 0, 1, 1, 0, 1]).\n\n :param computeScores: NO EFFECT\n\n :param overCategories: NO EFFECT\n\n :param partitionId: (int) If provided, all training vectors with partitionId\n equal to that of the input pattern are ignored.\n For example, this may be used to perform k-fold cross validation\n without repopulating the classifier. First partition all the data into\n k equal partitions numbered 0, 1, 2, ... and then call learn() for each\n vector passing in its partitionId. Then, during inference, by passing\n in the partition ID in the call to infer(), all other vectors with the\n same partitionId are ignored simulating the effect of repopulating the\n classifier while ommitting the training vectors in the same partition.\n\n :returns: 4-tuple with these keys:\n\n - ``winner``: The category with the greatest number of nearest neighbors\n within the kth nearest neighbors. If the inferenceResult contains no\n neighbors, the value of winner is None. This can happen, for example,\n in cases of exact matching, if there are no stored vectors, or if\n minSparsity is not met.\n - ``inferenceResult``: A list of length numCategories, each entry contains\n the number of neighbors within the top k neighbors that are in that\n category.\n - ``dist``: A list of length numPrototypes. Each entry is the distance\n from the unknown to that prototype. All distances are between 0.0 and\n 1.0.\n - ``categoryDist``: A list of length numCategories. Each entry is the\n distance from the unknown to the nearest prototype of\n that category. All distances are between 0 and 1.0.\n \"\"\"\n\n # Calculate sparsity. If sparsity is too low, we do not want to run\n # inference with this vector\n sparsity = 0.0\n if self.minSparsity > 0.0:\n sparsity = ( float(len(inputPattern.nonzero()[0])) /\n len(inputPattern) )\n\n if len(self._categoryList) == 0 or sparsity < self.minSparsity:\n # No categories learned yet; i.e. first inference w/ online learning or\n # insufficient sparsity\n winner = None\n inferenceResult = numpy.zeros(1)\n dist = numpy.ones(1)\n categoryDist = numpy.ones(1)\n\n else:\n maxCategoryIdx = max(self._categoryList)\n inferenceResult = numpy.zeros(maxCategoryIdx+1)\n dist = self._getDistances(inputPattern, partitionId=partitionId)\n validVectorCount = len(self._categoryList) - self._categoryList.count(-1)\n\n # Loop through the indices of the nearest neighbors.\n if self.exact:\n # Is there an exact match in the distances?\n exactMatches = numpy.where(dist<0.00001)[0]\n if len(exactMatches) > 0:\n for i in exactMatches[:min(self.k, validVectorCount)]:\n inferenceResult[self._categoryList[i]] += 1.0\n else:\n sorted = dist.argsort()\n for j in sorted[:min(self.k, validVectorCount)]:\n inferenceResult[self._categoryList[j]] += 1.0\n\n # Prepare inference results.\n if inferenceResult.any():\n winner = inferenceResult.argmax()\n inferenceResult /= inferenceResult.sum()\n else:\n winner = None\n categoryDist = min_score_per_category(maxCategoryIdx,\n self._categoryList, dist)\n categoryDist.clip(0, 1.0, categoryDist)\n\n if self.verbosity >= 1:\n print \"%s infer:\" % (g_debugPrefix)\n print \" active inputs:\", _labeledInput(inputPattern,\n cellsPerCol=self.cellsPerCol)\n print \" winner category:\", winner\n print \" pct neighbors of each category:\", inferenceResult\n print \" dist of each prototype:\", dist\n print \" dist of each category:\", categoryDist\n\n result = (winner, inferenceResult, dist, categoryDist)\n return result", "response": "Infer the category that best matches the input pattern."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the index of the pattern that is closest to inputPattern the distances of all patterns to inputPattern and the indices of the closest categories.", "response": "def getClosest(self, inputPattern, topKCategories=3):\n \"\"\"Returns the index of the pattern that is closest to inputPattern,\n the distances of all patterns to inputPattern, and the indices of the k\n closest categories.\n \"\"\"\n inferenceResult = numpy.zeros(max(self._categoryList)+1)\n dist = self._getDistances(inputPattern)\n\n sorted = dist.argsort()\n\n validVectorCount = len(self._categoryList) - self._categoryList.count(-1)\n for j in sorted[:min(self.k, validVectorCount)]:\n inferenceResult[self._categoryList[j]] += 1.0\n\n winner = inferenceResult.argmax()\n\n topNCats = []\n for i in range(topKCategories):\n topNCats.append((self._categoryList[sorted[i]], dist[sorted[i]] ))\n\n return winner, dist, topNCats"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef closestTrainingPattern(self, inputPattern, cat):\n dist = self._getDistances(inputPattern)\n sorted = dist.argsort()\n\n for patIdx in sorted:\n patternCat = self._categoryList[patIdx]\n\n # If closest pattern belongs to desired category, return it\n if patternCat == cat:\n if self.useSparseMemory:\n closestPattern = self._Memory.getRow(int(patIdx))\n else:\n closestPattern = self._M[patIdx]\n\n return closestPattern\n\n # No patterns were found!\n return None", "response": "Returns the closest training pattern to inputPattern that belongs to the specified category."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getPattern(self, idx, sparseBinaryForm=False, cat=None):\n if cat is not None:\n assert idx is None\n idx = self._categoryList.index(cat)\n\n if not self.useSparseMemory:\n pattern = self._Memory[idx]\n if sparseBinaryForm:\n pattern = pattern.nonzero()[0]\n\n else:\n (nz, values) = self._Memory.rowNonZeros(idx)\n if not sparseBinaryForm:\n pattern = numpy.zeros(self._Memory.nCols())\n numpy.put(pattern, nz, 1)\n else:\n pattern = nz\n\n return pattern", "response": "Gets a training pattern either by index or category number."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getPartitionId(self, i):\n if (i < 0) or (i >= self._numPatterns):\n raise RuntimeError(\"index out of bounds\")\n partitionId = self._partitionIdList[i]\n if partitionId == numpy.inf:\n return None\n else:\n return partitionId", "response": "Gets the partition id given an index."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _addPartitionId(self, index, partitionId=None):\n if partitionId is None:\n self._partitionIdList.append(numpy.inf)\n else:\n self._partitionIdList.append(partitionId)\n indices = self._partitionIdMap.get(partitionId, [])\n indices.append(index)\n self._partitionIdMap[partitionId] = indices", "response": "Adds partition id for pattern index"}