{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_xid_device(device_number):\n scanner = XidScanner()\n com = scanner.device_at_index(device_number)\n com.open()\n return XidDevice(com)", "response": "Returns the XidDevice object for the device at the passed in index."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef connect(self, receiver):\n if not callable(receiver):\n raise ValueError('Invalid receiver: %s' % receiver)\n self.receivers.append(receiver)", "response": "Append receiver to the list of receivers."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndisconnecting receiver from the list of receivers.", "response": "def disconnect(self, receiver):\n \"\"\"Remove receiver.\"\"\"\n try:\n self.receivers.remove(receiver)\n except ValueError:\n raise ValueError('Unknown receiver: %s' % receiver)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef send(self, instance, *args, **kwargs):\n for receiver in self.receivers:\n receiver(instance, *args, **kwargs)", "response": "Send an instance to all the receivers."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting database from given URI or object.", "response": "def get_database(obj, **params):\n \"\"\"Get database from given URI/Object.\"\"\"\n if isinstance(obj, string_types):\n return connect(obj, **params)\n return obj"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef close(self, response):\n LOGGER.info('Closing [%s]', os.getpid())\n if not self.database.is_closed():\n self.database.close()\n return response", "response": "Close connection to database."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nbind model to self database.", "response": "def Model(self):\n \"\"\"Bind model to self database.\"\"\"\n Model_ = self.app.config['PEEWEE_MODELS_CLASS']\n meta_params = {'database': self.database}\n if self.slaves and self.app.config['PEEWEE_USE_READ_SLAVES']:\n meta_params['read_slaves'] = self.slaves\n\n Meta = type('Meta', (), meta_params)\n return type('Model', (Model_,), {'Meta': Meta})"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning self. application models.", "response": "def models(self):\n \"\"\"Return self.application models.\"\"\"\n Model_ = self.app.config['PEEWEE_MODELS_CLASS']\n ignore = self.app.config['PEEWEE_MODELS_IGNORE']\n\n models = []\n if Model_ is not Model:\n try:\n mod = import_module(self.app.config['PEEWEE_MODELS_MODULE'])\n for model in dir(mod):\n models = getattr(mod, model)\n if not isinstance(model, pw.Model):\n continue\n models.append(models)\n except ImportError:\n return models\n elif isinstance(Model_, BaseSignalModel):\n models = BaseSignalModel.models\n\n return [m for m in models if m._meta.name not in ignore]"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a new migration.", "response": "def cmd_create(self, name, auto=False):\n \"\"\"Create a new migration.\"\"\"\n\n LOGGER.setLevel('INFO')\n LOGGER.propagate = 0\n\n router = Router(self.database,\n migrate_dir=self.app.config['PEEWEE_MIGRATE_DIR'],\n migrate_table=self.app.config['PEEWEE_MIGRATE_TABLE'])\n\n if auto:\n auto = self.models\n\n router.create(name, auto=auto)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nintegrate a Flask - Script.", "response": "def manager(self):\n \"\"\"Integrate a Flask-Script.\"\"\"\n from flask_script import Manager, Command\n\n manager = Manager(usage=\"Migrate database.\")\n manager.add_command('create', Command(self.cmd_create))\n manager.add_command('migrate', Command(self.cmd_migrate))\n manager.add_command('rollback', Command(self.cmd_rollback))\n manager.add_command('list', Command(self.cmd_list))\n manager.add_command('merge', Command(self.cmd_merge))\n\n return manager"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef restart_program():\n\n python = sys.executable\n os.execl(python, python, * sys.argv)", "response": "Restarts the current program with the current sequence of python modules."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __load(arff):\n attrs = arff['attributes']\n attrs_t = []\n for attr in attrs:\n if isinstance(attr[1], list):\n attrs_t.append(\"%s@{%s}\" % (attr[0], ','.join(attr[1])))\n else:\n attrs_t.append(\"%s@%s\" % (attr[0], attr[1]))\n\n df = pd.DataFrame(data=arff['data'], columns=attrs_t)\n return df", "response": "loads liac - arff to pandas DataFrame"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __dump(df,relation='data',description=''):\n attrs = []\n for col in df.columns:\n attr = col.split('@')\n if attr[1].count('{')>0 and attr[1].count('}')>0:\n vals = attr[1].replace('{','').replace('}','').split(',')\n attrs.append((attr[0],vals))\n else:\n attrs.append((attr[0],attr[1]))\n\n data = list(df.values)\n result = {\n 'attributes':attrs,\n 'data':data,\n 'description':description,\n 'relation':relation\n }\n return result", "response": "Dump a DataFrame to liac - arff dict"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dump(df,fp):\n arff = __dump(df)\n liacarff.dump(arff,fp)", "response": "Dump a DataFrame to a file in a Sequential fashion."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninserts marker at offset into text and return the marked line.", "response": "def markup_line(text, offset, marker='>>!<<'):\n \"\"\"Insert `marker` at `offset` into `text`, and return the marked\n line.\n\n .. code-block:: python\n\n >>> markup_line('0\\\\n1234\\\\n56', 3)\n 1>>!<<234\n\n \"\"\"\n\n begin = text.rfind('\\n', 0, offset)\n begin += 1\n\n end = text.find('\\n', offset)\n\n if end == -1:\n end = len(text)\n\n return text[begin:offset] + marker + text[offset:end]"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing a tokenizer. Should only be called by the :func:`~textparser.Parser.tokenize` method in the parser.", "response": "def tokenize_init(spec):\n \"\"\"Initialize a tokenizer. Should only be called by the\n :func:`~textparser.Parser.tokenize` method in the parser.\n\n \"\"\"\n\n tokens = [Token('__SOF__', '__SOF__', 0)]\n re_token = '|'.join([\n '(?P<{}>{})'.format(name, regex) for name, regex in spec\n ])\n\n return tokens, re_token"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef tokenize(self, text):\n\n names, specs = self._unpack_token_specs()\n keywords = self.keywords()\n tokens, re_token = tokenize_init(specs)\n\n for mo in re.finditer(re_token, text, re.DOTALL):\n kind = mo.lastgroup\n\n if kind == 'SKIP':\n pass\n elif kind != 'MISMATCH':\n value = mo.group(kind)\n\n if value in keywords:\n kind = value\n\n if kind in names:\n kind = names[kind]\n\n tokens.append(Token(kind, value, mo.start()))\n else:\n raise TokenizeError(text, mo.start())\n\n return tokens", "response": "Tokenize given string text and return a list of tokens. Raises TokenizeError on failure."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse given string text and return the parse tree.", "response": "def parse(self, text, token_tree=False, match_sof=False):\n \"\"\"Parse given string `text` and return the parse tree. Raises\n :class:`~textparser.ParseError` on failure.\n\n Returns a parse tree of tokens if `token_tree` is ``True``.\n\n .. code-block:: python\n\n >>> MyParser().parse('Hello, World!')\n ['Hello', ',', 'World', '!']\n >>> tree = MyParser().parse('Hello, World!', token_tree=True)\n >>> from pprint import pprint\n >>> pprint(tree)\n [Token(kind='WORD', value='Hello', offset=0),\n Token(kind=',', value=',', offset=5),\n Token(kind='WORD', value='World', offset=7),\n Token(kind='!', value='!', offset=12)]\n\n \"\"\"\n\n try:\n tokens = self.tokenize(text)\n\n if len(tokens) == 0 or tokens[-1].kind != '__EOF__':\n tokens.append(Token('__EOF__', '__EOF__', len(text)))\n\n if not match_sof:\n if len(tokens) > 0 and tokens[0].kind == '__SOF__':\n del tokens[0]\n\n return Grammar(self.grammar()).parse(tokens, token_tree)\n except (TokenizeError, GrammarError) as e:\n raise ParseError(text, e.offset)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef x_at_y(self, y, reverse=False):\n logger.info('Running %(name)s.y_at_x(y=%(y)s, reverse=%(rev)s)',\n {\"name\": self.__class__, \"y\": y, \"rev\": reverse})\n # positive or negative direction handles\n x_handle, y_handle = self.x, self.y\n if reverse:\n x_handle, y_handle = self.x[::-1], self.y[::-1]\n\n # find the index of first value in self.y greater or equal than y\n cond = y_handle >= y\n ind = np.argmax(cond)\n\n # two boundary conditions where x cannot be found:\n # A) y > max(self.y)\n # B) y < min(self.y)\n\n # A) if y > max(self.y) then condition self.y >= y\n # will never be satisfied\n # np.argmax( cond ) will be equal 0 and cond[ind] will be False\n if not cond[ind]:\n return np.nan\n\n # B) if y < min(self.y) then condition self.y >= y\n # will be satisfied on first item\n # np.argmax(cond) will be equal 0,\n # to exclude situation that y_handle[0] = y\n # we also check if y < y_handle[0]\n if ind == 0 and y < y_handle[0]:\n return np.nan\n\n # use lookup if y in self.y:\n if cond[ind] and y_handle[ind] == y:\n return x_handle[ind]\n\n # alternatively - pure python implementation\n # return x_handle[ind] - \\\n # ((x_handle[ind] - x_handle[ind - 1]) / \\\n # (y_handle[ind] - y_handle[ind - 1])) * \\\n # (y_handle[ind] - y)\n\n # use interpolation\n sl = slice(ind - 1, ind + 1)\n return np.interp(y, y_handle[sl], x_handle[sl])", "response": "Calculates inverse profile of x at given y"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef width(self, level):\n return self.x_at_y(level, reverse=True) - self.x_at_y(level)", "response": "Returns the width of the log entry at a given level."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nnormalize to 1 over - dt area.", "response": "def normalize(self, dt, allow_cast=True):\n \"\"\"\n Normalize to 1 over [-dt, +dt] area, if allow_cast is set\n to True, division not in place and casting may occur.\n If division in place is not possible and allow_cast is False\n an exception is raised.\n\n >>> a = Profile([[0, 0], [1, 5], [2, 10], [3, 5], [4, 0]])\n >>> a.normalize(1, allow_cast=True)\n >>> print(a.y)\n [0. 2. 4. 2. 0.]\n\n :param dt:\n :param allow_cast:\n \"\"\"\n if dt <= 0:\n raise ValueError(\"Expected positive input\")\n logger.info('Running %(name)s.normalize(dt=%(dt)s)', {\"name\": self.__class__, \"dt\": dt})\n try:\n ave = np.average(self.y[np.fabs(self.x) <= dt])\n except RuntimeWarning as e:\n logger.error('in normalize(). self class is %(name)s, dt=%(dt)s', {\"name\": self.__class__, \"dt\": dt})\n raise Exception(\"Scaling factor error: {0}\".format(e))\n try:\n self.y /= ave\n except TypeError as e:\n logger.warning(\"Division in place is impossible: %s\", e)\n if allow_cast:\n self.y = self.y / ave\n else:\n logger.error(\"Division in place impossible - allow_cast flag set to True should help\")\n raise"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef rescale(self, factor=1.0, allow_cast=True):\n try:\n self.y /= factor\n except TypeError as e:\n logger.warning(\"Division in place is impossible: %s\", e)\n if allow_cast:\n self.y = self.y / factor\n else:\n logger.error(\"allow_cast flag set to True should help\")\n raise", "response": "Rescales self. y by given factor."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef change_domain(self, domain):\n logger.info('Running %(name)s.change_domain() with new domain range:[%(ymin)s, %(ymax)s]',\n {\"name\": self.__class__, \"ymin\": np.min(domain), \"ymax\": np.max(domain)})\n\n # check if new domain includes in the original domain\n if np.max(domain) > np.max(self.x) or np.min(domain) < np.min(self.x):\n logger.error('Old domain range: [%(xmin)s, %(xmax)s] does not include new domain range:'\n '[%(ymin)s, %(ymax)s]', {\"xmin\": np.min(self.x), \"xmax\": np.max(self.x),\n \"ymin\": np.min(domain), \"ymax\": np.max(domain)})\n raise ValueError('in change_domain():' 'the old domain does not include the new one')\n\n y = np.interp(domain, self.x, self.y)\n # We need to join together domain and values (y) because we are recreating Curve object\n # (we pass it as argument to self.__class__)\n # np.dstack((arrays), axis=1) joins given arrays like np.dstack() but it also nests the result\n # in additional list and this is the reason why we use [0] to remove this extra layer of list like this:\n # np.dstack([[0, 5, 10], [0, 0, 0]]) gives [[[ 0, 0], [ 5, 0], [10, 0]]] so use dtack()[0]\n # to get this: [[0,0], [5, 5], [10, 0]]\n # which is a 2 dimensional array and can be used to create a new Curve object\n obj = self.__class__(np.dstack((domain, y))[0], **self.__dict__['metadata'])\n return obj", "response": "Creates a new Curve object with new domain and returns it."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef rebinned(self, step=0.1, fixp=0):\n logger.info('Running %(name)s.rebinned(step=%(st)s, fixp=%(fx)s)',\n {\"name\": self.__class__, \"st\": step, \"fx\": fixp})\n a, b = (np.min(self.x), np.max(self.x))\n count_start = abs(fixp - a) / step\n count_stop = abs(fixp - b) / step\n\n # depending on position of fixp with respect to the original domain\n # 3 cases may occur:\n if fixp < a:\n count_start = math.ceil(count_start)\n count_stop = math.floor(count_stop)\n elif fixp > b:\n count_start = -math.floor(count_start)\n count_stop = -math.ceil(count_stop)\n else:\n count_start = -count_start\n count_stop = count_stop\n\n domain = [fixp + n * step for n in range(int(count_start), int(count_stop) + 1)]\n return self.change_domain(domain)", "response": "Returns a new curve object with the domain of the new domain."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nevaluating at x - value", "response": "def evaluate_at_x(self, arg, def_val=0):\n \"\"\"\n Returns Y value at arg of self. Arg can be a scalar,\n but also might be np.array or other iterable\n (like list). If domain of self is not wide enough to\n interpolate the value of Y, method will return\n def_val for those arguments instead.\n\n Check the interpolation when arg in domain of self:\n >>> Curve([[0, 0], [2, 2], [4, 4]]).evaluate_at_x([1, 2 ,3])\n array([1., 2., 3.])\n\n Check if behavior of the method is correct when arg\n id partly outside the domain:\n >>> Curve([[0, 0], [2, 2], [4, 4]]).evaluate_at_x(\\\n [-1, 1, 2 ,3, 5], 100)\n array([100., 1., 2., 3., 100.])\n\n :param arg: x-value to calculate Y (may be an array or list as well)\n :param def_val: default value to return if can't interpolate at arg\n :return: np.array of Y-values at arg. If arg is a scalar,\n will return scalar as well\n \"\"\"\n y = np.interp(arg, self.x, self.y, left=def_val, right=def_val)\n return y"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef alert(text='', title='', button='OK'):\n messageBoxFunc(0, text, title, MB_OK | MB_SETFOREGROUND | MB_TOPMOST)\n return button", "response": "Displays a simple message box with text and a single OK button. Returns the text of the button clicked on."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef confirm(text='', title='', buttons=['OK', 'Cancel']):\n retVal = messageBoxFunc(0, text, title, MB_OKCANCEL | MB_ICONQUESTION | MB_SETFOREGROUND | MB_TOPMOST)\n if retVal == 1 or len(buttons) == 1:\n return buttons[0]\n elif retVal == 2:\n return buttons[1]\n else:\n assert False, 'Unexpected return value from MessageBox: %s' % (retVal)", "response": "Displays a message box with OK and Cancel buttons. Number and text of buttons can be customized. Returns the text of the button clicked on."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfunction calculates difference between two curves and returns new object which domain is an union of curve1 and curve2 domains", "response": "def subtract(curve1, curve2, def_val=0):\n \"\"\"\n Function calculates difference between curve1 and curve2\n and returns new object which domain is an union\n of curve1 and curve2 domains\n Returned object is of type type(curve1)\n and has same metadata as curve1 object\n\n :param curve1: first curve to calculate the difference\n :param curve2: second curve to calculate the difference\n :param def_val: default value for points that cannot be interpolated\n :return: new object of type type(curve1) with element-wise difference\n (using interpolation if necessary)\n \"\"\"\n coord1 = np.union1d(curve1.x, curve2.x)\n y1 = curve1.evaluate_at_x(coord1, def_val)\n y2 = curve2.evaluate_at_x(coord1, def_val)\n coord2 = y1 - y2\n # the below is explained at the end of curve.Curve.change_domain()\n obj = curve1.__class__(np.dstack((coord1, coord2))[0], **curve1.__dict__['metadata'])\n return obj"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\napplying a window - length median filter to a 1D array vector.", "response": "def medfilt(vector, window):\n \"\"\"\n Apply a window-length median filter to a 1D array vector.\n\n Should get rid of 'spike' value 15.\n >>> print(medfilt(np.array([1., 15., 1., 1., 1.]), 3))\n [1. 1. 1. 1. 1.]\n\n The 'edge' case is a bit tricky...\n >>> print(medfilt(np.array([15., 1., 1., 1., 1.]), 3))\n [15. 1. 1. 1. 1.]\n\n Inspired by: https://gist.github.com/bhawkins/3535131\n \"\"\"\n if not window % 2 == 1:\n raise ValueError(\"Median filter length must be odd.\")\n if not vector.ndim == 1:\n raise ValueError(\"Input must be one-dimensional.\")\n\n k = (window - 1) // 2 # window movement\n result = np.zeros((len(vector), window), dtype=vector.dtype)\n result[:, k] = vector\n for i in range(k):\n j = k - i\n result[j:, i] = vector[:-j]\n result[:j, i] = vector[0]\n result[:-j, -(i + 1)] = vector[j:]\n result[-j:, -(i + 1)] = vector[-1]\n\n return np.median(result, axis=1)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef interpn(*args, **kw):\n\tmethod = kw.pop('method', 'cubic')\n\tif kw:\n\t\traise ValueError(\"Unknown arguments: \" % kw.keys())\n\tnd = (len(args)-1)//2\n\tif len(args) != 2*nd+1:\n\t\traise ValueError(\"Wrong number of arguments\")\n\tq = args[:nd]\n\tqi = args[nd+1:]\n\ta = args[nd]\n\tfor j in range(nd):\n\t\t#print q[j].shape, a.shape\n\t\ta = interp1d(q[j], a, axis=j, kind=method)(qi[j])\n\treturn a", "response": "Interpolate on N - D. \n"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninterpolates on N - D.", "response": "def npinterpn(*args, **kw):\n\t\"\"\"Interpolation on N-D. \n\n\tai = interpn(x, y, z, ..., a, xi, yi, zi, ...)\n\twhere the arrays x, y, z, ... define a rectangular grid\n\tand a.shape == (len(x), len(y), len(z), ...) are the values\n\tinterpolate at xi, yi, zi, ...\n\t\"\"\"\n\tmethod = kw.pop('method', 'cubic')\n\tif kw:\n\t\traise ValueError(\"Unknown arguments: \" % kw.keys())\n\tnd = (len(args)-1)//2\n\tif len(args) != 2*nd+1:\n\t\traise ValueError(\"Wrong number of arguments\")\n\tq = args[:nd]\n\tqi = args[nd+1:]\n\ta = args[nd]\n\tfor j in range(nd):\n\t\t#print q[j].shape, a.shape\n\t\ta = interp(q[j], a, axis=j, kind=method)(qi[j])\n\treturn a"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngenerate a dictionary of fields for a given Peewee model.", "response": "def model_fields(model, allow_pk=False, only=None, exclude=None,\n field_args=None, converter=None):\n \"\"\"\n Generate a dictionary of fields for a given Peewee model.\n\n See `model_form` docstring for description of parameters.\n \"\"\"\n converter = converter or ModelConverter()\n field_args = field_args or {}\n\n model_fields = list(model._meta.sorted_fields)\n if not allow_pk:\n model_fields.pop(0)\n\n if only:\n model_fields = [x for x in model_fields if x.name in only]\n elif exclude:\n model_fields = [x for x in model_fields if x.name not in exclude]\n\n field_dict = {}\n for model_field in model_fields:\n name, field = converter.convert(\n model,\n model_field,\n field_args.get(model_field.name))\n field_dict[name] = field\n\n return field_dict"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef model_form(model, base_class=Form, allow_pk=False, only=None, exclude=None,\n field_args=None, converter=None):\n \"\"\"\n Create a wtforms Form for a given Peewee model class::\n\n from wtfpeewee.orm import model_form\n from myproject.myapp.models import User\n UserForm = model_form(User)\n\n :param model:\n A Peewee model class\n :param base_class:\n Base form class to extend from. Must be a ``wtforms.Form`` subclass.\n :param only:\n An optional iterable with the property names that should be included in\n the form. Only these properties will have fields.\n :param exclude:\n An optional iterable with the property names that should be excluded\n from the form. All other properties will have fields.\n :param field_args:\n An optional dictionary of field names mapping to keyword arguments used\n to construct each field object.\n :param converter:\n A converter to generate the fields based on the model properties. If\n not set, ``ModelConverter`` is used.\n \"\"\"\n field_dict = model_fields(model, allow_pk, only, exclude, field_args, converter)\n return type(model.__name__ + 'Form', (base_class, ), field_dict)", "response": "Returns a new form class for a given Peewee model class."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef alert(text='', title='', button=OK_TEXT, root=None, timeout=None):\n assert TKINTER_IMPORT_SUCCEEDED, 'Tkinter is required for pymsgbox'\n return _buttonbox(msg=text, title=title, choices=[str(button)], root=root, timeout=timeout)", "response": "Displays a simple message box with text and a single OK button. Returns the text of the button clicked on."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef confirm(text='', title='', buttons=[OK_TEXT, CANCEL_TEXT], root=None, timeout=None):\n assert TKINTER_IMPORT_SUCCEEDED, 'Tkinter is required for pymsgbox'\n return _buttonbox(msg=text, title=title, choices=[str(b) for b in buttons], root=root, timeout=timeout)", "response": "Displays a message box with OK and CANCEL buttons. Number and text of buttons can be customized. Returns the text of the button clicked on."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef prompt(text='', title='' , default='', root=None, timeout=None):\n assert TKINTER_IMPORT_SUCCEEDED, 'Tkinter is required for pymsgbox'\n return __fillablebox(text, title, default=default, mask=None,root=root, timeout=timeout)", "response": "Displays a message box with text input and OK & Cancel buttons. Returns the text entered or None."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndisplay a message a title and a set of buttons.", "response": "def _buttonbox(msg, title, choices, root=None, timeout=None):\n \"\"\"\n Display a msg, a title, and a set of buttons.\n The buttons are defined by the members of the choices list.\n Return the text of the button that the user selected.\n\n @arg msg: the msg to be displayed.\n @arg title: the window title\n @arg choices: a list or tuple of the choices to be displayed\n \"\"\"\n global boxRoot, __replyButtonText, __widgetTexts, buttonsFrame\n\n\n # Initialize __replyButtonText to the first choice.\n # This is what will be used if the window is closed by the close button.\n __replyButtonText = choices[0]\n\n if root:\n root.withdraw()\n boxRoot = tk.Toplevel(master=root)\n boxRoot.withdraw()\n else:\n boxRoot = tk.Tk()\n boxRoot.withdraw()\n\n boxRoot.title(title)\n boxRoot.iconname('Dialog')\n boxRoot.geometry(rootWindowPosition)\n boxRoot.minsize(400, 100)\n\n # ------------- define the messageFrame ---------------------------------\n messageFrame = tk.Frame(master=boxRoot)\n messageFrame.pack(side=tk.TOP, fill=tk.BOTH)\n\n # ------------- define the buttonsFrame ---------------------------------\n buttonsFrame = tk.Frame(master=boxRoot)\n buttonsFrame.pack(side=tk.TOP, fill=tk.BOTH)\n\n # -------------------- place the widgets in the frames -----------------------\n messageWidget = tk.Message(messageFrame, text=msg, width=400)\n messageWidget.configure(font=(PROPORTIONAL_FONT_FAMILY, PROPORTIONAL_FONT_SIZE))\n messageWidget.pack(side=tk.TOP, expand=tk.YES, fill=tk.X, padx='3m', pady='3m')\n\n __put_buttons_in_buttonframe(choices)\n\n # -------------- the action begins -----------\n # put the focus on the first button\n __firstWidget.focus_force()\n\n boxRoot.deiconify()\n if timeout is not None:\n boxRoot.after(timeout, timeoutBoxRoot)\n boxRoot.mainloop()\n try:\n boxRoot.destroy()\n except tk.TclError:\n if __replyButtonText != TIMEOUT_TEXT:\n __replyButtonText = None\n\n if root: root.deiconify()\n return __replyButtonText"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __put_buttons_in_buttonframe(choices):\n global __widgetTexts, __firstWidget, buttonsFrame\n\n __firstWidget = None\n __widgetTexts = {}\n\n i = 0\n\n for buttonText in choices:\n tempButton = tk.Button(buttonsFrame, takefocus=1, text=buttonText)\n _bindArrows(tempButton)\n tempButton.pack(expand=tk.YES, side=tk.LEFT, padx='1m', pady='1m', ipadx='2m', ipady='1m')\n\n # remember the text associated with this widget\n __widgetTexts[tempButton] = buttonText\n\n # remember the first widget, so we can put the focus there\n if i == 0:\n __firstWidget = tempButton\n i = 1\n\n # for the commandButton, bind activation events to the activation event handler\n commandButton = tempButton\n handler = __buttonEvent\n for selectionEvent in STANDARD_SELECTION_EVENTS:\n commandButton.bind('<%s>' % selectionEvent, handler)\n\n if CANCEL_TEXT in choices:\n commandButton.bind('', __cancelButtonEvent)", "response": "Put the buttons in the buttons frame"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfilling a box with a message.", "response": "def __fillablebox(msg, title='', default='', mask=None, root=None, timeout=None):\n \"\"\"\n Show a box in which a user can enter some text.\n You may optionally specify some default text, which will appear in the\n enterbox when it is displayed.\n Returns the text that the user entered, or None if he cancels the operation.\n \"\"\"\n\n global boxRoot, __enterboxText, __enterboxDefaultText\n global cancelButton, entryWidget, okButton\n\n if title == None:\n title == ''\n if default == None:\n default = ''\n __enterboxDefaultText = default\n __enterboxText = __enterboxDefaultText\n\n if root:\n root.withdraw()\n boxRoot = tk.Toplevel(master=root)\n boxRoot.withdraw()\n else:\n boxRoot = tk.Tk()\n boxRoot.withdraw()\n\n boxRoot.title(title)\n boxRoot.iconname('Dialog')\n boxRoot.geometry(rootWindowPosition)\n boxRoot.bind('', __enterboxCancel)\n\n # ------------- define the messageFrame ---------------------------------\n messageFrame = tk.Frame(master=boxRoot)\n messageFrame.pack(side=tk.TOP, fill=tk.BOTH)\n\n # ------------- define the buttonsFrame ---------------------------------\n buttonsFrame = tk.Frame(master=boxRoot)\n buttonsFrame.pack(side=tk.TOP, fill=tk.BOTH)\n\n\n # ------------- define the entryFrame ---------------------------------\n entryFrame = tk.Frame(master=boxRoot)\n entryFrame.pack(side=tk.TOP, fill=tk.BOTH)\n\n # ------------- define the buttonsFrame ---------------------------------\n buttonsFrame = tk.Frame(master=boxRoot)\n buttonsFrame.pack(side=tk.TOP, fill=tk.BOTH)\n\n #-------------------- the msg widget ----------------------------\n messageWidget = tk.Message(messageFrame, width='4.5i', text=msg)\n messageWidget.configure(font=(PROPORTIONAL_FONT_FAMILY, PROPORTIONAL_FONT_SIZE))\n messageWidget.pack(side=tk.RIGHT, expand=1, fill=tk.BOTH, padx='3m', pady='3m')\n\n # --------- entryWidget ----------------------------------------------\n entryWidget = tk.Entry(entryFrame, width=40)\n _bindArrows(entryWidget, skipArrowKeys=True)\n entryWidget.configure(font=(PROPORTIONAL_FONT_FAMILY, TEXT_ENTRY_FONT_SIZE))\n if mask:\n entryWidget.configure(show=mask)\n entryWidget.pack(side=tk.LEFT, padx='3m')\n entryWidget.bind('', __enterboxGetText)\n entryWidget.bind('', __enterboxCancel)\n\n # put text into the entryWidget and have it pre-highlighted\n if __enterboxDefaultText != '':\n entryWidget.insert(0,__enterboxDefaultText)\n entryWidget.select_range(0, tk.END)\n\n # ------------------ ok button -------------------------------\n okButton = tk.Button(buttonsFrame, takefocus=1, text=OK_TEXT)\n _bindArrows(okButton)\n okButton.pack(expand=1, side=tk.LEFT, padx='3m', pady='3m', ipadx='2m', ipady='1m')\n\n # for the commandButton, bind activation events to the activation event handler\n commandButton = okButton\n handler = __enterboxGetText\n for selectionEvent in STANDARD_SELECTION_EVENTS:\n commandButton.bind('<%s>' % selectionEvent, handler)\n\n\n # ------------------ cancel button -------------------------------\n cancelButton = tk.Button(buttonsFrame, takefocus=1, text=CANCEL_TEXT)\n _bindArrows(cancelButton)\n cancelButton.pack(expand=1, side=tk.RIGHT, padx='3m', pady='3m', ipadx='2m', ipady='1m')\n\n # for the commandButton, bind activation events to the activation event handler\n commandButton = cancelButton\n handler = __enterboxCancel\n for selectionEvent in STANDARD_SELECTION_EVENTS:\n commandButton.bind('<%s>' % selectionEvent, handler)\n\n # ------------------- time for action! -----------------\n entryWidget.focus_force() # put the focus on the entryWidget\n boxRoot.deiconify()\n if timeout is not None:\n boxRoot.after(timeout, timeoutBoxRoot)\n boxRoot.mainloop() # run it!\n\n # -------- after the run has completed ----------------------------------\n if root: root.deiconify()\n try:\n boxRoot.destroy() # button_click didn't destroy boxRoot, so we do it now\n except tk.TclError:\n if __enterboxText != TIMEOUT_TEXT:\n return None\n\n return __enterboxText"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse the login xml response and populate the internal attributes.", "response": "def parse(self, response):\n \"\"\"\n Parse the login xml response\n :param response: the login response from the RETS server\n :return: None\n \"\"\"\n self.headers = response.headers\n\n if 'xml' in self.headers.get('Content-Type'):\n # Got an XML response, likely an error code.\n xml = xmltodict.parse(response.text)\n self.analyze_reply_code(xml_response_dict=xml)\n\n lines = response.text.split('\\r\\n')\n if len(lines) < 3:\n lines = response.text.split('\\n')\n\n for line in lines:\n line = line.strip()\n\n name, value = self.read_line(line)\n if name:\n if name in self.valid_transactions or re.match(pattern='/^X\\-/', string=name):\n self.capabilities[name] = value\n else:\n self.details[name] = value"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads lines of XML and delimits strips and returns.", "response": "def read_line(line):\n \"\"\"Reads lines of XML and delimits, strips, and returns.\"\"\"\n name, value = '', ''\n\n if '=' in line:\n name, value = line.split('=', 1)\n\n return [name.strip(), value.strip()]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef generator(self, response):\n\n delim = '\\t' # Default to tab delimited\n columns = []\n response.raw.decode_content = True\n events = ET.iterparse(BytesIO(response.content))\n\n results = []\n for event, elem in events:\n # Analyze search record data\n if \"DATA\" == elem.tag:\n data_dict = {column: data for column, data in zip(columns, elem.text.split(delim)) if column != ''}\n self.parsed_rows += 1 # Rows parsed with all requests\n results.append(data_dict)\n\n # Handle reply code\n elif \"RETS\" == elem.tag:\n reply_code = elem.get('ReplyCode')\n reply_text = elem.get('ReplyText')\n\n if reply_code == '20201':\n # RETS Response 20201 - No Records Found\n # Generator should continue and return nothing\n continue\n elif reply_code != '0':\n msg = \"RETS Error {0!s}: {1!s}\".format(reply_code, reply_text)\n raise RETSException(msg)\n\n # Analyze delimiter\n elif \"DELIMITER\" == elem.tag:\n val = elem.get(\"value\")\n delim = chr(int(val))\n\n # Analyze columns\n elif \"COLUMNS\" == elem.tag:\n columns = elem.text.split(delim)\n\n # handle max rows\n elif \"MAXROWS\" == elem.tag:\n logger.debug(\"MAXROWS Tag reached in XML\")\n logger.debug(\"Received {0!s} results from this search\".format(self.parsed_rows))\n raise MaxrowException(results)\n\n else:\n # This is a tag we don't process (like COUNT)\n continue\n\n elem.clear()\n\n return results", "response": "Takes a Requests response socket connection and iteratively parses and yields the results as python dictionaries."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntaking a urlinfo object and returns a flat dictionary.", "response": "def flatten_urlinfo(urlinfo, shorter_keys=True):\n \"\"\" Takes a urlinfo object and returns a flat dictionary.\"\"\"\n def flatten(value, prefix=\"\"):\n if is_string(value):\n _result[prefix[1:]] = value\n return\n try:\n len(value)\n except (AttributeError, TypeError): # a leaf\n _result[prefix[1:]] = value\n return\n\n try:\n items = value.items()\n except AttributeError: # an iterable, but not a dict\n last_prefix = prefix.split(\".\")[-1]\n if shorter_keys:\n prefix = \".\" + last_prefix\n\n if last_prefix == \"Country\":\n for v in value:\n country = v.pop(\"@Code\")\n flatten(v, \".\".join([prefix, country]))\n elif last_prefix in [\"RelatedLink\", \"CategoryData\"]:\n for i, v in enumerate(value):\n flatten(v, \".\".join([prefix, str(i)]))\n elif value[0].get(\"TimeRange\"):\n for v in value:\n time_range = \".\".join(tuple(v.pop(\"TimeRange\").items())[0])\n # python 3 odict_items don't support indexing\n if v.get(\"DataUrl\"):\n time_range = \".\".join([v.pop(\"DataUrl\"), time_range])\n flatten(v, \".\".join([prefix, time_range]))\n else:\n msg = prefix + \" contains a list we don't know how to flatten.\"\n raise NotImplementedError(msg)\n else: # a dict, go one level deeper\n for k, v in items:\n flatten(v, \".\".join([prefix, k]))\n\n _result = {}\n info = xmltodict.parse(str(urlinfo))\n flatten(info[\"aws:UrlInfoResponse\"][\"Response\"][\"UrlInfoResult\"][\"Alexa\"])\n _result[\"OutputTimestamp\"] = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')\n return _result"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a V4 signature for the given request parameters.", "response": "def create_v4_signature(self, request_params):\n '''\n Create URI and signature headers based on AWS V4 signing process.\n Refer to https://docs.aws.amazon.com/AlexaWebInfoService/latest/ApiReferenceArticle.html for request params.\n :param request_params: dictionary of request parameters\n :return: URL and header to be passed to requests.get\n '''\n\n method = 'GET'\n service = 'awis'\n host = 'awis.us-west-1.amazonaws.com'\n region = 'us-west-1'\n endpoint = 'https://awis.amazonaws.com/api'\n request_parameters = urlencode([(key, request_params[key]) for key in sorted(request_params.keys())])\n\n # Key derivation functions. See:\n # http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python\n def sign(key, msg):\n return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()\n\n def getSignatureKey(key, dateStamp, regionName, serviceName):\n kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)\n kRegion = sign(kDate, regionName)\n kService = sign(kRegion, serviceName)\n kSigning = sign(kService, 'aws4_request')\n return kSigning\n\n # Create a date for headers and the credential string\n t = datetime.datetime.utcnow()\n amzdate = t.strftime('%Y%m%dT%H%M%SZ')\n datestamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope\n\n # Create canonical request\n canonical_uri = '/api'\n canonical_querystring = request_parameters\n canonical_headers = 'host:' + host + '\\n' + 'x-amz-date:' + amzdate + '\\n'\n signed_headers = 'host;x-amz-date'\n payload_hash = hashlib.sha256(''.encode('utf8')).hexdigest()\n canonical_request = method + '\\n' + canonical_uri + '\\n' + canonical_querystring + '\\n' + canonical_headers + '\\n' + signed_headers + '\\n' + payload_hash\n\n # Create string to sign\n algorithm = 'AWS4-HMAC-SHA256'\n credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'\n string_to_sign = algorithm + '\\n' + amzdate + '\\n' + credential_scope + '\\n' + hashlib.sha256(canonical_request.encode('utf8')).hexdigest()\n\n # Calculate signature\n signing_key = getSignatureKey(self.secret_access_key, datestamp, region, service)\n\n # Sign the string_to_sign using the signing_key\n signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()\n\n # Add signing information to the request\n authorization_header = algorithm + ' ' + 'Credential=' + self.access_id + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature\n headers = {'X-Amz-Date':amzdate, 'Authorization':authorization_header, 'Content-Type': 'application/xml', 'Accept': 'application/xml'}\n\n # Create request url\n request_url = endpoint + '?' + canonical_querystring\n\n return request_url, headers"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef urlinfo(self, domain, response_group = URLINFO_RESPONSE_GROUPS):\n '''\n Provide information about supplied domain as specified by the response group\n :param domain: Any valid URL\n :param response_group: Any valid urlinfo response group\n :return: Traffic and/or content data of the domain in XML format\n '''\n params = {\n 'Action': \"UrlInfo\",\n 'Url': domain,\n 'ResponseGroup': response_group\n }\n\n url, headers = self.create_v4_signature(params)\n return self.return_output(url, headers)", "response": "This method provides information about the supplied domain in XML format."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef traffichistory(self, domain, response_group=TRAFFICINFO_RESPONSE_GROUPS, myrange=31, start=20070801):\n '''\n Provide traffic history of supplied domain\n :param domain: Any valid URL\n :param response_group: Any valid traffic history response group\n :return: Traffic and/or content data of the domain in XML format\n '''\n params = {\n 'Action': \"TrafficHistory\",\n 'Url': domain,\n 'ResponseGroup': response_group,\n 'Range': myrange,\n 'Start': start,\n }\n\n url, headers = self.create_v4_signature(params)\n return self.return_output(url, headers)", "response": "This method provides a traffichistory of the supplied domain."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nproviding category browse information of specified domain :param domain: Any valid URL :param path: Valid category path :param response_group: Any valid traffic history response group :return: Traffic and/or content data of the domain in XML format", "response": "def cat_browse(self, domain, path, response_group=CATEGORYBROWSE_RESPONSE_GROUPS, descriptions='True'):\n '''\n Provide category browse information of specified domain\n :param domain: Any valid URL\n :param path: Valid category path\n :param response_group: Any valid traffic history response group\n :return: Traffic and/or content data of the domain in XML format\n '''\n params = {\n 'Action': \"CategoryListings\",\n 'ResponseGroup': 'Listings',\n 'Path': quote(path),\n 'Descriptions': descriptions\n }\n\n url, headers = self.create_v4_signature(params)\n return self.return_output(url, headers)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding a capability to the capabilities dictionary.", "response": "def add_capability(self, name, uri):\n \"\"\"\n Add a capability of the RETS board\n :param name: The name of the capability\n :param uri: The capability URI given by the RETS board\n :return: None\n \"\"\"\n\n parse_results = urlparse(uri)\n if parse_results.hostname is None:\n # relative URL given, so build this into an absolute URL\n login_url = self.capabilities.get('Login')\n if not login_url:\n logger.error(\"There is no login URL stored, so additional capabilities cannot be added.\")\n raise ValueError(\"Cannot automatically determine absolute path for {0!s} given.\".format(uri))\n\n parts = urlparse(login_url)\n port = ':{}'.format(parts.port) if parts.port else ''\n uri = parts.scheme + '://' + parts.hostname + port + '/' + uri.lstrip('/')\n\n self.capabilities[name] = uri"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef login(self):\n response = self._request('Login')\n parser = OneXLogin()\n parser.parse(response)\n\n self.session_id = response.cookies.get(self.session_id_cookie_name, '')\n\n if parser.headers.get('RETS-Version') is not None:\n self.version = str(parser.headers.get('RETS-Version'))\n self.client.headers['RETS-Version'] = self.version\n\n for k, v in parser.capabilities.items():\n self.add_capability(k, v)\n\n if self.capabilities.get('Action'):\n self._request('Action')\n return True", "response": "Login to the RETS board and return an instance of Bulletin"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting resource metadata for a resource", "response": "def get_resource_metadata(self, resource=None):\n \"\"\"\n Get resource metadata\n :param resource: The name of the resource to get metadata for\n :return: list\n \"\"\"\n result = self._make_metadata_request(meta_id=0, metadata_type='METADATA-RESOURCE')\n if resource:\n result = next((item for item in result if item['ResourceID'] == resource), None)\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_table_metadata(self, resource, resource_class):\n return self._make_metadata_request(meta_id=resource + ':' + resource_class, metadata_type='METADATA-TABLE')", "response": "Get the metadata for a given resource and class"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_lookup_values(self, resource, lookup_name):\n return self._make_metadata_request(meta_id=resource + ':' + lookup_name, metadata_type='METADATA-LOOKUP_TYPE')", "response": "Get possible lookup values for a given resource and lookup name"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _make_metadata_request(self, meta_id, metadata_type=None):\n # If this metadata _request has already happened, returned the saved result.\n key = '{0!s}:{1!s}'.format(metadata_type, meta_id)\n if key in self.metadata_responses and self.cache_metadata:\n response = self.metadata_responses[key]\n else:\n response = self._request(\n capability='GetMetadata',\n options={\n 'query': {\n 'Type': metadata_type,\n 'ID': meta_id,\n 'Format': self.metadata_format\n }\n }\n )\n self.metadata_responses[key] = response\n\n if self.metadata_format == 'COMPACT-DECODED':\n parser = CompactMetadata()\n else:\n parser = StandardXMLetadata()\n\n try:\n return parser.parse(response=response, metadata_type=metadata_type)\n except RETSException as e:\n # Remove response from cache\n self.metadata_responses.pop(key, None)\n\n # If the server responds with an invalid parameter for COMPACT-DECODED, try STANDARD-XML\n if self.metadata_format != 'STANDARD-XML' and e.reply_code in ['20513', '20514']:\n self.metadata_responses.pop(key, None)\n self.metadata_format = 'STANDARD-XML'\n return self._make_metadata_request(meta_id=meta_id, metadata_type=metadata_type)\n raise RETSException(e.reply_text, e.reply_code)", "response": "Make the metadata request."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_preferred_object(self, resource, object_type, content_id, location=0):\n collection = self.get_object(resource=resource, object_type=object_type,\n content_ids=content_id, object_ids='0', location=location)\n return collection[0]", "response": "Get the first object from a Resource\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_object(self, resource, object_type, content_ids, object_ids='*', location=0):\n object_helper = GetObject()\n request_ids = object_helper.ids(content_ids=content_ids, object_ids=object_ids)\n\n response = self._request(\n capability='GetObject',\n options={\n 'query':\n {\n \"Resource\": resource,\n \"Type\": object_type,\n \"ID\": ','.join(request_ids),\n \"Location\": location\n }\n }\n )\n\n if 'multipart' in response.headers.get('Content-Type'):\n parser = MultipleObjectParser()\n collection = parser.parse_image_response(response)\n else:\n parser = SingleObjectParser()\n collection = [parser.parse_image_response(response)]\n\n return collection", "response": "Get a list of objects from a resource"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef search(self, resource, resource_class, search_filter=None, dmql_query=None, limit=9999999, offset=0,\n optional_parameters=None, auto_offset=True, query_type='DMQL2', standard_names=0,\n response_format='COMPACT-DECODED'):\n \"\"\"\n Preform a search on the RETS board\n :param resource: The resource that contains the class to search\n :param resource_class: The class to search\n :param search_filter: The query as a dict\n :param dmql_query: The query in dmql format\n :param limit: Limit search values count\n :param offset: Offset for RETS request. Useful when RETS limits number of results or transactions\n :param optional_parameters: Values for option paramters\n :param auto_offset: Should the search be allowed to trigger subsequent searches.\n :param query_type: DMQL or DMQL2 depending on the rets server.\n :param standard_names: 1 to use standard names, 0 to use system names\n :param response_format: COMPACT-DECODED, COMPACT, or STANDARD-XML\n :return: dict\n \"\"\"\n\n if (search_filter and dmql_query) or (not search_filter and not dmql_query):\n raise ValueError(\"You may specify either a search_filter or dmql_query\")\n\n search_helper = DMQLHelper()\n\n if dmql_query:\n dmql_query = search_helper.dmql(query=dmql_query)\n else:\n dmql_query = search_helper.filter_to_dmql(filter_dict=search_filter)\n\n parameters = {\n 'SearchType': resource,\n 'Class': resource_class,\n 'Query': dmql_query,\n 'QueryType': query_type,\n 'Count': 1,\n 'Format': response_format,\n 'StandardNames': standard_names,\n }\n\n if not optional_parameters:\n optional_parameters = {}\n parameters.update(optional_parameters)\n\n # if the Select parameter given is an array, format it as it needs to be\n if 'Select' in parameters and isinstance(parameters.get('Select'), list):\n parameters['Select'] = ','.join(parameters['Select'])\n\n if limit:\n parameters['Limit'] = limit\n\n if offset:\n parameters['Offset'] = offset\n\n search_cursor = OneXSearchCursor()\n response = self._request(\n capability='Search',\n options={\n 'query': parameters,\n },\n stream=True\n )\n try:\n return search_cursor.generator(response=response)\n\n except MaxrowException as max_exception:\n # Recursive searching if automatically performing offsets for the client\n if auto_offset and limit > len(max_exception.rows_returned):\n new_limit = limit - len(max_exception.rows_returned) # have not returned results to the desired limit\n new_offset = offset + len(max_exception.rows_returned) # adjust offset\n results = self.search(resource=resource, resource_class=resource_class, search_filter=None,\n dmql_query=dmql_query, offset=new_offset, limit=new_limit,\n optional_parameters=optional_parameters, auto_offset=auto_offset)\n\n previous_results = max_exception.rows_returned\n return previous_results + results\n return max_exception.rows_returned", "response": "Perform a dmql search on the RETS board and return a dictionary of the results."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nmakes a request to the RETS server.", "response": "def _request(self, capability, options=None, stream=False):\n \"\"\"\n Make a _request to the RETS server\n :param capability: The name of the capability to use to get the URI\n :param options: Options to put into the _request\n :return: Response\n \"\"\"\n if options is None:\n options = {}\n\n options.update({\n 'headers': self.client.headers.copy()\n })\n\n url = self.capabilities.get(capability)\n\n if not url:\n msg = \"{0!s} tried but no valid endpoints was found. Did you forget to Login?\".format(capability)\n raise NotLoggedIn(msg)\n\n if self.user_agent_password:\n ua_digest = self._user_agent_digest_hash()\n options['headers']['RETS-UA-Authorization'] = 'Digest {0!s}'.format(ua_digest)\n\n if self.use_post_method and capability != 'Action': # Action Requests should always be GET\n query = options.get('query')\n response = self.client.post(url, data=query, headers=options['headers'], stream=stream)\n else:\n if 'query' in options:\n url += '?' + '&'.join('{0!s}={1!s}'.format(k, quote(str(v))) for k, v in options['query'].items())\n\n response = self.client.get(url, headers=options['headers'], stream=stream)\n\n if response.status_code in [400, 401]:\n if capability == 'Login':\n m = \"Could not log into the RETS server with the provided credentials.\"\n else:\n m = \"The RETS server returned a 401 status code. You must be logged in to make this request.\"\n raise NotLoggedIn(m)\n\n elif response.status_code == 404 and self.use_post_method:\n raise HTTPException(\"Got a 404 when making a POST request. Try setting use_post_method=False when \"\n \"initializing the Session.\")\n\n return response"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nhashes the user agent and password and return the md5 hash", "response": "def _user_agent_digest_hash(self):\n \"\"\"\n Hash the user agent and user agent password\n Section 3.10 of https://www.nar.realtor/retsorg.nsf/retsproto1.7d6.pdf\n :return: md5\n \"\"\"\n if not self.version:\n raise MissingVersion(\"A version is required for user agent auth. The RETS server should set this\"\n \"automatically but it has not. Please instantiate the session with a version argument\"\n \"to provide the version.\")\n version_number = self.version.strip('RETS/')\n user_str = '{0!s}:{1!s}'.format(self.user_agent, self.user_agent_password).encode('utf-8')\n a1 = hashlib.md5(user_str).hexdigest()\n session_id = self.session_id if self.session_id is not None else ''\n digest_str = '{0!s}::{1!s}:{2!s}'.format(a1, session_id, version_number).encode('utf-8')\n digest = hashlib.md5(digest_str).hexdigest()\n return digest"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef exempt(self, obj):\n name = '%s.%s' % (obj.__module__, obj.__name__)\n\n @wraps(obj)\n def __inner(*a, **k):\n return obj(*a, **k)\n\n self._exempt_routes.add(name)\n return __inner", "response": "Decorator to mark a view as exempt from htmlmin.\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dmql(query):\n if isinstance(query, dict):\n raise ValueError(\"You supplied a dictionary to the dmql_query parameter, but a string is required.\"\n \" Did you mean to pass this to the search_filter parameter? \")\n\n # automatically surround the given query with parentheses if it doesn't have them already\n if len(query) > 0 and query != \"*\" and query[0] != '(' and query[-1] != ')':\n query = '({})'.format(query)\n return query", "response": "Client supplied raw DMQL ensure quote wrap."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting a dictionary of filter values into a DMQL query string.", "response": "def filter_to_dmql(filter_dict):\n \"\"\"Converts the filter dictionary into DMQL\"\"\"\n\n if not isinstance(filter_dict, (dict, collections.OrderedDict)):\n raise TypeError('Expected a dictionary type buy got {} instead.'.format(type(filter_dict)))\n\n def is_date_time_type(val):\n \"\"\"Returns True if the value is a datetime\"\"\"\n return isinstance(val, (datetime.datetime, datetime.date, datetime.time))\n\n def evaluate_datetime(val):\n \"\"\"Converts the datetime object into the RETS expected format\"\"\"\n date_format = '%Y-%m-%d'\n time_format = '%H:%M:%S'\n datetime_format = '{}T{}'.format(date_format, time_format)\n\n if isinstance(val, datetime.datetime):\n evaluated = val.strftime(datetime_format)\n elif isinstance(val, datetime.date):\n evaluated = val.strftime(date_format)\n elif isinstance(val, datetime.time):\n evaluated = val.strftime(time_format)\n else:\n evaluated = val\n\n return evaluated\n\n def evaluate_operators(key_dict):\n \"\"\"Turns the custom filter operators into the expected RETS query\"\"\"\n allowed_operators = ['$gte', '$lte', '$contains', '$begins', '$ends', '$in', '$nin', '$neq']\n\n # If key not in allowed_operators, assume it is a field name with the and operation.\n if not all(op in allowed_operators for op in key_dict.keys()):\n raise ValueError(\"You have supplied an invalid operator. \"\n \"Please provide one of the following {}\".format(allowed_operators))\n\n # We can have a single operator key, or the combination of gte/lte\n keys = key_dict.keys()\n string = ''\n\n # Search between two numbers or two dates\n if len(keys) == 2 and all(k in ['$gte', '$lte'] for k in keys):\n if all(is_date_time_type(key_dict[v]) for v in keys):\n # comparing dates\n string = '{}-{}'.format(evaluate_datetime(key_dict['$gte']), evaluate_datetime(key_dict['$lte']))\n else:\n # comparing numbers\n try:\n float(key_dict['$gte'])\n float(key_dict['$lte'])\n except ValueError:\n raise ValueError(\"$gte and $lte expect numeric or datetime values\")\n string = '{:.2f}-{:.2f}'.format(key_dict['$gte'], key_dict['$lte'])\n\n # Using a single operator key\n elif len(keys) == 1:\n if '$gte' in key_dict:\n if is_date_time_type(key_dict['$gte']):\n string = '{}+'.format(evaluate_datetime(key_dict['$gte']))\n else:\n try:\n float(key_dict['$gte'])\n except ValueError:\n raise ValueError(\"$gte expects a numeric value or a datetime object\")\n string = '{:.2f}+'.format(key_dict['$gte'])\n\n elif '$lte' in key_dict:\n if is_date_time_type(key_dict['$lte']):\n string = '{}-'.format(evaluate_datetime(key_dict['$lte']))\n else:\n try:\n float(key_dict['$lte'])\n except ValueError:\n raise ValueError(\"$lte expects a numeric value or a datetime object\")\n string = '{:.2f}-'.format(key_dict['$lte'])\n\n elif '$in' in key_dict:\n if not isinstance(key_dict['$in'], list):\n raise ValueError(\"$in expects a list of strings\")\n key_dict['$in'] = [evaluate_datetime(v) for v in key_dict['$in']]\n if not all(isinstance(v, string_types) for v in key_dict['$in']):\n raise ValueError(\"$in expects a list of strings\")\n options = ','.join(key_dict['$in'])\n string = '{}'.format(options)\n\n elif '$nin' in key_dict:\n if not isinstance(key_dict['$nin'], list):\n raise ValueError(\"$nin expects a list of strings\")\n key_dict['$nin'] = [evaluate_datetime(v) for v in key_dict['$nin']]\n if not all(isinstance(v, string_types) for v in key_dict['$nin']):\n raise ValueError(\"$nin expects a list of strings\")\n options = ','.join(key_dict['$nin'])\n string = '~{}'.format(options)\n\n elif '$contains' in key_dict:\n if not isinstance(key_dict['$contains'], string_types):\n raise ValueError(\"$contains expects a string.\")\n string = '*{}*'.format(key_dict['$contains'])\n\n elif '$begins' in key_dict:\n if not isinstance(key_dict['$begins'], string_types):\n raise ValueError(\"$begins expects a string.\")\n string = '{}*'.format(key_dict['$begins'])\n\n elif '$ends' in key_dict:\n if not isinstance(key_dict['$ends'], string_types):\n raise ValueError(\"$ends expects a string.\")\n string = '*{}'.format(key_dict['$ends'])\n\n elif '$neq' in key_dict:\n string = '~{}'.format(key_dict['$neq'])\n\n else:\n # Provided too many or too few operators\n raise ValueError(\"Please supply $gte and $lte for getting values between numbers or 1 of {}\".format(\n allowed_operators))\n\n return string\n\n dmql_search_filters = []\n\n for filt, value in filter_dict.items():\n dmql_string = '({}='.format(filt)\n if isinstance(value, dict):\n # Applying an operator. This will need to be recursive because of the or possibility\n dmql_string += evaluate_operators(key_dict=value)\n else:\n # Simle equals statement\n dmql_string += '{}'.format(evaluate_datetime(value))\n dmql_string += ')'\n dmql_search_filters.append(dmql_string)\n\n search_string = ','.join(dmql_search_filters)\n # Converts the filter dictionary to dmqp string\n logger.debug(\"Filter returned the following DMQL: {}\".format(search_string))\n return search_string"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_attributes(input_dict):\n return {k.lstrip(\"@\"): v for k, v in input_dict.items() if k[0] == \"@\"}", "response": "Get attributes of xml tags in input_dict and creates a dictionary with the key and the value as the key and the value as the value"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef data_columns_to_dict(columns_string, dict_string, delimiter=None):\n if delimiter:\n return {k: v for k, v in zip(columns_string.split(delimiter), dict_string.split(delimiter))}\n else:\n return {k: v for k, v in zip(columns_string.split(), dict_string.split())}", "response": "Turn column names in a single string into a dictionary with the key being the column name and the value being the value in that column for each row."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef analyze_reply_code(self, xml_response_dict):\n if 'RETS-STATUS' in xml_response_dict:\n attributes = self.get_attributes(xml_response_dict['RETS-STATUS'])\n reply_code = attributes['ReplyCode']\n reply_text = attributes.get('ReplyText', 'RETS did not supply a Reply Text.')\n\n logger.debug(\"Received ReplyCode of {0!s} from the RETS Server: {0!s}\".format(reply_code, reply_text))\n if reply_code != '0':\n raise RETSException(reply_text, reply_code)\n\n elif 'RETS' not in xml_response_dict: # pragma: no cover\n raise RETSException(\"The tag was expected in the response XML but it was not found.\")\n\n attributes = self.get_attributes(input_dict=xml_response_dict['RETS'])\n if 'ReplyCode' not in attributes: # pragma: no cover\n # The RETS server did not return a response code.\n return True\n\n reply_code = attributes['ReplyCode']\n reply_text = attributes.get('ReplyText', 'RETS did not supply a Reply Text.')\n\n logger.debug(\"Received ReplyCode of {0!s} from the RETS Server: {0!s}\".format(reply_code, reply_text))\n if reply_code != '0':\n raise RETSException(reply_text, reply_code)", "response": "Checks the RETS Response Code and handles non - zero answers."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nappend the content and object ids how RETS expects them", "response": "def ids(self, content_ids, object_ids):\n \"\"\"Appends the content and object ids how RETS expects them\"\"\"\n result = []\n\n content_ids = self.split(content_ids, False)\n object_ids = self.split(object_ids)\n\n for cid in content_ids:\n result.append('{}:{}'.format(cid, ':'.join(object_ids)))\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsplit value into list of strings.", "response": "def split(value, dash_ranges=True):\n \"\"\"Splits \"\"\"\n if isinstance(value, list):\n value = [str(v) for v in value]\n else:\n str_value = str(value)\n dash_matches = re.match(pattern='(\\d+)\\-(\\d+)', string=str_value)\n\n if ':' in str_value or ',' in str_value:\n value = [v.strip() for v in str_value.replace(',', ':').split(':')]\n elif dash_ranges and dash_matches:\n start_range = int(dash_matches.group(1))\n end_range = int(dash_matches.group(2)) + 1\n rng = range(start_range, end_range)\n value = [str(r) for r in rng]\n else:\n value = [str_value]\n\n return value"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef set_value(self, key, value):\n file_cache = self.read_file()\n if file_cache:\n file_cache[key] = value\n else:\n file_cache = {}\n file_cache[key] = value\n self.update_file(file_cache)", "response": "Set the value of the key to the file."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef delete_value(self, key):\n response = {}\n response['status'] = False\n response['msg'] = \"key does not exist\"\n\n file_cache = self.read_file()\n if key in file_cache:\n del file_cache[key]\n self.update_file(file_cache)\n response['status'] = True\n response['msg'] = \"success\"\n return response", "response": "Delete the value in the cache."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef read_file(self):\n file_obj = open(self.file, 'r')\n content = file_obj.read()\n file_obj.close()\n if content:\n content = json.loads(content)\n return content\n else:\n return {}", "response": "Reads the file and returns the content in json formate."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse(self, response, metadata_type):\n xml = xmltodict.parse(response.text)\n self.analyze_reply_code(xml_response_dict=xml)\n base = xml.get('RETS', {}).get(metadata_type, {})\n attributes = self.get_attributes(base)\n\n parsed = []\n if base.get('System') or base.get('SYSTEM'):\n system_obj = {}\n\n if base.get('SYSTEM', {}).get('@SystemDescription'):\n system_obj['system_id'] = str(base['SYSTEM']['@SystemID'])\n\n if base.get('SYSTEM', {}).get('@SystemDescription'):\n system_obj['system_description'] = str(base['SYSTEM']['@SystemDescription'])\n\n if base.get('SYSTEM', {}).get('@TimeZoneOffset'):\n system_obj['timezone_offset'] = str(base['SYSTEM']['@TimeZoneOffset'])\n\n if base.get('SYSTEM', {}).get('Comments'):\n system_obj['comments'] = base['SYSTEM']['Comments']\n\n if base.get('@Version'):\n system_obj['version'] = base['@Version']\n\n parsed.append(system_obj)\n\n elif 'DATA' in base:\n if not isinstance(base['DATA'], list): # xmltodict could take single entry XML lists and turn them into str\n base['DATA'] = [base['DATA']]\n\n for data in base['DATA']:\n data_dict = self.data_columns_to_dict(columns_string=base.get('COLUMNS', ''), dict_string=data)\n data_dict.update(attributes)\n parsed.append(data_dict)\n\n return parsed", "response": "Parses RETS metadata using the COMPACT - DECODED format."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef parse(self, response, metadata_type):\n xml = xmltodict.parse(response.text)\n self.analyze_reply_code(xml_response_dict=xml)\n base = xml.get('RETS', {}).get('METADATA', {}).get(metadata_type, {})\n\n if metadata_type == 'METADATA-SYSTEM':\n syst = base.get('System', base.get('SYSTEM'))\n if not syst:\n raise ParseError(\"Could not get the System key from a METADATA-SYSTEM request.\")\n\n system_obj = {}\n if syst.get('SystemID'):\n system_obj['system_id'] = str(syst['SystemID'])\n if syst.get('SystemDescription'):\n system_obj['system_description'] = str(syst['SystemDescription'])\n if syst.get('Comments'):\n system_obj['comments'] = syst['Comments']\n if base.get('@Version'):\n system_obj['version'] = base['@Version']\n return [system_obj]\n\n elif metadata_type == 'METADATA-CLASS':\n key = 'class'\n elif metadata_type == 'METADATA-RESOURCE':\n key = 'resource'\n elif metadata_type == 'METADATA-LOOKUP_TYPE':\n key = 'lookuptype'\n elif metadata_type == 'METADATA-OBJECT':\n key = 'object'\n elif metadata_type == 'METADATA-TABLE':\n key = 'field'\n else:\n msg = \"Got an unknown metadata type of {0!s}\".format(metadata_type)\n raise ParseError(msg)\n\n # Get the version with the right capitalization from the dictionary\n key_cap = None\n for k in base.keys():\n if k.lower() == key:\n key_cap = k\n\n if not key_cap:\n msg = 'Could not find {0!s} in the response XML'.format(key)\n raise ParseError(msg)\n\n if isinstance(base[key_cap], list):\n return base[key_cap]\n else:\n return [base[key_cap]]", "response": "Parses the RETS metadata using the STANDARD - XML format."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_multiparts(response):\n boundary = None\n for part in response.headers.get('Content-Type', '').split(';'):\n if 'boundary=' in part:\n boundary = '--{}'.format(part.split('=', 1)[1].strip('\\\"'))\n break\n\n if not boundary:\n raise ParseError(\"Was not able to find the boundary between objects in a multipart response\")\n\n if response.content is None:\n return []\n\n response_string = response.content\n\n if six.PY3:\n # Python3 returns bytes, decode for string operations\n response_string = response_string.decode('latin-1')\n\n # help bad responses be more multipart compliant\n whole_body = response_string.strip('\\r\\n')\n no_front_boundary = whole_body.strip(boundary)\n # The boundary comes with some characters\n\n multi_parts = []\n for part in no_front_boundary.split(boundary):\n multi_parts.append(part.strip('\\r\\n'))\n\n return multi_parts", "response": "Get the list of objects in a multipart response."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_image_response(self, response):\n if 'xml' in response.headers.get('Content-Type'):\n # Got an XML response, likely an error code.\n xml = xmltodict.parse(response.text)\n self.analyze_reply_code(xml_response_dict=xml)\n\n multi_parts = self._get_multiparts(response)\n parsed = []\n # go through each part of the multipart message\n for part in multi_parts:\n clean_part = part.strip('\\r\\n\\r\\n')\n if '\\r\\n\\r\\n' in clean_part:\n header, body = clean_part.split('\\r\\n\\r\\n', 1)\n else:\n header = clean_part\n body = None\n part_header_dict = {k.strip(): v.strip() for k, v in (h.split(':', 1) for h in header.split('\\r\\n'))}\n\n # Some multipart requests respond with a text/XML part stating an error\n if 'xml' in part_header_dict.get('Content-Type'):\n # Got an XML response, likely an error code.\n # Some rets servers give characters after the closing brace.\n body = body[:body.index('/>') + 2] if '/>' in body else body\n xml = xmltodict.parse(body)\n try:\n self.analyze_reply_code(xml_response_dict=xml)\n except RETSException as e:\n if e.reply_code == '20403':\n # The requested object_id was not found.\n continue\n raise e\n\n if body:\n obj = self._response_object_from_header(\n obj_head_dict=part_header_dict,\n content=body.encode('latin-1') if six.PY3 else body)\n else:\n obj = self._response_object_from_header(obj_head_dict=part_header_dict)\n parsed.append(obj)\n return parsed", "response": "Parses the response from the RETS feed and returns a list of SingleObjectParser objects."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_image_response(self, response):\n if 'xml' in response.headers.get('Content-Type'):\n # Got an XML response, likely an error code.\n xml = xmltodict.parse(response.text)\n self.analyze_reply_code(xml_response_dict=xml)\n\n obj = self._response_object_from_header(\n obj_head_dict=response.headers,\n content=response.content)\n return obj", "response": "Parse a single object from the RETS feed and return it."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef auth(self):\n if not self.store_handler.has_value('access_token'):\n params = {}\n params[\"access_key\"] = self.access_key\n params[\"secret_key\"] = self.secret_key\n params[\"app_id\"] = self.app_id\n params[\"device_id\"] = self.device_id\n auth_url = self.api_path + \"auth/\"\n response = self.get_response(auth_url, params, \"post\")\n\n if 'auth' in response:\n self.store_handler.set_value(\"access_token\", response['auth']['access_token'])\n self.store_handler.set_value(\"expires\", response['auth']['expires'])\n logger.info('Getting new access token')\n else:\n msg = \"Error getting access_token, \" + \\\n \"please verify your access_key, secret_key and app_id\"\n logger.error(msg)\n raise Exception(\"Auth Failed, please check your access details\")", "response": "Auth function is used to get the access token for every request to the server."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the active access token.", "response": "def get_active_token(self):\n \"\"\"\n Getting the valid access token.\n\n Access token expires every 24 hours, It will expires then it will\n generate a new token.\n Return:\n active access token \n \"\"\"\n\n expire_time = self.store_handler.has_value(\"expires\")\n access_token = self.store_handler.has_value(\"access_token\")\n if expire_time and access_token:\n expire_time = self.store_handler.get_value(\"expires\")\n if not datetime.now() < datetime.fromtimestamp(float(expire_time)):\n self.store_handler.delete_value(\"access_token\")\n self.store_handler.delete_value(\"expires\")\n logger.info('Access token expired, going to get new token')\n self.auth()\n else:\n logger.info('Access token noy expired yet')\n else:\n self.auth()\n return self.store_handler.get_value(\"access_token\")"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncall the Match API. Arg: match_key: key of the match card_type: optional, default to full_card. Accepted values are micro_card, summary_card & full_card. Return: json data", "response": "def get_match(self, match_key, card_type=\"full_card\"):\n \"\"\"\n Calling the Match API.\n \n Arg:\n match_key: key of the match\n card_type: optional, default to full_card. Accepted values are \n micro_card, summary_card & full_card.\n Return:\n json data \n \"\"\"\n\n match_url = self.api_path + \"match/\" + match_key + \"/\"\n params = {}\n params[\"card_type\"] = card_type\n response = self.get_response(match_url, params)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalls the Recent Matches API.", "response": "def get_recent_matches(self, card_type=\"micro_card\"):\n \"\"\"\n Calling the Recent Matches API.\n\n Arg:\n card_type: optional, default to micro_card. Accepted values are\n micro_card & summary_card.\n Return:\n json data\n \"\"\"\n\n recent_matches_url = self.api_path + \"recent_matches/\"\n params = {}\n params[\"card_type\"] = card_type\n response = self.get_response(recent_matches_url, params)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_player_stats(self, player_key, board_key):\n player_stats_url = self.api_path + 'player/' + player_key + '/league/' + board_key + '/stats/'\n response = self.get_response(player_stats_url)\n return response", "response": "Calling the Player Stats API"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_ball_by_ball(self, match_key, over_key=None):\n\n if over_key:\n ball_by_ball_url = \"{base_path}match/{match_key}/balls/{over_key}/\".format(base_path=self.api_path, match_key=match_key, over_key=over_key)\n else:\n ball_by_ball_url = \"{base_path}match/{match_key}/balls/\".format(base_path=self.api_path, match_key=match_key)\n response = self.get_response(ball_by_ball_url)\n return response", "response": "Get the balls by ball"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalling specific season recent matches. Arg: season_key: key of the season. Return: json date", "response": "def get_recent_season_matches(self, season_key):\n \"\"\"\n Calling specific season recent matches.\n\n Arg:\n season_key: key of the season.\n Return:\n json date\n \"\"\"\n\n season_recent_matches_url = self.api_path + \"season/\" + season_key + \"/recent_matches/\"\n response = self.get_response(season_recent_matches_url)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_recent_seasons(self):\n\n recent_seasons_url = self.api_path + \"recent_seasons/\"\n response = self.get_response(recent_seasons_url)\n return response", "response": "This method returns the recent seasons for the current user"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalling the Schedule API.", "response": "def get_schedule(self, date=None):\n \"\"\"\n Calling the Schedule API.\n\n Return:\n json data\n \"\"\"\n\n schedule_url = self.api_path + \"schedule/\"\n params = {}\n if date:\n params['date'] = date\n response = self.get_response(schedule_url, params)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_season_schedule(self, season_key):\n\n schedule_url = self.api_path + \"season/\" + season_key + \"/schedule/\"\n response = self.get_response(schedule_url)\n return response", "response": "Calling specific season schedule"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_season(self, season_key, card_type=\"micro_card\"):\n\n season_url = self.api_path + \"season/\" + season_key + \"/\"\n params = {}\n params[\"card_type\"] = card_type\n response = self.get_response(season_url, params)\n return response", "response": "This function returns the neccesary data for a given season."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalling Season Stats API.", "response": "def get_season_stats(self, season_key):\n \"\"\"\n Calling Season Stats API.\n\n Arg:\n season_key: key of the season\n Return:\n json data\n \"\"\"\n\n season_stats_url = self.api_path + \"season/\" + season_key + \"/stats/\"\n response = self.get_response(season_stats_url)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_season_team(self, season_key, season_team_key,stats_type=None):\n params = {\"stats_type\": stats_type}\n season_team_url = self.api_path + 'season/' + season_key + '/team/' + season_team_key + '/'\n response = self.get_response(season_team_url, params=params)\n return response", "response": "Calling Season teams API"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_season_points(self, season_key):\n\n season_points_url = self.api_path + \"season/\" + season_key + \"/points/\"\n response = self.get_response(season_points_url)\n return response", "response": "This method returns the Season Points API."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_season_player_stats(self, season_key, player_key):\n\n season_player_stats_url = self.api_path + \"season/\" + season_key + \"/player/\" + player_key + \"/stats/\"\n response = self.get_response(season_player_stats_url)\n return response", "response": "Calling Season Player Stats API."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalling Overs Summary API", "response": "def get_overs_summary(self, match_key):\n \"\"\"\n Calling Overs Summary API\n\n Arg:\n match_key: key of the match\n Return:\n json data\n \"\"\"\n overs_summary_url = self.api_path + \"match/\" + match_key + \"/overs_summary/\"\n response = self.get_response(overs_summary_url)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncall News Aggregation API Return: json data", "response": "def get_news_aggregation(self):\n \"\"\"\n Calling News Aggregation API\n\n Return:\n json data\n \"\"\"\n\n news_aggregation_url = self.api_path + \"news_aggregation\" + \"/\"\n response = self.get_response(news_aggregation_url)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncall Fantasy Credit API Arg: match_key: key of the match Return: json data", "response": "def get_fantasy_credits(self, match_key):\n \"\"\"\n Calling Fantasy Credit API\n\n Arg:\n match_key: key of the match\n Return:\n json data\n \"\"\"\n\n fantasy_credit_url = self.api_path_v3 + \"fantasy-match-credits/\" + match_key + \"/\"\n response = self.get_response(fantasy_credit_url)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalling Fantasy Points API Arg: match_key: key of the match Return: json data", "response": "def get_fantasy_points(self, match_key):\n \"\"\"\n Calling Fantasy Points API\n\n Arg:\n match_key: key of the match\n Return:\n json data\n \"\"\"\n\n fantasy_points_url = self.api_path_v3 + \"fantasy-match-points/\" + match_key + \"/\"\n response = self.get_response(fantasy_points_url)\n return response"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates an output string by replacing the keywords in the format string with the corresponding values from a submission dictionary.", "response": "def main(self):\n \"\"\"\n Generates an output string by replacing the keywords in the format\n string with the corresponding values from a submission dictionary.\n \"\"\"\n self.manage_submissions()\n out_string = self.options['format']\n\n # Pop until we get something which len(title) <= max-chars\n length = float('inf')\n while length > self.options['max_chars']:\n self.selected_submission = self.submissions.pop()\n length = len(self.selected_submission['title'])\n\n for k, v in self.selected_submission.items():\n out_string = out_string.replace(k, self.h.unescape(str(v)))\n return self.output(out_string, out_string)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlogs into Reddit in order to display a personalised front page.", "response": "def login(self):\n \"\"\"\n Logs into Reddit in order to display a personalised front page.\n \"\"\"\n data = {'user': self.options['username'], 'passwd':\n self.options['password'], 'api_type': 'json'}\n response = self.client.post('http://www.reddit.com/api/login', data=data)\n self.client.modhash = response.json()['json']['data']['modhash']"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmanages URL creation and the specifics for front page and subreddit.", "response": "def manage_submissions(self):\n \"\"\"\n If there are no or only one submissions left, get new submissions.\n This function manages URL creation and the specifics for front page\n or subreddit mode.\n \"\"\"\n if not hasattr(self, 'submissions') or len(self.submissions) == 1:\n self.submissions = []\n if self.options['mode'] == 'front':\n # If there are no login details, the standard front\n # page will be displayed.\n if self.options['password'] and self.options['username']:\n self.login()\n url = 'http://reddit.com/.json?sort={0}'.format(self.options['sort'])\n self.submissions = self.get_submissions(url)\n elif self.options['mode'] == 'subreddit':\n for subreddit in self.options['subreddits']:\n url = 'http://reddit.com/r/{0}/.json?sort={1}'.format(\n subreddit, self.options['limit'])\n self.submissions += self.get_submissions(url)\n else:\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconnects to Reddit and gets a JSON representation of submissions.", "response": "def get_submissions(self, url):\n \"\"\"\n Connects to Reddit and gets a JSON representation of submissions.\n This JSON data is then processed and returned.\n\n url: A url that requests for submissions should be sent to.\n \"\"\"\n response = self.client.get(url, params={'limit': self.options['limit']})\n submissions = [x['data'] for x in response.json()['data']['children']]\n return submissions"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef main(self):\n try:\n # Setting stderr to subprocess.STDOUT seems to stop the error\n # message returned by the process from being output to STDOUT.\n cmus_output = subprocess.check_output(['cmus-remote', '-Q'],\n stderr=subprocess.STDOUT).decode('utf-8')\n except subprocess.CalledProcessError:\n return self.output(None, None)\n if 'duration' in cmus_output:\n status = self.convert_cmus_output(cmus_output)\n out_string = self.options['format']\n for k, v in status.items():\n out_string = out_string.replace(k, v)\n else:\n out_string = None\n return self.output(out_string, out_string)", "response": "A compulsary function that gets the output of the cmus - remote - Q command and converts it to unicode and then outputs the result."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nchange the newline separated string of output data into a dictionary which can then be used to replace the strings in the config format.", "response": "def convert_cmus_output(self, cmus_output):\n \"\"\"\n Change the newline separated string of output data into\n a dictionary which can then be used to replace the strings in the config\n format.\n\n cmus_output: A string with information about cmus that is newline\n seperated. Running cmus-remote -Q in a terminal will show you what\n you're dealing with.\n \"\"\"\n cmus_output = cmus_output.split('\\n')\n cmus_output = [x.replace('tag ', '') for x in cmus_output if not x in '']\n cmus_output = [x.replace('set ', '') for x in cmus_output]\n status = {}\n partitioned = (item.partition(' ') for item in cmus_output)\n status = {item[0]: item[2] for item in partitioned}\n status['duration'] = self.convert_time(status['duration'])\n status['position'] = self.convert_time(status['position'])\n return status"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\noutput all of the options and data for a segment.", "response": "def output(self, full_text, short_text):\n \"\"\"\n Output all of the options and data for a segment.\n\n full_text: A string representing the data that should be output to i3bar.\n short_text: A more concise version of full_text, in case there is minimal\n room on the i3bar.\n \"\"\"\n full_text = full_text.replace('\\n', '')\n short_text = short_text.replace('\\n', '')\n self.output_options.update({'full_text': full_text, 'short_text': short_text})\n self.output_options = {k: v for k, v in self.output_options.items() if v}\n return self.output_options"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef on_click(self, event):\n if event['button'] == 1 and 'button1' in self.options:\n subprocess.call(self.options['button1'].split())\n elif event['button'] == 2 and 'button2' in self.options:\n subprocess.call(self.options['button2'].split())\n elif event['button'] == 3 and 'button3' in self.options:\n subprocess.call(self.options['button3'].split())", "response": "A function that should be overwritten by a plugin that wishes to react to events other than running the react\n command related to a button."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef url_reverse(request):\n if request.method in ('GET', 'POST'): \n data = getattr(request, request.method) \n url_name = data.get('url_name') \n try: \n path = urls.reverse(url_name, args=data.getlist('args')) \n (view_func, args, kwargs) = urls.resolve(path)\n return http.HttpResponse(path, content_type='text/plain')\n except urls.NoReverseMatch: \n return http.HttpResponse('Error', content_type='text/plain')\n return http.HttpResponseNotAllowed(('GET', 'POST'))", "response": "Reverse the requested URL."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a JSON representation of the image in a complete path.", "response": "def url_image(request, image_id, thumb_options=None, width=None, height=None):\n \"\"\"\n Converts a filer image ID in a complete path\n\n :param request: Request object\n :param image_id: Filer image ID\n :param thumb_options: ThumbnailOption ID\n :param width: user-provided width\n :param height: user-provided height\n :return: JSON serialized URL components ('url', 'width', 'height')\n \"\"\"\n image = File.objects.get(pk=image_id)\n if getattr(image, 'canonical_url', None):\n url = image.canonical_url\n else:\n url = image.url\n thumb = _return_thumbnail(image, thumb_options, width, height)\n if thumb:\n image = thumb\n url = image.url\n data = {\n 'url': url,\n 'width': image.width,\n 'height': image.height,\n }\n return http.HttpResponse(json.dumps(data), content_type='application/json')"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the requested ThumbnailOption as JSON", "response": "def thumbnail_options(request):\n \"\"\"\n Returns the requested ThumbnailOption as JSON\n\n :param request: Request object\n :return: JSON serialized ThumbnailOption\n \"\"\"\n response_data = [{'id': opt.pk, 'name': opt.name} for opt in ThumbnailOption.objects.all()]\n return http.HttpResponse(json.dumps(response_data), content_type=\"application/json\")"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the content of an image in a Filer object.", "response": "def serve_image(request, image_id, thumb_options=None, width=None, height=None):\n \"\"\"\n returns the content of an image sized according to the parameters\n\n :param request: Request object\n :param image_id: Filer image ID\n :param thumb_options: ThumbnailOption ID\n :param width: user-provided width\n :param height: user-provided height\n :return: JSON serialized URL components ('url', 'width', 'height')\n \"\"\"\n image = File.objects.get(pk=image_id)\n if getattr(image, 'canonical_url', None):\n url = image.canonical_url\n else:\n url = image.url\n thumb = _return_thumbnail(image, thumb_options, width, height)\n if thumb:\n return server.serve(request, file_obj=thumb, save_as=False)\n else:\n return HttpResponseRedirect(url)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _touch_dir(self, path):\n try:\n os.makedirs(path)\n except OSError as e:\n if e.errno != errno.EEXIST:\n raise", "response": "A helper function to create a directory if it doesn t exist."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reload(self):\n self._conf = configparser.ConfigParser()\n # Preserve the case of sections and keys.\n self._conf.optionxform = str\n self._conf.read(self.config_file_path)\n if 'general' not in self._conf.keys():\n raise IncompleteConfigurationFile('Missing the general section')\n general = self._replace_data_types(dict(self._conf.items('general')))\n self._conf.remove_section('general')\n plugin = []\n for section in self._conf.sections():\n plugin.append(dict(self._conf.items(section)))\n plugin[-1].update({'name': section})\n plugin[-1] = self._replace_data_types(plugin[-1])\n return (plugin, general)", "response": "Reload the configuration from the file."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreplace strings with appropriate data types.", "response": "def _replace_data_types(dictionary):\n \"\"\"\n Replaces strings with appropriate data types (int, boolean).\n Also replaces the human readable logging levels with the integer form.\n\n dictionary: A dictionary returned from the config file.\n \"\"\"\n logging_levels = {'NONE': 0, 'NULL': 0, 'DEBUG': 10, 'INFO': 20,\n 'WARNING': 30, 'ERROR': 40, 'CRITICAL': 50}\n for k, v in dictionary.items():\n if v in ['true', 'True', 'on']:\n dictionary[k] = True\n elif v in ['false', 'False', 'off']:\n dictionary[k] = False\n elif k == 'log_file' and '~' in v:\n dictionary[k] = v.replace('~', os.path.expanduser('~'))\n elif v in logging_levels:\n dictionary[k] = logging_levels[v]\n elif isinstance(v, str) and v.isnumeric():\n dictionary[k] = int(v)\n elif ',' in v:\n dictionary[k] = [x.strip() for x in v.split(',')]\n return dictionary"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef main():\n coloredlogs.install(syslog=True)\n # Command line option defaults.\n rotation_scheme = {}\n kw = dict(include_list=[], exclude_list=[])\n parallel = False\n use_sudo = False\n # Internal state.\n selected_locations = []\n # Parse the command line arguments.\n try:\n options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [\n 'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=',\n 'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed',\n 'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=',\n 'verbose', 'quiet', 'help',\n ])\n for option, value in options:\n if option in ('-M', '--minutely'):\n rotation_scheme['minutely'] = coerce_retention_period(value)\n elif option in ('-H', '--hourly'):\n rotation_scheme['hourly'] = coerce_retention_period(value)\n elif option in ('-d', '--daily'):\n rotation_scheme['daily'] = coerce_retention_period(value)\n elif option in ('-w', '--weekly'):\n rotation_scheme['weekly'] = coerce_retention_period(value)\n elif option in ('-m', '--monthly'):\n rotation_scheme['monthly'] = coerce_retention_period(value)\n elif option in ('-y', '--yearly'):\n rotation_scheme['yearly'] = coerce_retention_period(value)\n elif option in ('-I', '--include'):\n kw['include_list'].append(value)\n elif option in ('-x', '--exclude'):\n kw['exclude_list'].append(value)\n elif option in ('-j', '--parallel'):\n parallel = True\n elif option in ('-p', '--prefer-recent'):\n kw['prefer_recent'] = True\n elif option in ('-r', '--relaxed'):\n kw['strict'] = False\n elif option in ('-i', '--ionice'):\n value = validate_ionice_class(value.lower().strip())\n kw['io_scheduling_class'] = value\n elif option in ('-c', '--config'):\n kw['config_file'] = parse_path(value)\n elif option in ('-u', '--use-sudo'):\n use_sudo = True\n elif option in ('-n', '--dry-run'):\n logger.info(\"Performing a dry run (because of %s option) ..\", option)\n kw['dry_run'] = True\n elif option in ('-C', '--removal-command'):\n removal_command = shlex.split(value)\n logger.info(\"Using custom removal command: %s\", removal_command)\n kw['removal_command'] = removal_command\n elif option in ('-v', '--verbose'):\n coloredlogs.increase_verbosity()\n elif option in ('-q', '--quiet'):\n coloredlogs.decrease_verbosity()\n elif option in ('-h', '--help'):\n usage(__doc__)\n return\n else:\n assert False, \"Unhandled option! (programming error)\"\n if rotation_scheme:\n logger.verbose(\"Rotation scheme defined on command line: %s\", rotation_scheme)\n if arguments:\n # Rotation of the locations given on the command line.\n location_source = 'command line arguments'\n selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments)\n else:\n # Rotation of all configured locations.\n location_source = 'configuration file'\n selected_locations.extend(\n location for location, rotation_scheme, options\n in load_config_file(configuration_file=kw.get('config_file'), expand=True)\n )\n # Inform the user which location(s) will be rotated.\n if selected_locations:\n logger.verbose(\"Selected %s based on %s:\",\n pluralize(len(selected_locations), \"location\"),\n location_source)\n for number, location in enumerate(selected_locations, start=1):\n logger.verbose(\" %i. %s\", number, location)\n else:\n # Show the usage message when no directories are given nor configured.\n logger.verbose(\"No location(s) to rotate selected.\")\n usage(__doc__)\n return\n except Exception as e:\n logger.error(\"%s\", e)\n sys.exit(1)\n # Rotate the backups in the selected directories.\n program = RotateBackups(rotation_scheme, **kw)\n if parallel:\n program.rotate_concurrent(*selected_locations)\n else:\n for location in selected_locations:\n program.rotate_backups(location)", "response": "Command line interface for the rotate - backups program."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_battery_state(self, prop):\n with open(os.path.join(self.options['battery_path'], prop), 'r') as f:\n return f.readline().strip()", "response": "Return the first line from the file located at battery_path / prop as a\n string."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef coerce_location(value, **options):\n # Location objects pass through untouched.\n if not isinstance(value, Location):\n # Other values are expected to be strings.\n if not isinstance(value, string_types):\n msg = \"Expected Location object or string, got %s instead!\"\n raise ValueError(msg % type(value))\n # Try to parse a remote location.\n ssh_alias, _, directory = value.partition(':')\n if ssh_alias and directory and '/' not in ssh_alias:\n options['ssh_alias'] = ssh_alias\n else:\n directory = value\n # Create the location object.\n value = Location(\n context=create_context(**options),\n directory=parse_path(directory),\n )\n return value", "response": "Coerce a string to a Location object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef coerce_retention_period(value):\n # Numbers pass through untouched.\n if not isinstance(value, numbers.Number):\n # Other values are expected to be strings.\n if not isinstance(value, string_types):\n msg = \"Expected string, got %s instead!\"\n raise ValueError(msg % type(value))\n # Check for the literal string `always'.\n value = value.strip()\n if value.lower() == 'always':\n value = 'always'\n else:\n # Evaluate other strings as expressions.\n value = simple_eval(value)\n if not isinstance(value, numbers.Number):\n msg = \"Expected numeric result, got %s instead!\"\n raise ValueError(msg % type(value))\n return value", "response": "Coerce a retention period to a Python value."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading a configuration file with backup directories and rotation schemes.", "response": "def load_config_file(configuration_file=None, expand=True):\n \"\"\"\n Load a configuration file with backup directories and rotation schemes.\n\n :param configuration_file: Override the pathname of the configuration file\n to load (a string or :data:`None`).\n :param expand: :data:`True` to expand filename patterns to their matches,\n :data:`False` otherwise.\n :returns: A generator of tuples with four values each:\n\n 1. An execution context created using :mod:`executor.contexts`.\n 2. The pathname of a directory with backups (a string).\n 3. A dictionary with the rotation scheme.\n 4. A dictionary with additional options.\n :raises: :exc:`~exceptions.ValueError` when `configuration_file` is given\n but doesn't exist or can't be loaded.\n\n This function is used by :class:`RotateBackups` to discover user defined\n rotation schemes and by :mod:`rotate_backups.cli` to discover directories\n for which backup rotation is configured. When `configuration_file` isn't\n given :class:`~update_dotdee.ConfigLoader` is used to search for\n configuration files in the following locations:\n\n - ``/etc/rotate-backups.ini`` and ``/etc/rotate-backups.d/*.ini``\n - ``~/.rotate-backups.ini`` and ``~/.rotate-backups.d/*.ini``\n - ``~/.config/rotate-backups.ini`` and ``~/.config/rotate-backups.d/*.ini``\n\n All of the available configuration files are loaded in the order given\n above, so that sections in user-specific configuration files override\n sections by the same name in system-wide configuration files.\n \"\"\"\n expand_notice_given = False\n if configuration_file:\n loader = ConfigLoader(available_files=[configuration_file], strict=True)\n else:\n loader = ConfigLoader(program_name='rotate-backups', strict=False)\n for section in loader.section_names:\n items = dict(loader.get_options(section))\n context_options = {}\n if coerce_boolean(items.get('use-sudo')):\n context_options['sudo'] = True\n if items.get('ssh-user'):\n context_options['ssh_user'] = items['ssh-user']\n location = coerce_location(section, **context_options)\n rotation_scheme = dict((name, coerce_retention_period(items[name]))\n for name in SUPPORTED_FREQUENCIES\n if name in items)\n options = dict(include_list=split(items.get('include-list', '')),\n exclude_list=split(items.get('exclude-list', '')),\n io_scheduling_class=items.get('ionice'),\n strict=coerce_boolean(items.get('strict', 'yes')),\n prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))\n # Don't override the value of the 'removal_command' property unless the\n # 'removal-command' configuration file option has a value set.\n if items.get('removal-command'):\n options['removal_command'] = shlex.split(items['removal-command'])\n # Expand filename patterns?\n if expand and location.have_wildcards:\n logger.verbose(\"Expanding filename pattern %s on %s ..\", location.directory, location.context)\n if location.is_remote and not expand_notice_given:\n logger.notice(\"Expanding remote filename patterns (may be slow) ..\")\n expand_notice_given = True\n for match in sorted(location.context.glob(location.directory)):\n if location.context.is_directory(match):\n logger.verbose(\"Matched directory: %s\", match)\n expanded = Location(context=location.context, directory=match)\n yield expanded, rotation_scheme, options\n else:\n logger.verbose(\"Ignoring match (not a directory): %s\", match)\n else:\n yield location, rotation_scheme, options"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrotate the backups in a directory according to a flexible rotation scheme.", "response": "def rotate_backups(directory, rotation_scheme, **options):\n \"\"\"\n Rotate the backups in a directory according to a flexible rotation scheme.\n\n .. note:: This function exists to preserve backwards compatibility with\n older versions of the `rotate-backups` package where all of the\n logic was exposed as a single function. Please refer to the\n documentation of the :class:`RotateBackups` initializer and the\n :func:`~RotateBackups.rotate_backups()` method for an explanation\n of this function's parameters.\n \"\"\"\n program = RotateBackups(rotation_scheme=rotation_scheme, **options)\n program.rotate_backups(directory)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef rotate_concurrent(self, *locations, **kw):\n timer = Timer()\n pool = CommandPool(concurrency=10)\n logger.info(\"Scanning %s ..\", pluralize(len(locations), \"backup location\"))\n for location in locations:\n for cmd in self.rotate_backups(location, prepare=True, **kw):\n pool.add(cmd)\n if pool.num_commands > 0:\n backups = pluralize(pool.num_commands, \"backup\")\n logger.info(\"Preparing to rotate %s (in parallel) ..\", backups)\n pool.run()\n logger.info(\"Successfully rotated %s in %s.\", backups, timer)", "response": "Rotate the backups in the given locations concurrently."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrotate the backups in a given location according to a flexible rotation scheme.", "response": "def rotate_backups(self, location, load_config=True, prepare=False):\n \"\"\"\n Rotate the backups in a directory according to a flexible rotation scheme.\n\n :param location: Any value accepted by :func:`coerce_location()`.\n :param load_config: If :data:`True` (so by default) the rotation scheme\n and other options can be customized by the user in\n a configuration file. In this case the caller's\n arguments are only used when the configuration file\n doesn't define a configuration for the location.\n :param prepare: If this is :data:`True` (not the default) then\n :func:`rotate_backups()` will prepare the required\n rotation commands without running them.\n :returns: A list with the rotation commands\n (:class:`~executor.ExternalCommand` objects).\n :raises: :exc:`~exceptions.ValueError` when the given location doesn't\n exist, isn't readable or isn't writable. The third check is\n only performed when dry run isn't enabled.\n\n This function binds the main methods of the :class:`RotateBackups`\n class together to implement backup rotation with an easy to use Python\n API. If you're using `rotate-backups` as a Python API and the default\n behavior is not satisfactory, consider writing your own\n :func:`rotate_backups()` function based on the underlying\n :func:`collect_backups()`, :func:`group_backups()`,\n :func:`apply_rotation_scheme()` and\n :func:`find_preservation_criteria()` methods.\n \"\"\"\n rotation_commands = []\n location = coerce_location(location)\n # Load configuration overrides by user?\n if load_config:\n location = self.load_config_file(location)\n # Collect the backups in the given directory.\n sorted_backups = self.collect_backups(location)\n if not sorted_backups:\n logger.info(\"No backups found in %s.\", location)\n return\n # Make sure the directory is writable.\n if not self.dry_run:\n location.ensure_writable()\n most_recent_backup = sorted_backups[-1]\n # Group the backups by the rotation frequencies.\n backups_by_frequency = self.group_backups(sorted_backups)\n # Apply the user defined rotation scheme.\n self.apply_rotation_scheme(backups_by_frequency, most_recent_backup.timestamp)\n # Find which backups to preserve and why.\n backups_to_preserve = self.find_preservation_criteria(backups_by_frequency)\n # Apply the calculated rotation scheme.\n for backup in sorted_backups:\n friendly_name = backup.pathname\n if not location.is_remote:\n # Use human friendly pathname formatting for local backups.\n friendly_name = format_path(backup.pathname)\n if backup in backups_to_preserve:\n matching_periods = backups_to_preserve[backup]\n logger.info(\"Preserving %s (matches %s retention %s) ..\",\n friendly_name, concatenate(map(repr, matching_periods)),\n \"period\" if len(matching_periods) == 1 else \"periods\")\n else:\n logger.info(\"Deleting %s ..\", friendly_name)\n if not self.dry_run:\n # Copy the list with the (possibly user defined) removal command.\n removal_command = list(self.removal_command)\n # Add the pathname of the backup as the final argument.\n removal_command.append(backup.pathname)\n # Construct the command object.\n command = location.context.prepare(\n command=removal_command,\n group_by=(location.ssh_alias, location.mount_point),\n ionice=self.io_scheduling_class,\n )\n rotation_commands.append(command)\n if not prepare:\n timer = Timer()\n command.wait()\n logger.verbose(\"Deleted %s in %s.\", friendly_name, timer)\n if len(backups_to_preserve) == len(sorted_backups):\n logger.info(\"Nothing to do! (all backups preserved)\")\n return rotation_commands"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nloads a configuration file and options from a configuration file.", "response": "def load_config_file(self, location):\n \"\"\"\n Load a rotation scheme and other options from a configuration file.\n\n :param location: Any value accepted by :func:`coerce_location()`.\n :returns: The configured or given :class:`Location` object.\n \"\"\"\n location = coerce_location(location)\n for configured_location, rotation_scheme, options in load_config_file(self.config_file, expand=False):\n if configured_location.match(location):\n logger.verbose(\"Loading configuration for %s ..\", location)\n if rotation_scheme:\n self.rotation_scheme = rotation_scheme\n for name, value in options.items():\n if value:\n setattr(self, name, value)\n # Create a new Location object based on the directory of the\n # given location and the execution context of the configured\n # location, because:\n #\n # 1. The directory of the configured location may be a filename\n # pattern whereas we are interested in the expanded name.\n #\n # 2. The execution context of the given location may lack some\n # details of the configured location.\n return Location(\n context=configured_location.context,\n directory=location.directory,\n )\n logger.verbose(\"No configuration found for %s.\", location)\n return location"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncollects the backups at the given location.", "response": "def collect_backups(self, location):\n \"\"\"\n Collect the backups at the given location.\n\n :param location: Any value accepted by :func:`coerce_location()`.\n :returns: A sorted :class:`list` of :class:`Backup` objects (the\n backups are sorted by their date).\n :raises: :exc:`~exceptions.ValueError` when the given directory doesn't\n exist or isn't readable.\n \"\"\"\n backups = []\n location = coerce_location(location)\n logger.info(\"Scanning %s for backups ..\", location)\n location.ensure_readable()\n for entry in natsort(location.context.list_entries(location.directory)):\n match = TIMESTAMP_PATTERN.search(entry)\n if match:\n if self.exclude_list and any(fnmatch.fnmatch(entry, p) for p in self.exclude_list):\n logger.verbose(\"Excluded %s (it matched the exclude list).\", entry)\n elif self.include_list and not any(fnmatch.fnmatch(entry, p) for p in self.include_list):\n logger.verbose(\"Excluded %s (it didn't match the include list).\", entry)\n else:\n try:\n backups.append(Backup(\n pathname=os.path.join(location.directory, entry),\n timestamp=datetime.datetime(*(int(group, 10) for group in match.groups('0'))),\n ))\n except ValueError as e:\n logger.notice(\"Ignoring %s due to invalid date (%s).\", entry, e)\n else:\n logger.debug(\"Failed to match time stamp in filename: %s\", entry)\n if backups:\n logger.info(\"Found %i timestamped backups in %s.\", len(backups), location)\n return sorted(backups)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef group_backups(self, backups):\n backups_by_frequency = dict((frequency, collections.defaultdict(list)) for frequency in SUPPORTED_FREQUENCIES)\n for b in backups:\n backups_by_frequency['minutely'][(b.year, b.month, b.day, b.hour, b.minute)].append(b)\n backups_by_frequency['hourly'][(b.year, b.month, b.day, b.hour)].append(b)\n backups_by_frequency['daily'][(b.year, b.month, b.day)].append(b)\n backups_by_frequency['weekly'][(b.year, b.week)].append(b)\n backups_by_frequency['monthly'][(b.year, b.month)].append(b)\n backups_by_frequency['yearly'][b.year].append(b)\n return backups_by_frequency", "response": "Groups the backups into a nested dictionary."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef apply_rotation_scheme(self, backups_by_frequency, most_recent_backup):\n if not self.rotation_scheme:\n raise ValueError(\"Refusing to use empty rotation scheme! (all backups would be deleted)\")\n for frequency, backups in backups_by_frequency.items():\n # Ignore frequencies not specified by the user.\n if frequency not in self.rotation_scheme:\n backups.clear()\n else:\n # Reduce the number of backups in each time slot of this\n # rotation frequency to a single backup (the oldest one or the\n # newest one).\n for period, backups_in_period in backups.items():\n index = -1 if self.prefer_recent else 0\n selected_backup = sorted(backups_in_period)[index]\n backups[period] = [selected_backup]\n # Check if we need to rotate away backups in old periods.\n retention_period = self.rotation_scheme[frequency]\n if retention_period != 'always':\n # Remove backups created before the minimum date of this\n # rotation frequency? (relative to the most recent backup)\n if self.strict:\n minimum_date = most_recent_backup - SUPPORTED_FREQUENCIES[frequency] * retention_period\n for period, backups_in_period in list(backups.items()):\n for backup in backups_in_period:\n if backup.timestamp < minimum_date:\n backups_in_period.remove(backup)\n if not backups_in_period:\n backups.pop(period)\n # If there are more periods remaining than the user\n # requested to be preserved we delete the oldest one(s).\n items_to_preserve = sorted(backups.items())[-retention_period:]\n backups_by_frequency[frequency] = dict(items_to_preserve)", "response": "This method applies the user defined rotation scheme to the result of group_backups method."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncollecting the criteria used to decide which backups to preserve.", "response": "def find_preservation_criteria(self, backups_by_frequency):\n \"\"\"\n Collect the criteria used to decide which backups to preserve.\n\n :param backups_by_frequency: A :class:`dict` in the format generated by\n :func:`group_backups()` which has been\n processed by :func:`apply_rotation_scheme()`.\n :returns: A :class:`dict` with :class:`Backup` objects as keys and\n :class:`list` objects containing strings (rotation\n frequencies) as values.\n \"\"\"\n backups_to_preserve = collections.defaultdict(list)\n for frequency, delta in ORDERED_FREQUENCIES:\n for period in backups_by_frequency[frequency].values():\n for backup in period:\n backups_to_preserve[backup].append(frequency)\n return backups_to_preserve"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the pathname of the mount point of the current user s virtual machine.", "response": "def mount_point(self):\n \"\"\"\n The pathname of the mount point of :attr:`directory` (a string or :data:`None`).\n\n If the ``stat --format=%m ...`` command that is used to determine the\n mount point fails, the value of this property defaults to :data:`None`.\n This enables graceful degradation on e.g. Mac OS X whose ``stat``\n implementation is rather bare bones compared to GNU/Linux.\n \"\"\"\n try:\n return self.context.capture('stat', '--format=%m', self.directory, silent=True)\n except ExternalCommandFailed:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmake sure the location exists.", "response": "def ensure_exists(self):\n \"\"\"Make sure the location exists.\"\"\"\n if not self.context.is_directory(self.directory):\n # This can also happen when we don't have permission to one of the\n # parent directories so we'll point that out in the error message\n # when it seems applicable (so as not to confuse users).\n if self.context.have_superuser_privileges:\n msg = \"The directory %s doesn't exist!\"\n raise ValueError(msg % self)\n else:\n raise ValueError(compact(\"\"\"\n The directory {location} isn't accessible, most likely\n because it doesn't exist or because of permissions. If\n you're sure the directory exists you can use the\n --use-sudo option.\n \"\"\", location=self))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmaking sure the location exists and is readable.", "response": "def ensure_readable(self):\n \"\"\"Make sure the location exists and is readable.\"\"\"\n self.ensure_exists()\n if not self.context.is_readable(self.directory):\n if self.context.have_superuser_privileges:\n msg = \"The directory %s isn't readable!\"\n raise ValueError(msg % self)\n else:\n raise ValueError(compact(\"\"\"\n The directory {location} isn't readable, most likely\n because of permissions. Consider using the --use-sudo\n option.\n \"\"\", location=self))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ensure_writable(self):\n self.ensure_exists()\n if not self.context.is_writable(self.directory):\n if self.context.have_superuser_privileges:\n msg = \"The directory %s isn't writable!\"\n raise ValueError(msg % self)\n else:\n raise ValueError(compact(\"\"\"\n The directory {location} isn't writable, most likely due\n to permissions. Consider using the --use-sudo option.\n \"\"\", location=self))", "response": "Make sure the directory exists and is writable."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks if the given location matches the current directory.", "response": "def match(self, location):\n \"\"\"\n Check if the given location \"matches\".\n\n :param location: The :class:`Location` object to try to match.\n :returns: :data:`True` if the two locations are on the same system and\n the :attr:`directory` can be matched as a filename pattern or\n a literal match on the normalized pathname.\n \"\"\"\n if self.ssh_alias != location.ssh_alias:\n # Never match locations on other systems.\n return False\n elif self.have_wildcards:\n # Match filename patterns using fnmatch().\n return fnmatch.fnmatch(location.directory, self.directory)\n else:\n # Compare normalized directory pathnames.\n self = os.path.normpath(self.directory)\n other = os.path.normpath(location.directory)\n return self == other"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef setup_file_logger(filename, formatting, log_level):\n logger = logging.getLogger()\n # If a stream handler has been attached, remove it.\n if logger.handlers:\n logger.removeHandler(logger.handlers[0])\n handler = logging.FileHandler(filename)\n logger.addHandler(handler)\n formatter = logging.Formatter(*formatting)\n handler.setFormatter(formatter)\n logger.setLevel(log_level)\n handler.setLevel(log_level)\n return logger", "response": "A helper function for creating a file logger."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\noutput data to stdout without buffering.", "response": "def output_to_bar(self, message, comma=True):\n \"\"\"\n Outputs data to stdout, without buffering.\n\n message: A string containing the data to be output.\n comma: Whether or not a comma should be placed at the end of the output.\n \"\"\"\n if comma:\n message += ','\n sys.stdout.write(message + '\\n')\n sys.stdout.flush()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef reload(self):\n logging.debug('Reloading config file as files have been modified.')\n self.config.plugin, self.config.general = self.config.reload()\n logging.debug('Reloading plugins as files have been modified.')\n self.loader = plugin_manager.PluginLoader(\n self._plugin_path, self.config.plugin)\n self._plugin_mod_time = os.path.getmtime(self._plugin_path)\n self._config_mod_time = os.path.getmtime(self._config_file_path)", "response": "Reload the installed plugins and the configuration file. This is called\n when either the plugins or config get updated."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a thread for each plugin and lets the thread_manager handle it.", "response": "def run_plugins(self):\n \"\"\"\n Creates a thread for each plugin and lets the thread_manager handle it.\n \"\"\"\n for obj in self.loader.objects:\n # Reserve a slot in the output_dict in order to ensure that the\n # items are in the correct order.\n self.output_dict[obj.output_options['name']] = None\n self.thread_manager.add_thread(obj.main, obj.options['interval'])"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nmonitors if the config file or plugins are updated. Also outputs the JSON data generated by the plugins, without needing to poll the threads.", "response": "def run(self):\n \"\"\"\n Monitors if the config file or plugins are updated. Also outputs the\n JSON data generated by the plugins, without needing to poll the threads.\n \"\"\"\n self.run_plugins()\n while True:\n # Reload plugins and config if either the config file or plugin\n # directory are modified.\n if self._config_mod_time != os.path.getmtime(self._config_file_path) or \\\n self._plugin_mod_time != os.path.getmtime(self._plugin_path):\n self.thread_manager.kill_all_threads()\n self.output_dict.clear()\n self.reload()\n self.run_plugins()\n self.output_to_bar(json.dumps(self._remove_empty_output()))\n time.sleep(self.config.general['interval'])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _remove_empty_output(self):\n output = []\n for key in self.output_dict:\n if self.output_dict[key] is not None and 'full_text' in self.output_dict[key]:\n output.append(self.output_dict[key])\n return output", "response": "Remove empty output from the output_dict."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a dictionary of all fields in the given model that match the filter.", "response": "def field_dict(self, model):\n \"\"\"\n Helper function that returns a dictionary of all fields in the given\n model. If self.field_filter is set, it only includes the fields that\n match the filter.\n \"\"\"\n if self.field_filter:\n return dict(\n [(f.name, f) for f in model._meta.fields\n if self.field_filter(f)]\n )\n else:\n return dict(\n [(f.name, f) for f in model._meta.fields\n if not f.rel and\n not f.primary_key and\n not f.unique and\n not isinstance(f, (models.AutoField, models.TextField))]\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalls the main function of a plugin and mutates the output dict with its return value.", "response": "def run(self):\n \"\"\"\n Calls the main function of a plugin and mutates the output dict\n with its return value. Provides an easy way to change the output\n whilst not needing to constantly poll a queue in another thread and\n allowing plugin's to manage their own intervals.\n \"\"\"\n self.running = True\n while self.running:\n ret = self.func()\n self.output_dict[ret['name']] = ret\n time.sleep(self.interval)\n return"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_thread(self, func, interval):\n t = Thread(func, interval, self.output_dict)\n t.start()\n self._thread_pool.append(t)", "response": "Adds a thread to the thread pool."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _compile_files(self):\n for f in glob.glob(os.path.join(self.dir_path, '*.py')):\n # Check for compiled Python files that aren't in the __pycache__.\n if not os.path.isfile(os.path.join(self.dir_path, f + 'c')):\n compileall.compile_dir(self.dir_path, quiet=True)\n logging.debug('Compiled plugins as a new plugin has been added.')\n return\n # Recompile if there are newer plugins.\n elif os.path.getmtime(os.path.join(self.dir_path, f)) > os.path.getmtime(\n os.path.join(self.dir_path, f + 'c')):\n compileall.compile_dir(self.dir_path, quiet=True)\n logging.debug('Compiled plugins as a plugin has been changed.')\n return", "response": "Compile the python plugin files."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads a compiled plugin and returns a module object.", "response": "def _load_compiled(self, file_path):\n \"\"\"\n Accepts a path to a compiled plugin and returns a module object.\n\n file_path: A string that represents a complete file path to a compiled\n plugin.\n \"\"\"\n name = os.path.splitext(os.path.split(file_path)[-1])[0]\n plugin_directory = os.sep.join(os.path.split(file_path)[0:-1])\n compiled_directory = os.path.join(plugin_directory, '__pycache__')\n # Use glob to autocomplete the filename.\n compiled_file = glob.glob(os.path.join(compiled_directory, (name + '.*')))[0]\n plugin = imp.load_compiled(name, compiled_file)\n return plugin"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads the objects based upon the plugins that have been specified in the config file.", "response": "def load_objects(self):\n \"\"\"\n Matches the plugins that have been specified in the config file\n with the available plugins. Returns instantiated objects based upon\n the classes defined in the plugins.\n \"\"\"\n objects = []\n for settings in self._config:\n if settings['plugin'] in self.plugins:\n module = self.plugins[settings['plugin']]\n # Trusts that the only item in __all__ is the name of the\n # plugin class.\n plugin_class = getattr(module, module.__all__)\n objects.append(plugin_class(settings))\n logging.debug('Loaded a plugin object based upon {0}'.format(\n settings['plugin']))\n else:\n logging.critical('Missing plugin {0} was not found in {1}'.format(\n settings['plugin'], self.dir_path))\n raise MissingPlugin('The plugin {0} was not found in {1}'.format(\n settings['plugin'], self.dir_path))\n return objects"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nrefreshes the available plugins and creates a module object for each available plugin.", "response": "def refresh_files(self):\n \"\"\"\n Discovers the available plugins and turns each into a module object.\n This is a seperate function to allow plugins to be updated\n dynamically by other parts of the application.\n \"\"\"\n plugins = {}\n _plugin_files = glob.glob(os.path.join(self.dir_path, '[!_]*.pyc'))\n for f in glob.glob(os.path.join(self.dir_path, '[!_]*.py')):\n if not any(os.path.splitext(f)[0] == os.path.splitext(x)[0]\n for x in _plugin_files):\n logging.debug('Adding plugin {0}'.format(f))\n _plugin_files.append(f)\n for f in _plugin_files:\n plugin = self._load_compiled(f)\n plugins[plugin.__name__] = plugin\n logging.debug('Loaded module object for plugin: {0}'.format(f))\n return plugins"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nhandling main URL routing for the databrowse app.", "response": "def root(self, request, url):\n \"\"\"\n Handles main URL routing for the databrowse app.\n\n `url` is the remainder of the URL -- e.g. 'objects/3'.\n \"\"\"\n # Delegate to the appropriate method, based on the URL.\n if url is None:\n return self.main_view(request)\n try:\n plugin_name, rest_of_url = url.split('/', 1)\n except ValueError: # need more than 1 value to unpack\n plugin_name, rest_of_url = url, None\n try:\n plugin = self.plugins[plugin_name]\n except KeyError:\n raise http.Http404('A plugin with the requested name '\n 'does not exist.')\n return plugin.model_view(request, self, rest_of_url)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nregister the given models with the given databrowse site.", "response": "def register(self, *model_list, **options):\n \"\"\"\n Registers the given model(s) with the given databrowse site.\n\n The model(s) should be Model classes, not instances.\n\n If a databrowse class isn't given, it will use DefaultModelDatabrowse\n (the default databrowse options).\n\n If a model is already registered, this will raise AlreadyRegistered.\n \"\"\"\n databrowse_class = options.pop('databrowse_class',\n DefaultModelDatabrowse)\n for model in model_list:\n if model in self.registry:\n raise AlreadyRegistered('The model %s is already registered' %\n model.__name__)\n self.registry[model] = databrowse_class"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef unregister(self, *model_list):\n for model in model_list:\n if model not in self.registry:\n raise NotRegistered('The model %s is not registered' %\n model.__name__)\n del self.registry[model]", "response": "Unregisters the given models."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nhandle main URL routing for the databrowse app.", "response": "def root(self, request, url):\n \"\"\"\n Handles main URL routing for the databrowse app.\n\n `url` is the remainder of the URL -- e.g. 'comments/comment/'.\n \"\"\"\n self.root_url = request.path[:len(request.path) - len(url)]\n url = url.rstrip('/') # Trim trailing slash, if it exists.\n\n if url == '':\n return self.index(request)\n elif '/' in url:\n return self.model_page(request, *url.split('/', 2))\n\n raise http.Http404('The requested databrowse page does not exist.')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a model - specific page.", "response": "def model_page(self, request, app_label, model_name, rest_of_url=None):\n \"\"\"\n Handles the model-specific functionality of the databrowse site,\n delegating 0, 'No samples were provided!')\n\n # Get total size of genome reference files. This is used for configuring disk size.\n genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size\n\n # 0: Generate processed BAM and BAI files for each sample\n # group preprocessing and variant calling steps in empty Job instance\n group_bam_jobs = Job()\n gvcfs = {}\n for sample in samples:\n # 0: Generate processed BAM and BAI files for each sample\n get_bam = group_bam_jobs.addChildJobFn(prepare_bam,\n sample.uuid,\n sample.url,\n config,\n paired_url=sample.paired_url,\n rg_line=sample.rg_line)\n\n # 1: Generate per sample gvcfs {uuid: gvcf_id}\n # The HaplotypeCaller disk requirement depends on the input bam, bai, the genome reference\n # files, and the output GVCF file. The output GVCF is smaller than the input BAM file.\n hc_disk = PromisedRequirement(lambda bam, bai, ref_size:\n 2 * bam.size + bai.size + ref_size,\n get_bam.rv(0),\n get_bam.rv(1),\n genome_ref_size)\n\n get_gvcf = get_bam.addFollowOnJobFn(gatk_haplotype_caller,\n get_bam.rv(0),\n get_bam.rv(1),\n config.genome_fasta, config.genome_fai, config.genome_dict,\n annotations=config.annotations,\n cores=config.cores,\n disk=hc_disk,\n memory=config.xmx,\n hc_output=config.hc_output)\n # Store cohort GVCFs in dictionary\n gvcfs[sample.uuid] = get_gvcf.rv()\n\n # Upload individual sample GVCF before genotyping to a sample specific output directory\n vqsr_name = '{}{}.g.vcf'.format(sample.uuid, config.suffix)\n get_gvcf.addChildJobFn(output_file_job,\n vqsr_name,\n get_gvcf.rv(),\n os.path.join(config.output_dir, sample.uuid),\n s3_key_path=config.ssec,\n disk=PromisedRequirement(lambda x: x.size, get_gvcf.rv()))\n\n # VQSR requires many variants in order to train a decent model. GATK recommends a minimum of\n # 30 exomes or one large WGS sample:\n # https://software.broadinstitute.org/gatk/documentation/article?id=3225\n\n filtered_vcfs = {}\n if config.joint_genotype:\n # Need to configure joint genotype in a separate function to resolve promises\n filtered_vcfs = group_bam_jobs.addFollowOnJobFn(joint_genotype_and_filter,\n gvcfs,\n config).rv()\n\n # If not joint genotyping, then iterate over cohort and genotype and filter individually.\n else:\n for uuid, gvcf_id in gvcfs.iteritems():\n filtered_vcfs[uuid] = group_bam_jobs.addFollowOnJobFn(genotype_and_filter,\n {uuid: gvcf_id},\n config).rv()\n\n job.addChild(group_bam_jobs)\n return filtered_vcfs", "response": "This function runs the GATK best practices pipeline for Germline SNP and INDEL discovery."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking for enough disk space for joint genotyping, then calls the genotype and filter pipeline function. :param JobFunctionWrappingJob job: passed automatically by Toil :param dict gvcfs: Dictionary of GVCFs {Sample ID: FileStoreID} :param Namespace config: Input parameters and reference FileStoreIDs Requires the following config attributes: config.genome_fasta FilesStoreID for reference genome fasta file config.genome_fai FilesStoreID for reference genome fasta index file config.genome_dict FilesStoreID for reference genome sequence dictionary file config.available_disk Total available disk space :returns: FileStoreID for the joint genotyped and filtered VCF file :rtype: str", "response": "def joint_genotype_and_filter(job, gvcfs, config):\n \"\"\"\n Checks for enough disk space for joint genotyping, then calls the genotype and filter pipeline function.\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param dict gvcfs: Dictionary of GVCFs {Sample ID: FileStoreID}\n :param Namespace config: Input parameters and reference FileStoreIDs\n Requires the following config attributes:\n config.genome_fasta FilesStoreID for reference genome fasta file\n config.genome_fai FilesStoreID for reference genome fasta index file\n config.genome_dict FilesStoreID for reference genome sequence dictionary file\n config.available_disk Total available disk space\n :returns: FileStoreID for the joint genotyped and filtered VCF file\n :rtype: str\n \"\"\"\n # Get the total size of genome reference files\n genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size\n\n # Require at least 2.5x the sum of the individual GVCF files\n cohort_size = sum(gvcf.size for gvcf in gvcfs.values())\n require(int(2.5 * cohort_size + genome_ref_size) < config.available_disk,\n 'There is not enough disk space to joint '\n 'genotype samples:\\n{}'.format('\\n'.join(gvcfs.keys())))\n\n job.fileStore.logToMaster('Merging cohort into a single GVCF file')\n\n return job.addChildJobFn(genotype_and_filter, gvcfs, config).rv()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef genotype_and_filter(job, gvcfs, config):\n # Get the total size of the genome reference\n genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size\n\n # GenotypeGVCF disk requirement depends on the input GVCF, the genome reference files, and\n # the output VCF file. The output VCF is smaller than the input GVCF.\n genotype_gvcf_disk = PromisedRequirement(lambda gvcf_ids, ref_size:\n 2 * sum(gvcf_.size for gvcf_ in gvcf_ids) + ref_size,\n gvcfs.values(),\n genome_ref_size)\n\n genotype_gvcf = job.addChildJobFn(gatk_genotype_gvcfs,\n gvcfs,\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n annotations=config.annotations,\n unsafe_mode=config.unsafe_mode,\n cores=config.cores,\n disk=genotype_gvcf_disk,\n memory=config.xmx)\n\n # Determine if output GVCF has multiple samples\n if len(gvcfs) == 1:\n uuid = gvcfs.keys()[0]\n else:\n uuid = 'joint_genotyped'\n\n genotyped_filename = '%s.genotyped%s.vcf' % (uuid, config.suffix)\n genotype_gvcf.addChildJobFn(output_file_job,\n genotyped_filename,\n genotype_gvcf.rv(),\n os.path.join(config.output_dir, uuid),\n s3_key_path=config.ssec,\n disk=PromisedRequirement(lambda x: x.size, genotype_gvcf.rv()))\n\n if config.run_vqsr:\n if not config.joint_genotype:\n job.fileStore.logToMaster('WARNING: Running VQSR without joint genotyping.')\n joint_genotype_vcf = genotype_gvcf.addFollowOnJobFn(vqsr_pipeline,\n uuid,\n genotype_gvcf.rv(),\n config)\n else:\n joint_genotype_vcf = genotype_gvcf.addFollowOnJobFn(hard_filter_pipeline,\n uuid,\n genotype_gvcf.rv(),\n config)\n return joint_genotype_vcf.rv()", "response": "This function takes a dictionary of GVCFs and runs either the VQSR or hard filtering pipeline."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef annotate_vcfs(job, vcfs, config):\n job.fileStore.logToMaster('Running Oncotator on the following samples:\\n%s' % '\\n'.join(vcfs.keys()))\n for uuid, vcf_id in vcfs.iteritems():\n # The Oncotator disk requirement depends on the input VCF, the Oncotator database\n # and the output VCF. The annotated VCF will be significantly larger than the input VCF.\n onco_disk = PromisedRequirement(lambda vcf, db: 3 * vcf.size + db.size,\n vcf_id,\n config.oncotator_db)\n\n annotated_vcf = job.addChildJobFn(run_oncotator,\n vcf_id,\n config.oncotator_db,\n disk=onco_disk,\n cores=config.cores,\n memory=config.xmx)\n\n output_dir = os.path.join(config.output_dir, uuid)\n filename = '{}.oncotator{}.vcf'.format(uuid, config.suffix)\n annotated_vcf.addChildJobFn(output_file_job,\n filename,\n annotated_vcf.rv(),\n output_dir,\n s3_key_path=config.ssec,\n disk=PromisedRequirement(lambda x: x.size, annotated_vcf.rv()))", "response": "Runs Oncotator on a group of VCF files."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_manifest(path_to_manifest):\n bam_re = r\"^(?P\\S+)\\s(?P\\S+[bsc][r]?am)\"\n fq_re = r\"^(?P\\S+)\\s(?P\\S+)\\s(?P\\S+)?\\s?(?P@RG\\S+)\"\n samples = []\n with open(path_to_manifest, 'r') as f:\n for line in f.readlines():\n line = line.strip()\n if line.startswith('#'):\n continue\n bam_match = re.match(bam_re, line)\n fastq_match = re.match(fq_re, line)\n if bam_match:\n uuid = bam_match.group('uuid')\n url = bam_match.group('url')\n paired_url = None\n rg_line = None\n require('.bam' in url.lower(),\n 'Expected .bam extension:\\n{}:\\t{}'.format(uuid, url))\n elif fastq_match:\n uuid = fastq_match.group('uuid')\n url = fastq_match.group('url')\n paired_url = fastq_match.group('paired_url')\n rg_line = fastq_match.group('rg_line')\n require('.fq' in url.lower() or '.fastq' in url.lower(),\n 'Expected .fq extension:\\n{}:\\t{}'.format(uuid, url))\n else:\n raise ValueError('Could not parse entry in manifest: %s\\n%s' % (f.name, line))\n # Checks that URL has a scheme\n require(urlparse(url).scheme, 'Invalid URL passed for {}'.format(url))\n samples.append(GermlineSample(uuid, url, paired_url, rg_line))\n return samples", "response": "Parses the manifest file for Toil Germline Pipeline and returns a list of namedtuples."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndownload shared reference files for the given pipeline and returns updated config with shared files downloaded", "response": "def download_shared_files(job, config):\n \"\"\"\n Downloads shared reference files for Toil Germline pipeline\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param Namespace config: Pipeline configuration options\n :return: Updated config with shared fileStoreIDS\n :rtype: Namespace\n \"\"\"\n job.fileStore.logToMaster('Downloading shared reference files')\n shared_files = {'genome_fasta', 'genome_fai', 'genome_dict'}\n nonessential_files = {'genome_fai', 'genome_dict'}\n\n # Download necessary files for pipeline configuration\n if config.run_bwa:\n shared_files |= {'amb', 'ann', 'bwt', 'pac', 'sa', 'alt'}\n nonessential_files.add('alt')\n if config.preprocess:\n shared_files |= {'g1k_indel', 'mills', 'dbsnp'}\n if config.run_vqsr:\n shared_files |= {'g1k_snp', 'mills', 'dbsnp', 'hapmap', 'omni'}\n if config.run_oncotator:\n shared_files.add('oncotator_db')\n for name in shared_files:\n try:\n url = getattr(config, name, None)\n if url is None:\n continue\n setattr(config, name, job.addChildJobFn(download_url_job,\n url,\n name=name,\n s3_key_path=config.ssec,\n disk='15G' # Estimated reference file size\n ).rv())\n finally:\n if getattr(config, name, None) is None and name not in nonessential_files:\n raise ValueError(\"Necessary configuration parameter is missing:\\n{}\".format(name))\n return job.addFollowOnJobFn(reference_preprocessing, config).rv()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a genome fasta index and sequence dictionary file if not already present in the pipeline config.", "response": "def reference_preprocessing(job, config):\n \"\"\"\n Creates a genome fasta index and sequence dictionary file if not already present in the pipeline config.\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param Namespace config: Pipeline configuration options and shared files.\n Requires FileStoreID for genome fasta file as config.genome_fasta\n :return: Updated config with reference index files\n :rtype: Namespace\n \"\"\"\n job.fileStore.logToMaster('Preparing Reference Files')\n genome_id = config.genome_fasta\n if getattr(config, 'genome_fai', None) is None:\n config.genome_fai = job.addChildJobFn(run_samtools_faidx,\n genome_id,\n cores=config.cores).rv()\n if getattr(config, 'genome_dict', None) is None:\n config.genome_dict = job.addChildJobFn(run_picard_create_sequence_dictionary,\n genome_id,\n cores=config.cores,\n memory=config.xmx).rv()\n return config"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprepares BAM file for GATK preprocessing.", "response": "def prepare_bam(job, uuid, url, config, paired_url=None, rg_line=None):\n \"\"\"\n Prepares BAM file for Toil germline pipeline.\n\n Steps in pipeline\n 0: Download and align BAM or FASTQ sample\n 1: Sort BAM\n 2: Index BAM\n 3: Run GATK preprocessing pipeline (Optional)\n - Uploads preprocessed BAM to output directory\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param str uuid: Unique identifier for the sample\n :param str url: URL or local path to BAM file or FASTQs\n :param Namespace config: Configuration options for pipeline\n Requires the following config attributes:\n config.genome_fasta FilesStoreID for reference genome fasta file\n config.genome_fai FilesStoreID for reference genome fasta index file\n config.genome_dict FilesStoreID for reference genome sequence dictionary file\n config.g1k_indel FileStoreID for 1000G INDEL resource file\n config.mills FileStoreID for Mills resource file\n config.dbsnp FileStoreID for dbSNP resource file\n config.suffix Suffix added to output filename\n config.output_dir URL or local path to output directory\n config.ssec Path to key file for SSE-C encryption\n config.cores Number of cores for each job\n config.xmx Java heap size in bytes\n :param str|None paired_url: URL or local path to paired FASTQ file, default is None\n :param str|None rg_line: RG line for BWA alignment (i.e. @RG\\tID:foo\\tSM:bar), default is None\n :return: BAM and BAI FileStoreIDs\n :rtype: tuple\n \"\"\"\n # 0: Align FASTQ or realign BAM\n if config.run_bwa:\n get_bam = job.wrapJobFn(setup_and_run_bwakit,\n uuid,\n url,\n rg_line,\n config,\n paired_url=paired_url).encapsulate()\n\n # 0: Download BAM\n elif '.bam' in url.lower():\n job.fileStore.logToMaster(\"Downloading BAM: %s\" % uuid)\n get_bam = job.wrapJobFn(download_url_job,\n url,\n name='toil.bam',\n s3_key_path=config.ssec,\n disk=config.file_size).encapsulate()\n else:\n raise ValueError('Could not generate BAM file for %s\\n'\n 'Provide a FASTQ URL and set run-bwa or '\n 'provide a BAM URL that includes .bam extension.' % uuid)\n\n # 1: Sort BAM file if necessary\n # Realigning BAM file shuffles read order\n if config.sorted and not config.run_bwa:\n sorted_bam = get_bam\n\n else:\n # The samtools sort disk requirement depends on the input bam, the tmp files, and the\n # sorted output bam.\n sorted_bam_disk = PromisedRequirement(lambda bam: 3 * bam.size, get_bam.rv())\n sorted_bam = get_bam.addChildJobFn(run_samtools_sort,\n get_bam.rv(),\n cores=config.cores,\n disk=sorted_bam_disk)\n\n # 2: Index BAM\n # The samtools index disk requirement depends on the input bam and the output bam index\n index_bam_disk = PromisedRequirement(lambda bam: bam.size, sorted_bam.rv())\n index_bam = job.wrapJobFn(run_samtools_index, sorted_bam.rv(), disk=index_bam_disk)\n\n job.addChild(get_bam)\n sorted_bam.addChild(index_bam)\n\n if config.preprocess:\n preprocess = job.wrapJobFn(run_gatk_preprocessing,\n sorted_bam.rv(),\n index_bam.rv(),\n config.genome_fasta,\n config.genome_dict,\n config.genome_fai,\n config.g1k_indel,\n config.mills,\n config.dbsnp,\n memory=config.xmx,\n cores=config.cores).encapsulate()\n sorted_bam.addChild(preprocess)\n index_bam.addChild(preprocess)\n\n # Update output BAM promises\n output_bam_promise = preprocess.rv(0)\n output_bai_promise = preprocess.rv(1)\n\n # Save processed BAM\n output_dir = os.path.join(config.output_dir, uuid)\n filename = '{}.preprocessed{}.bam'.format(uuid, config.suffix)\n output_bam = job.wrapJobFn(output_file_job,\n filename,\n preprocess.rv(0),\n output_dir,\n s3_key_path=config.ssec)\n preprocess.addChild(output_bam)\n\n else:\n output_bam_promise = sorted_bam.rv()\n output_bai_promise = index_bam.rv()\n\n return output_bam_promise, output_bai_promise"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef setup_and_run_bwakit(job, uuid, url, rg_line, config, paired_url=None):\n bwa_config = deepcopy(config)\n bwa_config.uuid = uuid\n bwa_config.rg_line = rg_line\n\n # bwa_alignment uses a different naming convention\n bwa_config.ref = config.genome_fasta\n bwa_config.fai = config.genome_fai\n\n # Determine if sample is a FASTQ or BAM file using the file extension\n basename, ext = os.path.splitext(url)\n ext = ext.lower()\n if ext == '.gz':\n _, ext = os.path.splitext(basename)\n ext = ext.lower()\n\n # The pipeline currently supports FASTQ and BAM files\n require(ext in ['.fq', '.fastq', '.bam'],\n 'Please use .fq or .bam file extensions:\\n%s' % url)\n\n # Download fastq files\n samples = []\n input1 = job.addChildJobFn(download_url_job,\n url,\n name='file1',\n s3_key_path=config.ssec,\n disk=config.file_size)\n\n samples.append(input1.rv())\n\n # If the extension is for a BAM file, then configure bwakit to realign the BAM file.\n if ext == '.bam':\n bwa_config.bam = input1.rv()\n else:\n bwa_config.r1 = input1.rv()\n\n # Download the paired FASTQ URL\n if paired_url:\n input2 = job.addChildJobFn(download_url_job,\n paired_url,\n name='file2',\n s3_key_path=config.ssec,\n disk=config.file_size)\n samples.append(input2.rv())\n bwa_config.r2 = input2.rv()\n\n # The bwakit disk requirement depends on the size of the input files and the index\n # Take the sum of the input files and scale it by a factor of 4\n bwa_index_size = sum([getattr(config, index_file).size\n for index_file in ['amb', 'ann', 'bwt', 'pac', 'sa', 'alt']\n if getattr(config, index_file, None) is not None])\n\n bwakit_disk = PromisedRequirement(lambda lst, index_size:\n int(4 * sum(x.size for x in lst) + index_size),\n samples,\n bwa_index_size)\n\n return job.addFollowOnJobFn(run_bwakit,\n bwa_config,\n sort=False, # BAM files are sorted later in the pipeline\n trim=config.trim,\n cores=config.cores,\n disk=bwakit_disk).rv()", "response": "Downloads and runs bwakit for the given sample identifier."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nuses GATK HaplotypeCaller to identify SNPs and INDELs. Outputs variants in a Genomic VCF file. :param JobFunctionWrappingJob job: passed automatically by Toil :param str bam: FileStoreID for BAM file :param str bai: FileStoreID for BAM index file :param str ref: FileStoreID for reference genome fasta file :param str ref_dict: FileStoreID for reference sequence dictionary file :param str fai: FileStoreID for reference fasta index file :param list[str] annotations: List of GATK variant annotations, default is None :param float emit_threshold: Minimum phred-scale confidence threshold for a variant to be emitted, default is 10.0 :param float call_threshold: Minimum phred-scale confidence threshold for a variant to be called, default is 30.0 :param bool unsafe_mode: If True, runs gatk UNSAFE mode: \"-U ALLOW_SEQ_DICT_INCOMPATIBILITY\" :param str hc_output: URL or local path to pre-cooked VCF file, default is None :return: FileStoreID for GVCF file :rtype: str", "response": "def gatk_haplotype_caller(job,\n bam, bai,\n ref, fai, ref_dict,\n annotations=None,\n emit_threshold=10.0, call_threshold=30.0,\n unsafe_mode=False,\n hc_output=None):\n \"\"\"\n Uses GATK HaplotypeCaller to identify SNPs and INDELs. Outputs variants in a Genomic VCF file.\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param str bam: FileStoreID for BAM file\n :param str bai: FileStoreID for BAM index file\n :param str ref: FileStoreID for reference genome fasta file\n :param str ref_dict: FileStoreID for reference sequence dictionary file\n :param str fai: FileStoreID for reference fasta index file\n :param list[str] annotations: List of GATK variant annotations, default is None\n :param float emit_threshold: Minimum phred-scale confidence threshold for a variant to be emitted, default is 10.0\n :param float call_threshold: Minimum phred-scale confidence threshold for a variant to be called, default is 30.0\n :param bool unsafe_mode: If True, runs gatk UNSAFE mode: \"-U ALLOW_SEQ_DICT_INCOMPATIBILITY\"\n :param str hc_output: URL or local path to pre-cooked VCF file, default is None\n :return: FileStoreID for GVCF file\n :rtype: str\n \"\"\"\n job.fileStore.logToMaster('Running GATK HaplotypeCaller')\n\n inputs = {'genome.fa': ref,\n 'genome.fa.fai': fai,\n 'genome.dict': ref_dict,\n 'input.bam': bam,\n 'input.bam.bai': bai}\n\n work_dir = job.fileStore.getLocalTempDir()\n for name, file_store_id in inputs.iteritems():\n job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))\n\n # Call GATK -- HaplotypeCaller with parameters to produce a genomic VCF file:\n # https://software.broadinstitute.org/gatk/documentation/article?id=2803\n command = ['-T', 'HaplotypeCaller',\n '-nct', str(job.cores),\n '-R', 'genome.fa',\n '-I', 'input.bam',\n '-o', 'output.g.vcf',\n '-stand_call_conf', str(call_threshold),\n '-stand_emit_conf', str(emit_threshold),\n '-variant_index_type', 'LINEAR',\n '-variant_index_parameter', '128000',\n '--genotyping_mode', 'Discovery',\n '--emitRefConfidence', 'GVCF']\n\n if unsafe_mode:\n command = ['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'] + command\n\n if annotations:\n for annotation in annotations:\n command.extend(['-A', annotation])\n\n # Uses docker_call mock mode to replace output with hc_output file\n outputs = {'output.g.vcf': hc_output}\n docker_call(job=job, work_dir=work_dir,\n env={'JAVA_OPTS': '-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)},\n parameters=command,\n tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',\n inputs=inputs.keys(),\n outputs=outputs,\n mock=True if outputs['output.g.vcf'] else False)\n return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.g.vcf'))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef main():\n # Define Parser object and add to jobTree\n parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)\n\n # Generate subparsers\n subparsers = parser.add_subparsers(dest='command')\n subparsers.add_parser('generate-config',\n help='Generates an editable config in the current working directory.')\n subparsers.add_parser('generate-manifest',\n help='Generates an editable manifest in the current working directory.')\n subparsers.add_parser('generate',\n help='Generates a config and manifest in the current working directory.')\n\n # Run subparser\n parser_run = subparsers.add_parser('run', help='Runs the GATK germline pipeline')\n parser_run.add_argument('--config',\n required=True,\n type=str,\n help='Path to the (filled in) config file, generated with '\n '\"generate-config\".')\n parser_run.add_argument('--manifest',\n type=str,\n help='Path to the (filled in) manifest file, generated with '\n '\"generate-manifest\".\\nDefault value: \"%(default)s\".')\n parser_run.add_argument('--sample',\n default=None,\n nargs=2,\n type=str,\n help='Input sample identifier and BAM file URL or local path')\n parser_run.add_argument('--output-dir',\n default=None,\n help='Path/URL to output directory')\n parser_run.add_argument('-s', '--suffix',\n default=None,\n help='Additional suffix to add to the names of the output files')\n parser_run.add_argument('--preprocess-only',\n action='store_true',\n help='Only runs preprocessing steps')\n\n Job.Runner.addToilOptions(parser_run)\n options = parser.parse_args()\n\n cwd = os.getcwd()\n if options.command == 'generate-config' or options.command == 'generate':\n generate_file(os.path.join(cwd, 'config-toil-germline.yaml'), generate_config)\n if options.command == 'generate-manifest' or options.command == 'generate':\n generate_file(os.path.join(cwd, 'manifest-toil-germline.tsv'), generate_manifest)\n elif options.command == 'run':\n # Program checks\n for program in ['curl', 'docker']:\n require(next(which(program)),\n program + ' must be installed on every node.'.format(program))\n\n require(os.path.exists(options.config), '{} not found. Please run \"generate-config\"'.format(options.config))\n\n # Read sample manifest\n samples = []\n if options.manifest:\n samples.extend(parse_manifest(options.manifest))\n\n # Add BAM sample from command line\n if options.sample:\n uuid, url = options.sample\n # samples tuple: (uuid, url, paired_url, rg_line)\n # BAM samples should not have as paired URL or read group line\n samples.append(GermlineSample(uuid, url, None, None))\n\n require(len(samples) > 0,\n 'No samples were detected in the manifest or on the command line')\n\n # Parse inputs\n inputs = {x.replace('-', '_'): y for x, y in\n yaml.load(open(options.config).read()).iteritems()}\n\n required_fields = {'genome_fasta',\n 'output_dir',\n 'run_bwa',\n 'sorted',\n 'snp_filter_annotations',\n 'indel_filter_annotations',\n 'preprocess',\n 'preprocess_only',\n 'run_vqsr',\n 'joint_genotype',\n 'run_oncotator',\n 'cores',\n 'file_size',\n 'xmx',\n 'suffix'}\n\n input_fields = set(inputs.keys())\n require(input_fields > required_fields,\n 'Missing config parameters:\\n{}'.format(', '.join(required_fields - input_fields)))\n\n if inputs['output_dir'] is None:\n inputs['output_dir'] = options.output_dir\n\n require(inputs['output_dir'] is not None,\n 'Missing output directory PATH/URL')\n\n if inputs['suffix'] is None:\n inputs['suffix'] = options.suffix if options.suffix else ''\n\n if inputs['preprocess_only'] is None:\n inputs['preprocess_only'] = options.preprocess_only\n\n if inputs['run_vqsr']:\n # Check that essential VQSR parameters are present\n vqsr_fields = {'g1k_snp', 'mills', 'dbsnp', 'hapmap', 'omni'}\n require(input_fields > vqsr_fields,\n 'Missing parameters for VQSR:\\n{}'.format(', '.join(vqsr_fields - input_fields)))\n\n # Check that hard filtering parameters are present. If only running preprocessing steps, then we do\n # not need filtering information.\n elif not inputs['preprocess_only']:\n hard_filter_fields = {'snp_filter_name', 'snp_filter_expression',\n 'indel_filter_name', 'indel_filter_expression'}\n require(input_fields > hard_filter_fields,\n 'Missing parameters for hard filtering:\\n{}'.format(', '.join(hard_filter_fields - input_fields)))\n\n # Check for falsey hard filtering parameters\n for hard_filter_field in hard_filter_fields:\n require(inputs[hard_filter_field], 'Missing %s value for hard filtering, '\n 'got %s.' % (hard_filter_field, inputs[hard_filter_field]))\n\n # Set resource parameters\n inputs['xmx'] = human2bytes(inputs['xmx'])\n inputs['file_size'] = human2bytes(inputs['file_size'])\n inputs['cores'] = int(inputs['cores'])\n\n inputs['annotations'] = set(inputs['snp_filter_annotations'] + inputs['indel_filter_annotations'])\n\n # HaplotypeCaller test data for testing\n inputs['hc_output'] = inputs.get('hc_output', None)\n\n # It is a toil-scripts convention to store input parameters in a Namespace object\n config = argparse.Namespace(**inputs)\n\n root = Job.wrapJobFn(run_gatk_germline_pipeline, samples, config)\n Job.Runner.startToil(root, options)", "response": "This function is the main function of the GATK germline pipeline. It is responsible for running the GATK germline pipeline with variant filtering and annotation."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nloop over the sample_ids in the manifest creating child jobs for each sample_id", "response": "def sample_loop(job, uuid_list, inputs):\n \"\"\"\n Loops over the sample_ids (uuids) in the manifest, creating child jobs to process each\n \"\"\"\n\n for uuid_rg in uuid_list:\n\n uuid_items = uuid_rg.split(',')\n uuid = uuid_items[0]\n rg_line = None\n if len(uuid_items) > 1:\n rg_line = uuid_items[1]\n\n job.addChildJobFn(static_dag, uuid, rg_line, inputs)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprefer this here as it allows us to pull the job functions from other jobs without rewrapping the job functions back together. bwa_inputs: Input arguments to be passed to BWA. adam_inputs: Input arguments to be passed to ADAM. gatk_preprocess_inputs: Input arguments to be passed to GATK preprocessing. gatk_adam_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of ADAM preprocessing. gatk_gatk_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of GATK preprocessing.", "response": "def static_dag(job, uuid, rg_line, inputs):\n \"\"\"\n Prefer this here as it allows us to pull the job functions from other jobs\n without rewrapping the job functions back together.\n\n bwa_inputs: Input arguments to be passed to BWA.\n adam_inputs: Input arguments to be passed to ADAM.\n gatk_preprocess_inputs: Input arguments to be passed to GATK preprocessing.\n gatk_adam_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of ADAM preprocessing.\n gatk_gatk_call_inputs: Input arguments to be passed to GATK haplotype caller for the result of GATK preprocessing.\n \"\"\"\n\n # get work directory\n work_dir = job.fileStore.getLocalTempDir()\n\n inputs.cpu_count = cpu_count()\n inputs.maxCores = sys.maxint\n args = {'uuid': uuid,\n 's3_bucket': inputs.s3_bucket,\n 'sequence_dir': inputs.sequence_dir,\n 'dir_suffix': inputs.dir_suffix}\n\n # get head BWA alignment job function and encapsulate it\n inputs.rg_line = rg_line\n inputs.output_dir = 's3://{s3_bucket}/alignment{dir_suffix}'.format(**args)\n bwa = job.wrapJobFn(download_reference_files,\n inputs,\n [[uuid,\n ['s3://{s3_bucket}/{sequence_dir}/{uuid}_1.fastq.gz'.format(**args),\n 's3://{s3_bucket}/{sequence_dir}/{uuid}_2.fastq.gz'.format(**args)]]]).encapsulate()\n\n # get head ADAM preprocessing job function and encapsulate it\n adam_preprocess = job.wrapJobFn(static_adam_preprocessing_dag,\n inputs,\n 's3://{s3_bucket}/alignment{dir_suffix}/{uuid}.bam'.format(**args),\n 's3://{s3_bucket}/analysis{dir_suffix}/{uuid}'.format(**args),\n suffix='.adam').encapsulate()\n\n # Configure options for Toil Germline pipeline. This function call only runs the preprocessing steps.\n gatk_preprocessing_inputs = copy.deepcopy(inputs)\n gatk_preprocessing_inputs.suffix = '.gatk'\n gatk_preprocessing_inputs.preprocess = True\n gatk_preprocessing_inputs.preprocess_only = True\n gatk_preprocessing_inputs.output_dir = 's3://{s3_bucket}/analysis{dir_suffix}'.format(**args)\n\n # get head GATK preprocessing job function and encapsulate it\n gatk_preprocess = job.wrapJobFn(run_gatk_germline_pipeline,\n GermlineSample(uuid,\n 's3://{s3_bucket}/alignment{dir_suffix}/{uuid}.bam'.format(**args),\n None, # Does not require second URL or RG_Line\n None),\n gatk_preprocessing_inputs).encapsulate()\n\n # Configure options for Toil Germline pipeline for preprocessed ADAM BAM file.\n adam_call_inputs = inputs\n adam_call_inputs.suffix = '.adam'\n adam_call_inputs.sorted = True\n adam_call_inputs.preprocess = False\n adam_call_inputs.run_vqsr = False\n adam_call_inputs.joint_genotype = False\n adam_call_inputs.output_dir = 's3://{s3_bucket}/analysis{dir_suffix}'.format(**args)\n\n # get head GATK haplotype caller job function for the result of ADAM preprocessing and encapsulate it\n gatk_adam_call = job.wrapJobFn(run_gatk_germline_pipeline,\n GermlineSample(uuid,\n 's3://{s3_bucket}/analysis{dir_suffix}/{uuid}/{uuid}.adam.bam'.format(**args),\n None,\n None),\n adam_call_inputs).encapsulate()\n\n # Configure options for Toil Germline pipeline for preprocessed GATK BAM file.\n gatk_call_inputs = copy.deepcopy(inputs)\n gatk_call_inputs.sorted = True\n gatk_call_inputs.preprocess = False\n gatk_call_inputs.run_vqsr = False\n gatk_call_inputs.joint_genotype = False\n gatk_call_inputs.output_dir = 's3://{s3_bucket}/analysis{dir_suffix}'.format(**args)\n\n # get head GATK haplotype caller job function for the result of GATK preprocessing and encapsulate it\n gatk_gatk_call = job.wrapJobFn(run_gatk_germline_pipeline,\n GermlineSample(uuid,\n 'S3://{s3_bucket}/analysis{dir_suffix}/{uuid}/{uuid}.gatk.bam'.format(**args),\n None, None),\n gatk_call_inputs).encapsulate()\n\n # wire up dag\n if not inputs.skip_alignment:\n job.addChild(bwa)\n\n if (inputs.pipeline_to_run == \"adam\" or\n inputs.pipeline_to_run == \"both\"):\n\n if inputs.skip_preprocessing:\n\n job.addChild(gatk_adam_call)\n else:\n if inputs.skip_alignment:\n job.addChild(adam_preprocess)\n else:\n bwa.addChild(adam_preprocess)\n\n adam_preprocess.addChild(gatk_adam_call)\n\n if (inputs.pipeline_to_run == \"gatk\" or\n inputs.pipeline_to_run == \"both\"):\n\n if inputs.skip_preprocessing:\n\n job.addChild(gatk_gatk_call)\n else:\n if inputs.skip_alignment:\n job.addChild(gatk_preprocess)\n else:\n bwa.addChild(gatk_preprocess)\n\n gatk_preprocess.addChild(gatk_gatk_call)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef main():\n # Define Parser object and add to Toil\n if mock_mode():\n usage_msg = 'You have the TOIL_SCRIPTS_MOCK_MODE environment variable set, so this pipeline ' \\\n 'will run in mock mode. To disable mock mode, set TOIL_SCRIPTS_MOCK_MODE=0'\n else:\n usage_msg = None\n\n parser = argparse.ArgumentParser(usage=usage_msg)\n subparsers = parser.add_subparsers(dest='command')\n subparsers.add_parser('generate-config', help='Generates an editable config in the current working directory.')\n subparsers.add_parser('generate-manifest', help='Generates an editable manifest in the current working directory.')\n subparsers.add_parser('generate', help='Generates a config and manifest in the current working directory.')\n # Run subparser \n parser_run = subparsers.add_parser('run', help='Runs the ADAM/GATK pipeline')\n default_config = 'adam-gatk-mock.config' if mock_mode() else 'adam-gatk.config'\n default_manifest = 'adam-gatk-mock-manifest.csv' if mock_mode() else 'adam-gatk-manifest.csv'\n parser_run.add_argument('--config', default=default_config, type=str,\n help='Path to the (filled in) config file, generated with \"generate-config\".')\n parser_run.add_argument('--manifest', default=default_manifest,\n type=str, help='Path to the (filled in) manifest file, generated with \"generate-manifest\". '\n '\\nDefault value: \"%(default)s\".')\n Job.Runner.addToilOptions(parser_run)\n args = parser.parse_args()\n\n cwd = os.getcwd()\n if args.command == 'generate-config' or args.command == 'generate':\n generate_file(os.path.join(cwd, default_config), generate_config)\n if args.command == 'generate-manifest' or args.command == 'generate':\n generate_file(os.path.join(cwd, default_manifest), generate_manifest)\n # Pipeline execution\n elif args.command == 'run':\n require(os.path.exists(args.config), '{} not found. Please run '\n 'generate-config'.format(args.config))\n if not hasattr(args, 'sample'):\n require(os.path.exists(args.manifest), '{} not found and no samples provided. Please '\n 'run \"generate-manifest\"'.format(args.manifest))\n # Parse config\n parsed_config = {x.replace('-', '_'): y for x, y in yaml.load(open(args.config).read()).iteritems()}\n inputs = argparse.Namespace(**parsed_config)\n\n # Parse manifest file\n uuid_list = []\n with open(args.manifest) as f_manifest:\n for line in f_manifest:\n if not line.isspace() and not line.startswith('#'):\n uuid_list.append(line.strip())\n\n inputs.sort = False\n if not inputs.dir_suffix:\n inputs.dir_suffix = ''\n if not inputs.s3_bucket:\n inputs.s3_bucket = ''\n\n if inputs.master_ip and inputs.num_nodes:\n raise ValueError(\"Exactly one of master_ip (%s) and num_nodes (%d) must be provided.\" %\n (inputs.master_ip, inputs.num_nodes))\n\n if not hasattr(inputs, 'master_ip') and inputs.num_nodes <= 1:\n raise ValueError('num_nodes allocates one Spark/HDFS master and n-1 workers, and thus must be greater '\n 'than 1. %d was passed.' % inputs.num_nodes)\n\n if (inputs.pipeline_to_run != \"adam\" and\n inputs.pipeline_to_run != \"gatk\" and\n inputs.pipeline_to_run != \"both\"):\n raise ValueError(\"pipeline_to_run must be either 'adam', 'gatk', or 'both'. %s was passed.\" % inputs.pipeline_to_run)\n\n Job.Runner.startToil(Job.wrapJobFn(sample_loop, uuid_list, inputs), args)", "response": "This is the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function of the main function."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating a unique key for a BD2K URL.", "response": "def generate_unique_key(master_key_path, url):\n \"\"\"\n Input1: Path to the BD2K Master Key (for S3 Encryption)\n Input2: S3 URL (e.g. https://s3-us-west-2.amazonaws.com/cgl-driver-projects-encrypted/wcdt/exome_bams/DTB-111-N.bam)\n\n Returns: 32-byte unique key generated for that URL\n \"\"\"\n with open(master_key_path, 'r') as f:\n master_key = f.read()\n assert len(master_key) == 32, 'Invalid Key! Must be 32 characters. ' \\\n 'Key: {}, Length: {}'.format(master_key, len(master_key))\n new_key = hashlib.sha256(master_key + url).digest()\n assert len(new_key) == 32, 'New key is invalid and is not 32 characters: {}'.format(new_key)\n return new_key"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndownloading encrypted file from S3 using the given key", "response": "def download_encrypted_file(work_dir, url, key_path, name):\n \"\"\"\n Downloads encrypted file from S3\n\n Input1: Working directory\n Input2: S3 URL to be downloaded\n Input3: Path to key necessary for decryption\n Input4: name of file to be downloaded\n \"\"\"\n file_path = os.path.join(work_dir, name)\n key = generate_unique_key(key_path, url)\n encoded_key = base64.b64encode(key)\n encoded_key_md5 = base64.b64encode(hashlib.md5(key).digest())\n h1 = 'x-amz-server-side-encryption-customer-algorithm:AES256'\n h2 = 'x-amz-server-side-encryption-customer-key:{}'.format(encoded_key)\n h3 = 'x-amz-server-side-encryption-customer-key-md5:{}'.format(encoded_key_md5)\n try:\n subprocess.check_call(['curl', '-fs', '--retry', '5', '-H', h1, '-H', h2, '-H', h3, url, '-o', file_path])\n except OSError:\n raise RuntimeError('Failed to find \"curl\". Install via \"apt-get install curl\"')\n assert os.path.exists(file_path)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef return_input_paths(job, work_dir, ids, *args):\n paths = OrderedDict()\n for name in args:\n if not os.path.exists(os.path.join(work_dir, name)):\n file_path = job.fileStore.readGlobalFile(ids[name], os.path.join(work_dir, name))\n else:\n file_path = os.path.join(work_dir, name)\n paths[name] = file_path\n if len(args) == 1:\n return file_path\n\n return paths.values()", "response": "Returns the paths to the files requested by the FileStore"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmove files from work_dir to output_dir", "response": "def move_to_output_dir(work_dir, output_dir, uuid=None, files=list()):\n \"\"\"\n Moves files from work_dir to output_dir\n\n Input1: Working directory\n Input2: Output directory\n Input3: UUID to be preprended onto file name\n Input4: list of file names to be moved from working dir to output dir\n \"\"\"\n for fname in files:\n if uuid is None:\n shutil.move(os.path.join(work_dir, fname), os.path.join(output_dir, fname))\n else:\n shutil.move(os.path.join(work_dir, fname), os.path.join(output_dir, '{}.{}'.format(uuid, fname)))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndownloading shared files and places them in the jobstore.", "response": "def batch_start(job, input_args):\n \"\"\"\n Downloads shared files that are used by all samples for alignment and places them in the jobstore.\n \"\"\"\n shared_files = ['ref.fa', 'ref.fa.amb', 'ref.fa.ann', 'ref.fa.bwt', 'ref.fa.pac', 'ref.fa.sa', 'ref.fa.fai']\n shared_ids = {}\n for fname in shared_files:\n url = input_args[fname]\n shared_ids[fname] = job.addChildJobFn(download_from_url, url, fname).rv()\n job.addFollowOnJobFn(spawn_batch_jobs, shared_ids, input_args)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nspawn an alignment job for every sample in the input configuration file", "response": "def spawn_batch_jobs(job, shared_ids, input_args):\n \"\"\"\n Spawns an alignment job for every sample in the input configuration file\n \"\"\"\n samples = []\n config = input_args['config']\n with open(config, 'r') as f_in:\n for line in f_in:\n line = line.strip().split(',')\n uuid = line[0]\n urls = line[1:]\n samples.append((uuid, urls))\n for sample in samples:\n job.addChildJobFn(alignment, shared_ids, input_args, sample, cores=32, memory='20 G', disk='100 G')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrunning BWA and then Bamsort on the supplied fastqs for this sample Input1: Toil Job instance Input2: jobstore id dictionary Input3: Input arguments dictionary Input4: Sample tuple -- contains uuid and urls for the sample", "response": "def alignment(job, ids, input_args, sample):\n \"\"\"\n Runs BWA and then Bamsort on the supplied fastqs for this sample\n\n Input1: Toil Job instance\n Input2: jobstore id dictionary\n Input3: Input arguments dictionary\n Input4: Sample tuple -- contains uuid and urls for the sample\n \"\"\"\n uuid, urls = sample\n # ids['bam'] = job.fileStore.getEmptyFileStoreID()\n work_dir = job.fileStore.getLocalTempDir()\n output_dir = input_args['output_dir']\n key_path = input_args['ssec']\n cores = multiprocessing.cpu_count()\n\n # I/O\n return_input_paths(job, work_dir, ids, 'ref.fa', 'ref.fa.amb', 'ref.fa.ann',\n 'ref.fa.bwt', 'ref.fa.pac', 'ref.fa.sa', 'ref.fa.fai')\n # Get fastqs associated with this sample\n for url in urls:\n download_encrypted_file(work_dir, url, key_path, os.path.basename(url))\n\n # Parameters for BWA and Bamsort\n docker_cmd = ['docker', 'run', '--rm', '-v', '{}:/data'.format(work_dir)]\n\n bwa_command = [\"jvivian/bwa\",\n \"mem\",\n \"-R\", \"@RG\\tID:{0}\\tPL:Illumina\\tSM:{0}\\tLB:KapaHyper\".format(uuid),\n \"-T\", str(0),\n \"-t\", str(cores),\n \"/data/ref.fa\"] + [os.path.join('/data/', os.path.basename(x)) for x in urls]\n\n bamsort_command = [\"jeltje/biobambam\",\n \"/usr/local/bin/bamsort\",\n \"inputformat=sam\",\n \"level=1\",\n \"inputthreads={}\".format(cores),\n \"outputthreads={}\".format(cores),\n \"calmdnm=1\",\n \"calmdnmrecompindetonly=1\",\n \"calmdnmreference=/data/ref.fa\",\n \"I=/data/{}\".format(uuid + '.sam')]\n # Piping the output to a file handle\n with open(os.path.join(work_dir, uuid + '.sam'), 'w') as f_out:\n subprocess.check_call(docker_cmd + bwa_command, stdout=f_out)\n\n with open(os.path.join(work_dir, uuid + '.bam'), 'w') as f_out:\n subprocess.check_call(docker_cmd + bamsort_command, stdout=f_out)\n\n # Save in JobStore\n # job.fileStore.updateGlobalFile(ids['bam'], os.path.join(work_dir, uuid + '.bam'))\n ids['bam'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, uuid + '.bam'))\n # Copy file to S3\n if input_args['s3_dir']:\n job.addChildJobFn(upload_bam_to_s3, ids, input_args, sample, cores=32, memory='20 G', disk='30 G')\n # Move file in output_dir\n if input_args['output_dir']:\n move_to_output_dir(work_dir, output_dir, uuid=None, files=[uuid + '.bam'])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef upload_bam_to_s3(job, ids, input_args, sample):\n uuid, urls = sample\n key_path = input_args['ssec']\n work_dir = job.fileStore.getLocalTempDir()\n # Parse s3_dir to get bucket and s3 path\n s3_dir = input_args['s3_dir']\n bucket_name = s3_dir.lstrip('/').split('/')[0]\n bucket_dir = '/'.join(s3_dir.lstrip('/').split('/')[1:])\n base_url = 'https://s3-us-west-2.amazonaws.com/'\n url = os.path.join(base_url, bucket_name, bucket_dir, uuid + '.bam')\n #I/O\n job.fileStore.readGlobalFile(ids['bam'], os.path.join(work_dir, uuid + '.bam'))\n # Generate keyfile for upload\n with open(os.path.join(work_dir, uuid + '.key'), 'wb') as f_out:\n f_out.write(generate_unique_key(key_path, url))\n # Commands to upload to S3 via S3AM\n s3am_command = ['s3am',\n 'upload',\n '--sse-key-file', os.path.join(work_dir, uuid + '.key'),\n 'file://{}'.format(os.path.join(work_dir, uuid + '.bam')),\n bucket_name,\n os.path.join(bucket_dir, uuid + '.bam')]\n\n subprocess.check_call(s3am_command)", "response": "Uploads BAM from sample to S3"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns GATK Variant Quality Score Recalibration. 0: Start 0 --> 1 --> 3 --> 4 --> 5 1: Recalibrate SNPs | | 2: Recalibrate INDELS +-> 2 -+ 3: Apply SNP Recalibration 4: Apply INDEL Recalibration 5: Write VCF to output directory :param JobFunctionWrappingJob job: passed automatically by Toil :param str uuid: unique sample identifier :param str vcf_id: VCF FileStoreID :param Namespace config: Pipeline configuration options and shared files Requires the following config attributes: config.genome_fasta FilesStoreID for reference genome fasta file config.genome_fai FilesStoreID for reference genome fasta index file config.genome_dict FilesStoreID for reference genome sequence dictionary file config.cores Number of cores for each job config.xmx Java heap size in bytes config.suffix Suffix for output filename config.output_dir URL or local path to output directory config.ssec Path to key file for SSE-C encryption SNP VQSR attributes: config.snp_filter_annotations List of GATK variant annotations config.hapmap FileStoreID for HapMap resource file config.omni FileStoreID for Omni resource file config.dbsnp FileStoreID for dbSNP resource file config.g1k_snp FileStoreID for 1000G SNP resource file INDEL VQSR attributes: config.indel_filter_annotations List of GATK variant annotations config.dbsnp FileStoreID for dbSNP resource file config.mills FileStoreID for Mills resource file :return: SNP and INDEL VQSR VCF FileStoreID :rtype: str", "response": "def vqsr_pipeline(job, uuid, vcf_id, config):\n \"\"\"\n Runs GATK Variant Quality Score Recalibration.\n\n 0: Start 0 --> 1 --> 3 --> 4 --> 5\n 1: Recalibrate SNPs | |\n 2: Recalibrate INDELS +-> 2 -+\n 3: Apply SNP Recalibration\n 4: Apply INDEL Recalibration\n 5: Write VCF to output directory\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param str uuid: unique sample identifier\n :param str vcf_id: VCF FileStoreID\n :param Namespace config: Pipeline configuration options and shared files\n Requires the following config attributes:\n config.genome_fasta FilesStoreID for reference genome fasta file\n config.genome_fai FilesStoreID for reference genome fasta index file\n config.genome_dict FilesStoreID for reference genome sequence dictionary file\n config.cores Number of cores for each job\n config.xmx Java heap size in bytes\n config.suffix Suffix for output filename\n config.output_dir URL or local path to output directory\n config.ssec Path to key file for SSE-C encryption\n\n SNP VQSR attributes:\n config.snp_filter_annotations List of GATK variant annotations\n config.hapmap FileStoreID for HapMap resource file\n config.omni FileStoreID for Omni resource file\n config.dbsnp FileStoreID for dbSNP resource file\n config.g1k_snp FileStoreID for 1000G SNP resource file\n\n INDEL VQSR attributes:\n config.indel_filter_annotations List of GATK variant annotations\n config.dbsnp FileStoreID for dbSNP resource file\n config.mills FileStoreID for Mills resource file\n\n :return: SNP and INDEL VQSR VCF FileStoreID\n :rtype: str\n \"\"\"\n # Get the total size of the genome reference\n genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size\n\n # The VariantRecalibator disk requirement depends on the input VCF, the resource files,\n # the genome reference files, and the output recalibration table, tranche file, and plots.\n # The sum of these output files are less than the input VCF.\n snp_resources = ['hapmap', 'omni', 'dbsnp', 'g1k_snp']\n snp_resource_size = sum(getattr(config, resource).size for resource in snp_resources)\n snp_recal_disk = PromisedRequirement(lambda in_vcf, ref_size, resource_size:\n 2 * in_vcf.size + ref_size + resource_size,\n vcf_id,\n genome_ref_size,\n snp_resource_size)\n\n snp_recal = job.wrapJobFn(gatk_variant_recalibrator,\n 'SNP',\n vcf_id,\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n get_short_annotations(config.snp_filter_annotations),\n hapmap=config.hapmap,\n omni=config.omni,\n phase=config.g1k_snp,\n dbsnp=config.dbsnp,\n unsafe_mode=config.unsafe_mode,\n disk=snp_recal_disk,\n cores=config.cores,\n memory=config.xmx)\n\n indel_resource_size = config.mills.size + config.dbsnp.size\n indel_recal_disk = PromisedRequirement(lambda in_vcf, ref_size, resource_size:\n 2 * in_vcf.size + ref_size + resource_size,\n vcf_id,\n genome_ref_size,\n indel_resource_size)\n\n indel_recal = job.wrapJobFn(gatk_variant_recalibrator,\n 'INDEL',\n vcf_id,\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n get_short_annotations(config.indel_filter_annotations),\n dbsnp=config.dbsnp,\n mills=config.mills,\n unsafe_mode=config.unsafe_mode,\n disk=indel_recal_disk,\n cores=config.cores,\n memory=config.xmx)\n\n # The ApplyRecalibration disk requirement depends on the input VCF size, the variant\n # recalibration table, the tranche file, the genome reference file, and the output VCF.\n # This step labels variants as filtered, so the output VCF file should be slightly larger\n # than the input file. Estimate a 10% increase in the VCF file size.\n apply_snp_recal_disk = PromisedRequirement(lambda in_vcf, recal, tranche, ref_size:\n int(2.1 * in_vcf.size + recal.size + tranche.size + ref_size),\n vcf_id,\n snp_recal.rv(0),\n snp_recal.rv(1),\n genome_ref_size)\n\n apply_snp_recal = job.wrapJobFn(gatk_apply_variant_recalibration,\n 'SNP',\n vcf_id,\n snp_recal.rv(0), snp_recal.rv(1),\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n unsafe_mode=config.unsafe_mode,\n disk=apply_snp_recal_disk,\n cores=config.cores,\n memory=config.xmx)\n\n apply_indel_recal_disk = PromisedRequirement(lambda in_vcf, recal, tranche, ref_size:\n int(2.1 * in_vcf.size + recal.size + tranche.size + ref_size),\n vcf_id,\n indel_recal.rv(0),\n indel_recal.rv(1),\n genome_ref_size)\n\n apply_indel_recal = job.wrapJobFn(gatk_apply_variant_recalibration,\n 'INDEL',\n apply_snp_recal.rv(),\n indel_recal.rv(0), indel_recal.rv(1),\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n unsafe_mode=config.unsafe_mode,\n disk=apply_indel_recal_disk,\n cores=config.cores,\n memory=config.xmx)\n\n job.addChild(snp_recal)\n job.addChild(indel_recal)\n snp_recal.addChild(apply_snp_recal)\n indel_recal.addChild(apply_indel_recal)\n apply_snp_recal.addChild(apply_indel_recal)\n\n # Output recalibrated VCF\n output_dir = config.output_dir\n output_dir = os.path.join(output_dir, uuid)\n vqsr_name = '%s.vqsr%s.vcf' % (uuid, config.suffix)\n output_vqsr = job.wrapJobFn(output_file_job,\n vqsr_name,\n apply_indel_recal.rv(),\n output_dir,\n s3_key_path=config.ssec,\n disk=PromisedRequirement(lambda x: x.size, apply_indel_recal.rv()))\n apply_indel_recal.addChild(output_vqsr)\n return apply_indel_recal.rv()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert full GATK annotation name to the shortened version of the GATK annotation name.", "response": "def get_short_annotations(annotations):\n \"\"\"\n Converts full GATK annotation name to the shortened version\n :param annotations:\n :return:\n \"\"\"\n # Annotations need to match VCF header\n short_name = {'QualByDepth': 'QD',\n 'FisherStrand': 'FS',\n 'StrandOddsRatio': 'SOR',\n 'ReadPosRankSumTest': 'ReadPosRankSum',\n 'MappingQualityRankSumTest': 'MQRankSum',\n 'RMSMappingQuality': 'MQ',\n 'InbreedingCoeff': 'ID'}\n\n short_annotations = []\n for annotation in annotations:\n if annotation in short_name:\n annotation = short_name[annotation]\n short_annotations.append(annotation)\n return short_annotations"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse_sra(path_to_config):\n samples = []\n with open(path_to_config, 'r') as f:\n for line in f.readlines():\n if not line.isspace():\n samples.append(line.strip())\n return samples", "response": "Parses genetorrent config file. Returns list of samples."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntaring a group of files together into a tarball", "response": "def tarball_files(work_dir, tar_name, uuid=None, files=None):\n \"\"\"\n Tars a group of files together into a tarball\n\n work_dir: str Current Working Directory\n tar_name: str Name of tarball\n uuid: str UUID to stamp files with\n files: str(s) List of filenames to place in the tarball from working directory\n \"\"\"\n with tarfile.open(os.path.join(work_dir, tar_name), 'w:gz') as f_out:\n for fname in files:\n if uuid:\n f_out.add(os.path.join(work_dir, fname), arcname=uuid + '.' + fname)\n else:\n f_out.add(os.path.join(work_dir, fname), arcname=fname)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndownloads a sample from dbGaP then uses S3AM to transfer it to CGHub", "response": "def download_and_transfer_sample(job, input_args, samples):\n \"\"\"\n Downloads a sample from dbGaP via SRAToolKit, then uses S3AM to transfer it to S3\n\n input_args: dict Dictionary of input arguments\n analysis_id: str An analysis ID for a sample in CGHub\n \"\"\"\n if len(samples) > 1:\n a = samples[len(samples)/2:]\n b = samples[:len(samples)/2]\n job.addChildJobFn(download_and_transfer_sample, input_args, a, disk='30G')\n job.addChildJobFn(download_and_transfer_sample, input_args, b, disk='30G')\n else:\n analysis_id = samples[0]\n work_dir = job.fileStore.getLocalTempDir()\n sudo = input_args['sudo']\n # Acquire dbgap_key\n shutil.copy(input_args['dbgap_key'], os.path.join(work_dir, 'dbgap.ngc'))\n # Call to fastq-dump to pull down SRA files and convert to fastq\n if input_args['single_end']:\n parameters = [analysis_id]\n else:\n parameters = ['--split-files', analysis_id]\n docker_call(tool='quay.io/ucsc_cgl/fastq-dump:2.5.7--4577a6c1a3c94adaa0c25dd6c03518ee610433d1',\n work_dir=work_dir, tool_parameters=parameters, sudo=sudo)\n # Collect files and encapsulate into a tarball\n shutil.rmtree(os.path.join(work_dir, 'sra'))\n sample_name = analysis_id + '.tar.gz'\n if input_args['single_end']:\n r = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*.f*'))]\n tarball_files(work_dir, tar_name=sample_name, files=r)\n else:\n r1 = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*_1*'))]\n r2 = [os.path.basename(x) for x in glob.glob(os.path.join(work_dir, '*_2*'))]\n tarball_files(work_dir, tar_name=sample_name, files=r1 + r2)\n # Parse s3_dir to get bucket and s3 path\n key_path = input_args['ssec']\n s3_dir = input_args['s3_dir']\n bucket_name = s3_dir.lstrip('/').split('/')[0]\n base_url = 'https://s3-us-west-2.amazonaws.com/'\n url = os.path.join(base_url, bucket_name, sample_name)\n # Generate keyfile for upload\n with open(os.path.join(work_dir, 'temp.key'), 'wb') as f_out:\n f_out.write(generate_unique_key(key_path, url))\n # Upload to S3 via S3AM\n s3am_command = ['s3am',\n 'upload',\n '--sse-key-file', os.path.join(work_dir, 'temp.key'),\n 'file://{}'.format(os.path.join(work_dir, sample_name)),\n 's3://' + bucket_name + '/']\n subprocess.check_call(s3am_command)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ntransferring gTEX data from dbGaP (NCBI) to S3", "response": "def main():\n \"\"\"\n Transfer gTEX data from dbGaP (NCBI) to S3\n \"\"\"\n # Define Parser object and add to toil\n parser = build_parser()\n Job.Runner.addToilOptions(parser)\n args = parser.parse_args()\n # Store inputs from argparse\n inputs = {'sra': args.sra,\n 'dbgap_key': args.dbgap_key,\n 'ssec': args.ssec,\n 's3_dir': args.s3_dir,\n 'single_end': args.single_end,\n 'sudo': args.sudo}\n # Sanity checks\n if args.ssec:\n assert os.path.isfile(args.ssec)\n if args.sra:\n assert os.path.isfile(args.sra)\n if args.dbgap_key:\n assert os.path.isfile(args.dbgap_key)\n # Start Pipeline\n Job.Runner.startToil(Job.wrapJobFn(start_batch, inputs), args)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef output_file_job(job, filename, file_id, output_dir, s3_key_path=None):\n job.fileStore.logToMaster('Writing {} to {}'.format(filename, output_dir))\n work_dir = job.fileStore.getLocalTempDir()\n filepath = job.fileStore.readGlobalFile(file_id, os.path.join(work_dir, filename))\n if urlparse(output_dir).scheme == 's3':\n s3am_upload(job=job, fpath=os.path.join(work_dir, filepath),\n s3_dir=output_dir,\n s3_key_path=s3_key_path)\n elif os.path.exists(os.path.join(output_dir, filename)):\n job.fileStore.logToMaster(\"File already exists: {}\".format(filename))\n else:\n mkdir_p(output_dir)\n copy_files([filepath], output_dir)", "response": "Uploads a file from the FileStore to an output directory on the local filesystem or S3."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndownload encrypted files from S3 via header injection", "response": "def download_encrypted_file(job, input_args, name):\n \"\"\"\n Downloads encrypted files from S3 via header injection\n\n input_args: dict Input dictionary defined in main()\n name: str Symbolic name associated with file\n \"\"\"\n work_dir = job.fileStore.getLocalTempDir()\n key_path = input_args['ssec']\n file_path = os.path.join(work_dir, name)\n url = input_args[name]\n\n with open(key_path, 'r') as f:\n key = f.read()\n if len(key) != 32:\n raise RuntimeError('Invalid Key! Must be 32 bytes: {}'.format(key))\n\n key = generate_unique_key(key_path, url)\n\n encoded_key = base64.b64encode(key)\n encoded_key_md5 = base64.b64encode(hashlib.md5(key).digest())\n h1 = 'x-amz-server-side-encryption-customer-algorithm:AES256'\n h2 = 'x-amz-server-side-encryption-customer-key:{}'.format(encoded_key)\n h3 = 'x-amz-server-side-encryption-customer-key-md5:{}'.format(encoded_key_md5)\n try:\n subprocess.check_call(['curl', '-fs', '--retry', '5', '-H', h1, '-H', h2, '-H', h3, url, '-o', file_path])\n except OSError:\n raise RuntimeError('Failed to find \"curl\". Install via \"apt-get install curl\"')\n assert os.path.exists(file_path)\n return job.fileStore.writeGlobalFile(file_path)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndownload a single file from a given URL.", "response": "def download_from_url(job, url):\n \"\"\"\n Simple curl request made for a given url\n\n url: str URL to download\n \"\"\"\n work_dir = job.fileStore.getLocalTempDir()\n file_path = os.path.join(work_dir, os.path.basename(url))\n if not os.path.exists(file_path):\n if url.startswith('s3:'):\n download_from_s3_url(file_path, url)\n else:\n try:\n subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])\n except OSError:\n raise RuntimeError('Failed to find \"curl\". Install via \"apt-get install curl\"')\n assert os.path.exists(file_path)\n return job.fileStore.writeGlobalFile(file_path)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef docker_call(work_dir, tool_parameters, tool, java_opts=None, outfile=None, sudo=False):\n base_docker_call = 'docker run --log-driver=none --rm -v {}:/data'.format(work_dir).split()\n if sudo:\n base_docker_call = ['sudo'] + base_docker_call\n if java_opts:\n base_docker_call = base_docker_call + ['-e', 'JAVA_OPTS={}'.format(java_opts)]\n try:\n if outfile:\n subprocess.check_call(base_docker_call + [tool] + tool_parameters, stdout=outfile)\n else:\n subprocess.check_call(base_docker_call + [tool] + tool_parameters)\n except subprocess.CalledProcessError:\n raise RuntimeError('docker command returned a non-zero exit status. Check error logs.')\n except OSError:\n raise RuntimeError('docker not found on system. Install on all nodes.')", "response": "Makes subprocess call of a command to a docker container."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncopy files from work_dir to output_dir.", "response": "def copy_to_output_dir(work_dir, output_dir, uuid=None, files=list()):\n \"\"\"\n A list of files to move from work_dir to output_dir.\n\n work_dir: str Current working directory\n output_dir: str Output directory for files to go\n uuid: str UUID to \"stamp\" onto output files\n files: list List of files to iterate through\n \"\"\"\n for fname in files:\n if uuid is None:\n shutil.copy(os.path.join(work_dir, fname), os.path.join(output_dir, fname))\n else:\n shutil.copy(os.path.join(work_dir, fname), os.path.join(output_dir, '{}.{}'.format(uuid, fname)))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking that dependency programs are installed on every node.", "response": "def program_checks(job, input_args):\n \"\"\"\n Checks that dependency programs are installed.\n\n input_args: dict Dictionary of input arguments (from main())\n \"\"\"\n # Program checks\n for program in ['curl', 'docker', 'unzip', 'samtools']:\n assert which(program), 'Program \"{}\" must be installed on every node.'.format(program)\n job.addChildJobFn(download_shared_files, input_args)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef download_shared_files(job, input_args):\n shared_files = ['unc.bed', 'hg19.transcripts.fa', 'composite_exons.bed', 'normalize.pl', 'rsem_ref.zip',\n 'ebwt.zip', 'chromosomes.zip']\n shared_ids = {}\n for f in shared_files:\n shared_ids[f] = job.addChildJobFn(download_from_url, input_args[f]).rv()\n if input_args['config'] or input_args['config_fastq']:\n job.addFollowOnJobFn(parse_config_file, shared_ids, input_args)\n else:\n sample_path = input_args['input']\n uuid = os.path.splitext(os.path.basename(sample_path))[0]\n sample = (uuid, sample_path)\n job.addFollowOnJobFn(download_sample, shared_ids, input_args, sample)", "response": "Downloads and stores shared files in the FileStore."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses the config file and adds the pipeline for each sample.", "response": "def parse_config_file(job, ids, input_args):\n \"\"\"\n Launches pipeline for each sample.\n\n shared_ids: dict Dictionary of fileStore IDs\n input_args: dict Dictionary of input arguments\n \"\"\"\n samples = []\n config = input_args['config']\n with open(config, 'r') as f:\n for line in f.readlines():\n if not line.isspace():\n sample = line.strip().split(',')\n samples.append(sample)\n for sample in samples:\n job.addChildJobFn(download_sample, ids, input_args, sample)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef download_sample(job, ids, input_args, sample):\n if len(sample) == 2:\n uuid, sample_location = sample\n url1, url2 = None, None\n else:\n uuid, url1, url2 = sample\n sample_location = None\n # Update values unique to sample\n sample_input = dict(input_args)\n sample_input['uuid'] = uuid\n sample_input['sample.tar'] = sample_location\n if sample_input['output_dir']:\n sample_input['output_dir'] = os.path.join(input_args['output_dir'], uuid)\n sample_input['cpu_count'] = multiprocessing.cpu_count()\n job_vars = (sample_input, ids)\n # Download or locate local file and place in the jobStore\n if sample_input['input']:\n ids['sample.tar'] = job.fileStore.writeGlobalFile(os.path.abspath(sample_location))\n elif sample_input['config_fastq']:\n ids['R1.fastq'] = job.fileStore.writeGlobalFile(urlparse(url1).path)\n ids['R2.fastq'] = job.fileStore.writeGlobalFile(urlparse(url2).path)\n else:\n if sample_input['ssec']:\n ids['sample.tar'] = job.addChildJobFn(download_encrypted_file, sample_input, 'sample.tar', disk='25G').rv()\n else:\n ids['sample.tar'] = job.addChildJobFn(download_from_url, sample_input['sample.tar'], disk='25G').rv()\n job.addFollowOnJobFn(static_dag_launchpoint, job_vars)", "response": "Download a sample from the file store and place it in the jobStore."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef merge_fastqs(job, job_vars):\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n cores = input_args['cpu_count']\n single_end_reads = input_args['single_end_reads']\n # I/O\n sample = return_input_paths(job, work_dir, ids, 'sample.tar')\n # Untar File\n # subprocess.check_call(['unzip', sample, '-d', work_dir])\n subprocess.check_call(['tar', '-xvf', sample, '-C', work_dir])\n # Remove large files before creating concat versions.\n os.remove(os.path.join(work_dir, 'sample.tar'))\n # Zcat files in parallel\n if single_end_reads:\n files = sorted(glob.glob(os.path.join(work_dir, '*')))\n with open(os.path.join(work_dir, 'R1.fastq'), 'w') as f1:\n subprocess.check_call(['zcat'] + files, stdout=f1)\n # FileStore\n ids['R1.fastq'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1.fastq'))\n else:\n r1_files = sorted(glob.glob(os.path.join(work_dir, '*R1*')))\n r2_files = sorted(glob.glob(os.path.join(work_dir, '*R2*')))\n with open(os.path.join(work_dir, 'R1.fastq'), 'w') as f1:\n p1 = subprocess.Popen(['zcat'] + r1_files, stdout=f1)\n with open(os.path.join(work_dir, 'R2.fastq'), 'w') as f2:\n p2 = subprocess.Popen(['zcat'] + r2_files, stdout=f2)\n p1.wait()\n p2.wait()\n # FileStore\n ids['R1.fastq'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1.fastq'))\n ids['R2.fastq'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R2.fastq'))\n job.fileStore.deleteGlobalFile(ids['sample.tar'])\n # Spawn child job\n return job.addChildJobFn(mapsplice, job_vars, cores=cores, disk='130 G').rv()", "response": "Merge the fastqs of the Read1 and Read2 groups together."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef mapsplice(job, job_vars):\n # Unpack variables\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n cores = input_args['cpu_count']\n sudo = input_args['sudo']\n single_end_reads = input_args['single_end_reads']\n files_to_delete = ['R1.fastq']\n # I/O\n return_input_paths(job, work_dir, ids, 'ebwt.zip', 'chromosomes.zip')\n if single_end_reads:\n return_input_paths(job, work_dir, ids, 'R1.fastq')\n else:\n return_input_paths(job, work_dir, ids, 'R1.fastq', 'R2.fastq')\n files_to_delete.extend(['R2.fastq'])\n for fname in ['chromosomes.zip', 'ebwt.zip']:\n subprocess.check_call(['unzip', '-o', os.path.join(work_dir, fname), '-d', work_dir])\n # Command and call\n parameters = ['-p', str(cores),\n '-s', '25',\n '--bam',\n '--min-map-len', '50',\n '-x', '/data/ebwt',\n '-c', '/data/chromosomes',\n '-1', '/data/R1.fastq',\n '-o', '/data']\n if not single_end_reads:\n parameters.extend(['-2', '/data/R2.fastq'])\n docker_call(tool='quay.io/ucsc_cgl/mapsplice:2.1.8--dd5ac549b95eb3e5d166a5e310417ef13651994e',\n tool_parameters=parameters, work_dir=work_dir, sudo=sudo)\n # Write to FileStore\n for fname in ['alignments.bam', 'stats.txt']:\n ids[fname] = job.fileStore.writeGlobalFile(os.path.join(work_dir, fname))\n for fname in files_to_delete:\n job.fileStore.deleteGlobalFile(ids[fname])\n # Run child job\n # map_id = job.addChildJobFn(mapping_stats, job_vars).rv()\n if input_args['upload_bam_to_s3'] and input_args['s3_dir']:\n job.addChildJobFn(upload_bam_to_s3, job_vars)\n output_ids = job.addChildJobFn(add_read_groups, job_vars, disk='30 G').rv()\n return output_ids", "response": "Maps RNA - Seq reads to a reference genome."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_read_groups(job, job_vars):\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n sudo = input_args['sudo']\n # I/O\n alignments = return_input_paths(job, work_dir, ids, 'alignments.bam')\n output = os.path.join(work_dir, 'rg_alignments.bam')\n # Command and callg\n parameter = ['AddOrReplaceReadGroups',\n 'INPUT={}'.format(docker_path(alignments)),\n 'OUTPUT={}'.format(docker_path(output)),\n 'RGSM={}'.format(input_args['uuid']),\n 'RGID={}'.format(input_args['uuid']),\n 'RGLB=TruSeq',\n 'RGPL=illumina',\n 'RGPU=barcode',\n 'VALIDATION_STRINGENCY=SILENT']\n docker_call(tool='quay.io/ucsc_cgl/picardtools:1.95--dd5ac549b95eb3e5d166a5e310417ef13651994e',\n tool_parameters=parameter, work_dir=work_dir, sudo=sudo)\n # Write to FileStore\n ids['rg_alignments.bam'] = job.fileStore.writeGlobalFile(output)\n # Run child job\n return job.addChildJobFn(bamsort_and_index, job_vars, disk='30 G').rv()", "response": "This function adds read groups to the headers\n "} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsort bam file and produces index file", "response": "def bamsort_and_index(job, job_vars):\n \"\"\"\n Sorts bam file and produces index file\n\n job_vars: tuple Tuple of dictionaries: input_args and ids\n \"\"\"\n # Unpack variables\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n sudo = input_args['sudo']\n # I/O\n rg_alignments = return_input_paths(job, work_dir, ids, 'rg_alignments.bam')\n output = os.path.join(work_dir, 'sorted.bam')\n # Command -- second argument is \"Output Prefix\"\n cmd1 = ['sort', docker_path(rg_alignments), docker_path('sorted')]\n cmd2 = ['index', docker_path(output)]\n docker_call(tool='quay.io/ucsc_cgl/samtools:0.1.19--dd5ac549b95eb3e5d166a5e310417ef13651994e',\n tool_parameters=cmd1, work_dir=work_dir, sudo=sudo)\n docker_call(tool='quay.io/ucsc_cgl/samtools:0.1.19--dd5ac549b95eb3e5d166a5e310417ef13651994e',\n tool_parameters=cmd2, work_dir=work_dir, sudo=sudo)\n # Write to FileStore\n ids['sorted.bam'] = job.fileStore.writeGlobalFile(output)\n ids['sorted.bam.bai'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'sorted.bam.bai'))\n # Run child job\n output_ids = job.addChildJobFn(sort_bam_by_reference, job_vars, disk='50 G').rv()\n rseq_id = job.addChildJobFn(rseq_qc, job_vars, disk='20 G').rv()\n return rseq_id, output_ids"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef rseq_qc(job, job_vars):\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n uuid = input_args['uuid']\n sudo = input_args['sudo']\n # I/O\n return_input_paths(job, work_dir, ids, 'sorted.bam', 'sorted.bam.bai')\n # Command\n docker_call(tool='jvivian/qc', tool_parameters=['/opt/cgl-docker-lib/RseqQC_v2.sh', '/data/sorted.bam', uuid],\n work_dir=work_dir, sudo=sudo)\n # Write to FileStore\n output_files = [f for f in glob.glob(os.path.join(work_dir, '*')) if 'sorted.bam' not in f]\n tarball_files(work_dir, tar_name='qc.tar.gz', uuid=None, files=output_files)\n return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'qc.tar.gz'))", "response": "This module is used to run RseqQC on the BAM post alignment. It is used to run RseqQC on the BAM post alignment."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsorting the bam by reference", "response": "def sort_bam_by_reference(job, job_vars):\n \"\"\"\n Sorts the bam by reference\n\n job_vars: tuple Tuple of dictionaries: input_args and ids\n \"\"\"\n # Unpack variables\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n # I/O\n sorted_bam, sorted_bai = return_input_paths(job, work_dir, ids, 'sorted.bam', 'sorted.bam.bai')\n output = os.path.join(work_dir, 'sort_by_ref.bam')\n # Call: Samtools\n ref_seqs = []\n handle = subprocess.Popen([\"samtools\", \"view\", \"-H\", sorted_bam], stdout=subprocess.PIPE).stdout\n for line in handle:\n if line.startswith(\"@SQ\"):\n tmp = line.split(\"\\t\")\n chrom = tmp[1].split(\":\")[1]\n ref_seqs.append(chrom)\n handle.close()\n # Iterate through chromosomes to create mini-bams\n for chrom in ref_seqs:\n # job.addChildJobFn(sbbr_child, chrom, os.path.join(work_dir, chrom), sorted_bam)\n cmd_view = [\"samtools\", \"view\", \"-b\", sorted_bam, chrom]\n cmd_sort = [\"samtools\", \"sort\", \"-m\", \"3000000000\", \"-n\", \"-\", os.path.join(work_dir, chrom)]\n p1 = subprocess.Popen(cmd_view, stdout=subprocess.PIPE)\n subprocess.check_call(cmd_sort, stdin=p1.stdout)\n sorted_files = [os.path.join(work_dir, chrom) + '.bam' for chrom in ref_seqs]\n cmd = [\"samtools\", \"cat\", \"-o\", output] + sorted_files\n subprocess.check_call(cmd)\n # Write to FileStore\n ids['sort_by_ref.bam'] = job.fileStore.writeGlobalFile(output)\n rsem_id = job.addChildJobFn(transcriptome, job_vars, disk='30 G', memory='30 G').rv()\n exon_id = job.addChildJobFn(exon_count, job_vars, disk='30 G').rv()\n return exon_id, rsem_id"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef exon_count(job, job_vars):\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n uuid = input_args['uuid']\n sudo = input_args['sudo']\n # I/O\n sort_by_ref, normalize_pl, composite_bed = return_input_paths(job, work_dir, ids, 'sort_by_ref.bam',\n 'normalize.pl', 'composite_exons.bed')\n # Command\n tool = 'jvivian/bedtools'\n cmd_1 = ['coverage',\n '-split',\n '-abam', docker_path(sort_by_ref),\n '-b', docker_path(composite_bed)]\n cmd_2 = ['perl',\n os.path.join(work_dir, 'normalize.pl'),\n sort_by_ref,\n composite_bed]\n\n popen_docker = ['docker', 'run', '-v', '{}:/data'.format(work_dir), tool]\n if sudo:\n popen_docker = ['sudo'] + popen_docker\n p = subprocess.Popen(popen_docker + cmd_1, stdout=subprocess.PIPE)\n with open(os.path.join(work_dir, 'exon_quant'), 'w') as f:\n subprocess.check_call(cmd_2, stdin=p.stdout, stdout=f)\n\n p1 = subprocess.Popen(['cat', os.path.join(work_dir, 'exon_quant')], stdout=subprocess.PIPE)\n p2 = subprocess.Popen(['tr', '\":\"', '\"\\t\"'], stdin=p1.stdout, stdout=subprocess.PIPE)\n p3 = subprocess.Popen(['tr', '\"-\"', '\"\\t\"'], stdin=p2.stdout, stdout=subprocess.PIPE)\n with open(os.path.join(work_dir, 'exon_quant.bed'), 'w') as f:\n subprocess.check_call(['cut', '-f1-4'], stdin=p3.stdout, stdout=f)\n # Create zip, upload to fileStore, and move to output_dir as a backup\n output_files = ['exon_quant.bed', 'exon_quant']\n tarball_files(work_dir, tar_name='exon.tar.gz', uuid=uuid, files=output_files)\n return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'exon.tar.gz'))", "response": "Generates exon counts for a given set of exons."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a bam of just the transcriptome job_vars: tuple Tuple of dictionaries: input_args and ids", "response": "def transcriptome(job, job_vars):\n \"\"\"\n Creates a bam of just the transcriptome\n\n job_vars: tuple Tuple of dictionaries: input_args and ids\n \"\"\"\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n sudo = input_args['sudo']\n # I/O\n sort_by_ref, bed, hg19_fa = return_input_paths(job, work_dir, ids, 'sort_by_ref.bam',\n 'unc.bed', 'hg19.transcripts.fa')\n output = os.path.join(work_dir, 'transcriptome.bam')\n # Command\n parameters = ['sam-xlate',\n '--bed', docker_path(bed),\n '--in', docker_path(sort_by_ref),\n '--order', docker_path(hg19_fa),\n '--out', docker_path(output),\n '--xgtag',\n '--reverse']\n docker_call(tool='quay.io/ucsc_cgl/ubu:1.2--02806964cdf74bf5c39411b236b4c4e36d026843',\n tool_parameters=parameters, work_dir=work_dir, java_opts='-Xmx30g', sudo=sudo)\n # Write to FileStore\n ids['transcriptome.bam'] = job.fileStore.writeGlobalFile(output)\n # Run child job\n return job.addChildJobFn(filter_bam, job_vars, memory='30G', disk='30G').rv()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform filtering on the transcriptome bam job_vars: tuple Tuple of dictionaries: input_args and ids", "response": "def filter_bam(job, job_vars):\n \"\"\"\n Performs filtering on the transcriptome bam\n\n job_vars: tuple Tuple of dictionaries: input_args and ids\n \"\"\"\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n cores = input_args['cpu_count']\n sudo = input_args['sudo']\n # I/O\n transcriptome_bam = return_input_paths(job, work_dir, ids, 'transcriptome.bam')\n output = os.path.join(work_dir, 'filtered.bam')\n # Command\n parameters = ['sam-filter',\n '--strip-indels',\n '--max-insert', '1000',\n '--mapq', '1',\n '--in', docker_path(transcriptome_bam),\n '--out', docker_path(output)]\n docker_call(tool='quay.io/ucsc_cgl/ubu:1.2--02806964cdf74bf5c39411b236b4c4e36d026843',\n tool_parameters=parameters, work_dir=os.path.dirname(output), java_opts='-Xmx30g', sudo=sudo)\n # Write to FileStore\n ids['filtered.bam'] = job.fileStore.writeGlobalFile(output)\n # Run child job\n return job.addChildJobFn(rsem, job_vars, cores=cores, disk='30 G').rv()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun RSEM to produce counts", "response": "def rsem(job, job_vars):\n \"\"\"\n Runs RSEM to produce counts\n\n job_vars: tuple Tuple of dictionaries: input_args and ids\n \"\"\"\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n cpus = input_args['cpu_count']\n sudo = input_args['sudo']\n single_end_reads = input_args['single_end_reads']\n # I/O\n filtered_bam, rsem_ref = return_input_paths(job, work_dir, ids, 'filtered.bam', 'rsem_ref.zip')\n subprocess.check_call(['unzip', '-o', os.path.join(work_dir, 'rsem_ref.zip'), '-d', work_dir])\n output_prefix = 'rsem'\n # Make tool call to Docker\n parameters = ['--quiet',\n '--no-qualities',\n '-p', str(cpus),\n '--forward-prob', '0.5',\n '--seed-length', '25',\n '--fragment-length-mean', '-1.0',\n '--bam', docker_path(filtered_bam)]\n if not single_end_reads:\n parameters.extend(['--paired-end'])\n parameters.extend(['/data/rsem_ref/hg19_M_rCRS_ref', output_prefix])\n\n docker_call(tool='quay.io/ucsc_cgl/rsem:1.2.25--4e8d1b31d4028f464b3409c6558fb9dfcad73f88',\n tool_parameters=parameters, work_dir=work_dir, sudo=sudo)\n os.rename(os.path.join(work_dir, output_prefix + '.genes.results'), os.path.join(work_dir, 'rsem_gene.tab'))\n os.rename(os.path.join(work_dir, output_prefix + '.isoforms.results'), os.path.join(work_dir, 'rsem_isoform.tab'))\n # Write to FileStore\n ids['rsem_gene.tab'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rsem_gene.tab'))\n ids['rsem_isoform.tab'] = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rsem_isoform.tab'))\n # Run child jobs\n return job.addChildJobFn(rsem_postprocess, job_vars).rv()"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef consolidate_output(job, job_vars, output_ids):\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n uuid = input_args['uuid']\n # Retrieve IDs\n rseq_id, exon_id, rsem_id = flatten(output_ids)\n # Retrieve output file paths to consolidate\n # map_tar = job.fileStore.readGlobalFile(map_id, os.path.join(work_dir, 'map.tar.gz'))\n qc_tar = job.fileStore.readGlobalFile(rseq_id, os.path.join(work_dir, 'qc.tar.gz'))\n exon_tar = job.fileStore.readGlobalFile(exon_id, os.path.join(work_dir, 'exon.tar.gz'))\n rsem_tar = job.fileStore.readGlobalFile(rsem_id, os.path.join(work_dir, 'rsem.tar.gz'))\n # I/O\n out_tar = os.path.join(work_dir, uuid + '.tar.gz')\n # Consolidate separate tarballs\n with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:\n for tar in [rsem_tar, exon_tar, qc_tar]:\n with tarfile.open(tar, 'r') as f_in:\n for tarinfo in f_in:\n with closing(f_in.extractfile(tarinfo)) as f_in_file:\n if tar == qc_tar:\n tarinfo.name = os.path.join(uuid, 'rseq_qc', os.path.basename(tarinfo.name))\n else:\n tarinfo.name = os.path.join(uuid, os.path.basename(tarinfo.name))\n f_out.addfile(tarinfo, fileobj=f_in_file)\n # Move to output directory of selected\n if input_args['output_dir']:\n output_dir = input_args['output_dir']\n mkdir_p(output_dir)\n copy_to_output_dir(work_dir, output_dir, uuid=None, files=[uuid + '.tar.gz'])\n # Write output file to fileStore\n ids['uuid.tar.gz'] = job.fileStore.writeGlobalFile(out_tar)\n # If S3 bucket argument specified, upload to S3\n if input_args['s3_dir']:\n job.addChildJobFn(upload_output_to_s3, job_vars)", "response": "Combine the contents of separate zipped outputs into one."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nuploading the output of the job to S3 using boto.", "response": "def upload_output_to_s3(job, job_vars):\n \"\"\"\n If s3_dir is specified in arguments, file will be uploaded to S3 using boto.\n WARNING: ~/.boto credentials are necessary for this to succeed!\n\n job_vars: tuple Tuple of dictionaries: input_args and ids\n \"\"\"\n import boto\n from boto.s3.key import Key\n\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n uuid = input_args['uuid']\n # Parse s3_dir\n s3_dir = input_args['s3_dir']\n bucket_name = s3_dir.split('/')[0]\n bucket_dir = '/'.join(s3_dir.split('/')[1:])\n # I/O\n uuid_tar = return_input_paths(job, work_dir, ids, 'uuid.tar.gz')\n # Upload to S3 via boto\n conn = boto.connect_s3()\n bucket = conn.get_bucket(bucket_name)\n k = Key(bucket)\n k.key = os.path.join(bucket_dir, uuid + '.tar.gz')\n k.set_contents_from_filename(uuid_tar)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nupload bam to S3", "response": "def upload_bam_to_s3(job, job_vars):\n \"\"\"\n Upload bam to S3. Requires S3AM and a ~/.boto config file.\n \"\"\"\n input_args, ids = job_vars\n work_dir = job.fileStore.getLocalTempDir()\n uuid = input_args['uuid']\n # I/O\n job.fileStore.readGlobalFile(ids['alignments.bam'], os.path.join(work_dir, 'alignments.bam'))\n bam_path = os.path.join(work_dir, 'alignments.bam')\n sample_name = uuid + '.bam'\n # Parse s3_dir to get bucket and s3 path\n s3_dir = input_args['s3_dir']\n bucket_name = s3_dir.split('/')[0]\n bucket_dir = os.path.join('/'.join(s3_dir.split('/')[1:]), 'bam_files')\n # Upload to S3 via S3AM\n s3am_command = ['s3am',\n 'upload',\n 'file://{}'.format(bam_path),\n os.path.join('s3://', bucket_name, bucket_dir, sample_name)]\n subprocess.check_call(s3am_command)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef main():\n # Define Parser object and add to toil\n parser = build_parser()\n Job.Runner.addToilOptions(parser)\n args = parser.parse_args()\n # Store inputs from argparse\n inputs = {'config': args.config,\n 'config_fastq': args.config_fastq,\n 'input': args.input,\n 'unc.bed': args.unc,\n 'hg19.transcripts.fa': args.fasta,\n 'composite_exons.bed': args.composite_exons,\n 'normalize.pl': args.normalize,\n 'output_dir': args.output_dir,\n 'rsem_ref.zip': args.rsem_ref,\n 'chromosomes.zip': args.chromosomes,\n 'ebwt.zip': args.ebwt,\n 'ssec': args.ssec,\n 's3_dir': args.s3_dir,\n 'sudo': args.sudo,\n 'single_end_reads': args.single_end_reads,\n 'upload_bam_to_s3': args.upload_bam_to_s3,\n 'uuid': None,\n 'sample.tar': None,\n 'cpu_count': None}\n\n # Launch jobs\n Job.Runner.startToil(Job.wrapJobFn(download_shared_files, inputs), args)", "response": "This is the main function for the UNC best practice RNA - Seq analysis. It is a Toil pipeline for the UNC best practice RNA - Seq analysis. It is a Toil pipeline for the UNC best practice RNA - Seq analysis."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nremove the given file from hdfs with the given IP address.", "response": "def remove_file(master_ip, filename, spark_on_toil):\n \"\"\"\n Remove the given file from hdfs with master at the given IP address\n\n :type masterIP: MasterAddress\n \"\"\"\n master_ip = master_ip.actual\n\n ssh_call = ['ssh', '-o', 'StrictHostKeyChecking=no', master_ip]\n\n if spark_on_toil:\n output = check_output(ssh_call + ['docker', 'ps'])\n container_id = next(line.split()[0] for line in output.splitlines() if 'apache-hadoop-master' in line)\n ssh_call += ['docker', 'exec', container_id]\n\n try:\n check_call(ssh_call + ['hdfs', 'dfs', '-rm', '-r', '/' + filename])\n except:\n pass"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef download_data(job, master_ip, inputs, known_snps, bam, hdfs_snps, hdfs_bam):\n\n log.info(\"Downloading known sites file %s to %s.\", known_snps, hdfs_snps)\n call_conductor(job, master_ip, known_snps, hdfs_snps, memory=inputs.memory)\n\n log.info(\"Downloading input BAM %s to %s.\", bam, hdfs_bam)\n call_conductor(job, master_ip, bam, hdfs_bam, memory=inputs.memory)", "response": "Downloads input data files from S3."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert input sam file and known SNPs file into ADAM format.", "response": "def adam_convert(job, master_ip, inputs, in_file, in_snps, adam_file, adam_snps, spark_on_toil):\n \"\"\"\n Convert input sam/bam file and known SNPs file into ADAM format\n \"\"\"\n\n log.info(\"Converting input BAM to ADAM.\")\n call_adam(job, master_ip,\n [\"transform\", in_file, adam_file],\n memory=inputs.memory,\n run_local=inputs.run_local,\n native_adam_path=inputs.native_adam_path)\n\n in_file_name = in_file.split(\"/\")[-1]\n remove_file(master_ip, in_file_name, spark_on_toil)\n\n log.info(\"Converting known sites VCF to ADAM.\")\n\n call_adam(job, master_ip,\n [\"vcf2adam\", \"-only_variants\", in_snps, adam_snps],\n memory=inputs.memory,\n run_local=inputs.run_local,\n native_adam_path=inputs.native_adam_path)\n\n in_snps_name = in_snps.split(\"/\")[-1]\n remove_file(master_ip, in_snps_name, spark_on_toil)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntransforms an ADAT file into a single file.", "response": "def adam_transform(job, master_ip, inputs, in_file, snp_file, hdfs_dir, out_file, spark_on_toil):\n \"\"\"\n Preprocess in_file with known SNPs snp_file:\n - mark duplicates\n - realign indels\n - recalibrate base quality scores\n \"\"\"\n\n log.info(\"Marking duplicate reads.\")\n call_adam(job, master_ip,\n [\"transform\",\n in_file, hdfs_dir + \"/mkdups.adam\",\n \"-aligned_read_predicate\",\n \"-limit_projection\",\n \"-mark_duplicate_reads\"],\n memory=inputs.memory,\n run_local=inputs.run_local,\n native_adam_path=inputs.native_adam_path)\n\n #FIXME\n in_file_name = in_file.split(\"/\")[-1]\n remove_file(master_ip, in_file_name + \"*\", spark_on_toil)\n\n log.info(\"Realigning INDELs.\")\n call_adam(job, master_ip,\n [\"transform\",\n hdfs_dir + \"/mkdups.adam\",\n hdfs_dir + \"/ri.adam\",\n \"-realign_indels\"],\n memory=inputs.memory,\n run_local=inputs.run_local,\n native_adam_path=inputs.native_adam_path)\n\n remove_file(master_ip, hdfs_dir + \"/mkdups.adam*\", spark_on_toil)\n\n log.info(\"Recalibrating base quality scores.\")\n call_adam(job, master_ip,\n [\"transform\",\n hdfs_dir + \"/ri.adam\",\n hdfs_dir + \"/bqsr.adam\",\n \"-recalibrate_base_qualities\",\n \"-known_snps\", snp_file],\n memory=inputs.memory,\n run_local=inputs.run_local,\n native_adam_path=inputs.native_adam_path)\n\n remove_file(master_ip, \"ri.adam*\", spark_on_toil)\n\n log.info(\"Sorting reads and saving a single BAM file.\")\n call_adam(job, master_ip,\n [\"transform\",\n hdfs_dir + \"/bqsr.adam\",\n out_file,\n \"-sort_reads\", \"-single\"],\n memory=inputs.memory,\n run_local=inputs.run_local,\n native_adam_path=inputs.native_adam_path)\n\n remove_file(master_ip, \"bqsr.adam*\", spark_on_toil)\n\n return out_file"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nupload file hdfsName from hdfs to s3", "response": "def upload_data(job, master_ip, inputs, hdfs_name, upload_name, spark_on_toil):\n \"\"\"\n Upload file hdfsName from hdfs to s3\n \"\"\"\n\n if mock_mode():\n truncate_file(master_ip, hdfs_name, spark_on_toil)\n\n log.info(\"Uploading output BAM %s to %s.\", hdfs_name, upload_name)\n call_conductor(job, master_ip, hdfs_name, upload_name, memory=inputs.memory)\n remove_file(master_ip, hdfs_name, spark_on_toil)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndownloads and upload the data from the master and store it in the job store.", "response": "def download_run_and_upload(job, master_ip, inputs, spark_on_toil):\n \"\"\"\n Monolithic job that calls data download, conversion, transform, upload.\n Previously, this was not monolithic; change came in due to #126/#134.\n \"\"\"\n master_ip = MasterAddress(master_ip)\n\n bam_name = inputs.sample.split('://')[-1].split('/')[-1]\n sample_name = \".\".join(os.path.splitext(bam_name)[:-1])\n\n hdfs_subdir = sample_name + \"-dir\"\n\n if inputs.run_local:\n inputs.local_dir = job.fileStore.getLocalTempDir()\n if inputs.native_adam_path is None:\n hdfs_dir = \"/data/\"\n else:\n hdfs_dir = inputs.local_dir\n else:\n inputs.local_dir = None\n hdfs_dir = \"hdfs://{0}:{1}/{2}\".format(master_ip, HDFS_MASTER_PORT, hdfs_subdir)\n\n try:\n hdfs_prefix = hdfs_dir + \"/\" + sample_name\n hdfs_bam = hdfs_dir + \"/\" + bam_name\n\n hdfs_snps = hdfs_dir + \"/\" + inputs.dbsnp.split('://')[-1].split('/')[-1]\n\n if not inputs.run_local:\n download_data(job, master_ip, inputs, inputs.dbsnp, inputs.sample, hdfs_snps, hdfs_bam)\n else:\n copy_files([inputs.sample, inputs.dbsnp], inputs.local_dir)\n\n adam_input = hdfs_prefix + \".adam\"\n adam_snps = hdfs_dir + \"/snps.var.adam\"\n adam_convert(job, master_ip, inputs, hdfs_bam, hdfs_snps, adam_input, adam_snps, spark_on_toil)\n\n adam_output = hdfs_prefix + \".processed.bam\"\n adam_transform(job, master_ip, inputs, adam_input, adam_snps, hdfs_dir, adam_output, spark_on_toil)\n\n out_file = inputs.output_dir + \"/\" + sample_name + inputs.suffix + \".bam\"\n\n if not inputs.run_local:\n upload_data(job, master_ip, inputs, adam_output, out_file, spark_on_toil)\n else:\n local_adam_output = \"%s/%s.processed.bam\" % (inputs.local_dir, sample_name)\n move_files([local_adam_output], inputs.output_dir)\n\n remove_file(master_ip, hdfs_subdir, spark_on_toil)\n except:\n remove_file(master_ip, hdfs_subdir, spark_on_toil)\n raise"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef hard_filter_pipeline(job, uuid, vcf_id, config):\n job.fileStore.logToMaster('Running Hard Filter on {}'.format(uuid))\n\n # Get the total size of the genome reference\n genome_ref_size = config.genome_fasta.size + config.genome_fai.size + config.genome_dict.size\n\n # The SelectVariants disk requirement depends on the input VCF, the genome reference files,\n # and the output VCF. The output VCF is smaller than the input VCF. The disk requirement\n # is identical for SNPs and INDELs.\n select_variants_disk = PromisedRequirement(lambda vcf, ref_size: 2 * vcf.size + ref_size,\n vcf_id,\n genome_ref_size)\n select_snps = job.wrapJobFn(gatk_select_variants,\n 'SNP',\n vcf_id,\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n memory=config.xmx,\n disk=select_variants_disk)\n\n # The VariantFiltration disk requirement depends on the input VCF, the genome reference files,\n # and the output VCF. The filtered VCF is smaller than the input VCF.\n snp_filter_disk = PromisedRequirement(lambda vcf, ref_size: 2 * vcf.size + ref_size,\n select_snps.rv(),\n genome_ref_size)\n\n snp_filter = job.wrapJobFn(gatk_variant_filtration,\n select_snps.rv(),\n config.snp_filter_name,\n config.snp_filter_expression,\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n memory=config.xmx,\n disk=snp_filter_disk)\n\n select_indels = job.wrapJobFn(gatk_select_variants,\n 'INDEL',\n vcf_id,\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n memory=config.xmx,\n disk=select_variants_disk)\n\n indel_filter_disk = PromisedRequirement(lambda vcf, ref_size: 2 * vcf.size + ref_size,\n select_indels.rv(),\n genome_ref_size)\n\n indel_filter = job.wrapJobFn(gatk_variant_filtration,\n select_indels.rv(),\n config.indel_filter_name,\n config.indel_filter_expression,\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n memory=config.xmx,\n disk=indel_filter_disk)\n\n # The CombineVariants disk requirement depends on the SNP and INDEL input VCFs and the\n # genome reference files. The combined VCF is approximately the same size as the input files.\n combine_vcfs_disk = PromisedRequirement(lambda vcf1, vcf2, ref_size:\n 2 * (vcf1.size + vcf2.size) + ref_size,\n indel_filter.rv(),\n snp_filter.rv(),\n genome_ref_size)\n\n combine_vcfs = job.wrapJobFn(gatk_combine_variants,\n {'SNPs': snp_filter.rv(), 'INDELs': indel_filter.rv()},\n config.genome_fasta,\n config.genome_fai,\n config.genome_dict,\n merge_option='UNSORTED', # Merges variants from a single sample\n memory=config.xmx,\n disk=combine_vcfs_disk)\n\n job.addChild(select_snps)\n job.addChild(select_indels)\n\n select_snps.addChild(snp_filter)\n snp_filter.addChild(combine_vcfs)\n\n select_indels.addChild(indel_filter)\n indel_filter.addChild(combine_vcfs)\n\n # Output the hard filtered VCF\n output_dir = os.path.join(config.output_dir, uuid)\n output_filename = '%s.hard_filter%s.vcf' % (uuid, config.suffix)\n output_vcf = job.wrapJobFn(output_file_job,\n output_filename,\n combine_vcfs.rv(),\n output_dir,\n s3_key_path=config.ssec,\n disk=PromisedRequirement(lambda x: x.size, combine_vcfs.rv()))\n combine_vcfs.addChild(output_vcf)\n return combine_vcfs.rv()", "response": "This function runs GATK Hard Filtering on a Genomic VCF file and uploads the results to the output directory."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef download_and_transfer_sample(job, sample, inputs):\n\n \"\"\"\n Downloads a sample from CGHub via GeneTorrent, then uses S3AM to transfer it to S3\n\n input_args: dict Dictionary of input arguments\n analysis_id: str An analysis ID for a sample in CGHub\n \"\"\"\n analysis_id = sample[0]\n work_dir = job.fileStore.getLocalTempDir()\n folder_path = os.path.join(work_dir, os.path.basename(analysis_id))\n # Acquire genetorrent key and download sample\n shutil.copy(inputs['genetorrent_key'], os.path.join(work_dir, 'cghub.key'))\n parameters = ['-vv', '-c', 'cghub.key', '-d', analysis_id]\n docker_call(job=job, tool='quay.io/ucsc_cgl/genetorrent:3.8.7--9911761265b6f08bc3ef09f53af05f56848d805b',\n work_dir=work_dir, parameters=parameters)\n try:\n sample = glob.glob(os.path.join(folder_path, '*tar*'))[0]\n except KeyError as e:\n print 'No tarfile found inside of folder: '.format(e)\n raise\n # Upload sample to S3AM\n key_path = inputs['ssec']\n if sample.endswith('gz'):\n sample_name = analysis_id + '.tar.gz'\n shutil.move(sample, os.path.join(work_dir, sample_name))\n else:\n sample_name = analysis_id + '.tar'\n shutil.move(sample, os.path.join(work_dir, sample_name))\n # Parse s3_dir to get bucket and s3 path\n s3_dir = inputs['s3_dir']\n bucket_name = s3_dir.lstrip('/').split('/')[0]\n base_url = 'https://s3-us-west-2.amazonaws.com/'\n url = os.path.join(base_url, bucket_name, sample_name)\n # Generate keyfile for upload\n with open(os.path.join(work_dir, 'temp.key'), 'wb') as f_out:\n f_out.write(generate_unique_key(key_path, url))\n # Upload to S3 via S3AM\n s3am_command = ['s3am',\n 'upload',\n '--sse-key-file', os.path.join(work_dir, 'temp.key'),\n 'file://{}'.format(os.path.join(work_dir, sample_name)),\n 's3://' + bucket_name + '/']\n subprocess.check_call(s3am_command)", "response": "Downloads a sample from CGHub then uses S3AM to transfer it to CGHub"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nvalidating a hexidecimal IPv6 ip address.", "response": "def validate_ip(s):\n \"\"\"Validate a hexidecimal IPv6 ip address.\n\n\n >>> validate_ip('::')\n True\n >>> validate_ip('::1')\n True\n >>> validate_ip('2001:db8:85a3::8a2e:370:7334')\n True\n >>> validate_ip('2001:db8:85a3:0:0:8a2e:370:7334')\n True\n >>> validate_ip('2001:0db8:85a3:0000:0000:8a2e:0370:7334')\n True\n >>> validate_ip('2001:db8::1:0:0:1')\n True\n >>> validate_ip('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')\n True\n >>> validate_ip('::ffff:192.0.2.128')\n True\n >>> validate_ip('::ff::ff')\n False\n >>> validate_ip('::fffff')\n False\n >>> validate_ip('::ffff:192.0.2.300')\n False\n >>> validate_ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: expected string or buffer\n >>> validate_ip('1080:0:0:0:8:800:200c:417a')\n True\n\n\n :param s: String to validate as a hexidecimal IPv6 ip address.\n :type s: str\n :returns: ``True`` if a valid hexidecimal IPv6 ip address,\n ``False`` otherwise.\n :raises: TypeError\n \"\"\"\n if _HEX_RE.match(s):\n return len(s.split('::')) <= 2\n if _DOTTED_QUAD_RE.match(s):\n halves = s.split('::')\n if len(halves) > 2:\n return False\n hextets = s.split(':')\n quads = hextets[-1].split('.')\n for q in quads:\n if int(q) > 255:\n return False\n return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ip2long(ip):\n if not validate_ip(ip):\n return None\n\n if '.' in ip:\n # convert IPv4 suffix to hex\n chunks = ip.split(':')\n v4_int = ipv4.ip2long(chunks.pop())\n if v4_int is None:\n return None\n chunks.append('%x' % ((v4_int >> 16) & 0xffff))\n chunks.append('%x' % (v4_int & 0xffff))\n ip = ':'.join(chunks)\n\n halves = ip.split('::')\n hextets = halves[0].split(':')\n if len(halves) == 2:\n h2 = halves[1].split(':')\n for z in range(8 - (len(hextets) + len(h2))):\n hextets.append('0')\n for h in h2:\n hextets.append(h)\n # end if\n\n lngip = 0\n for h in hextets:\n if '' == h:\n h = '0'\n lngip = (lngip << 16) | int(h, 16)\n return lngip", "response": "Convert a hexidecimal IPv6 address to a network byte order 128 - bit integer."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert a network byte order 128 - bit integer to a canonical IPv6 address.", "response": "def long2ip(l, rfc1924=False):\n \"\"\"Convert a network byte order 128-bit integer to a canonical IPv6\n address.\n\n\n >>> long2ip(2130706433)\n '::7f00:1'\n >>> long2ip(42540766411282592856904266426630537217)\n '2001:db8::1:0:0:1'\n >>> long2ip(MIN_IP)\n '::'\n >>> long2ip(MAX_IP)\n 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'\n >>> long2ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: unsupported operand type(s) for >>: 'NoneType' and 'int'\n >>> long2ip(-1) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: expected int between 0 and inclusive\n >>> long2ip(MAX_IP + 1) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: expected int between 0 and inclusive\n >>> long2ip(ip2long('1080::8:800:200C:417A'), rfc1924=True)\n '4)+k&C#VzJ4br>0wv%Yp'\n >>> long2ip(ip2long('::'), rfc1924=True)\n '00000000000000000000'\n\n\n :param l: Network byte order 128-bit integer.\n :type l: int\n :param rfc1924: Encode in RFC 1924 notation (base 85)\n :type rfc1924: bool\n :returns: Canonical IPv6 address (eg. '::1').\n :raises: TypeError\n \"\"\"\n if MAX_IP < l or l < MIN_IP:\n raise TypeError(\n \"expected int between %d and %d inclusive\" % (MIN_IP, MAX_IP))\n\n if rfc1924:\n return long2rfc1924(l)\n\n # format as one big hex value\n hex_str = '%032x' % l\n # split into double octet chunks without padding zeros\n hextets = ['%x' % int(hex_str[x:x + 4], 16) for x in range(0, 32, 4)]\n\n # find and remove left most longest run of zeros\n dc_start, dc_len = (-1, 0)\n run_start, run_len = (-1, 0)\n for idx, hextet in enumerate(hextets):\n if '0' == hextet:\n run_len += 1\n if -1 == run_start:\n run_start = idx\n if run_len > dc_len:\n dc_len, dc_start = (run_len, run_start)\n else:\n run_len, run_start = (0, -1)\n # end for\n if dc_len > 1:\n dc_end = dc_start + dc_len\n if dc_end == len(hextets):\n hextets += ['']\n hextets[dc_start:dc_end] = ['']\n if dc_start == 0:\n hextets = [''] + hextets\n # end if\n\n return ':'.join(hextets)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef long2rfc1924(l):\n if MAX_IP < l or l < MIN_IP:\n raise TypeError(\n \"expected int between %d and %d inclusive\" % (MIN_IP, MAX_IP))\n o = []\n r = l\n while r > 85:\n o.append(_RFC1924_ALPHABET[r % 85])\n r = r // 85\n o.append(_RFC1924_ALPHABET[r])\n return ''.join(reversed(o)).zfill(20)", "response": "Convert a network byte order 128 - bit integer to an RFC 1924 IPv6 address."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef rfc19242long(s):\n global _RFC1924_REV\n if not _RFC1924_RE.match(s):\n return None\n if _RFC1924_REV is None:\n _RFC1924_REV = {v: k for k, v in enumerate(_RFC1924_ALPHABET)}\n x = 0\n for c in s:\n x = x * 85 + _RFC1924_REV[c]\n if x > MAX_IP:\n return None\n return x", "response": "Convert an RFC 1924 IPv6 address to a network byte order 128 - bit integer."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nvalidating a CIDR notation ip address.", "response": "def validate_cidr(s):\n \"\"\"Validate a CIDR notation ip address.\n\n The string is considered a valid CIDR address if it consists of a valid\n IPv6 address in hextet format followed by a forward slash (/) and a bit\n mask length (0-128).\n\n\n >>> validate_cidr('::/128')\n True\n >>> validate_cidr('::/0')\n True\n >>> validate_cidr('fc00::/7')\n True\n >>> validate_cidr('::ffff:0:0/96')\n True\n >>> validate_cidr('::')\n False\n >>> validate_cidr('::/129')\n False\n >>> validate_cidr(None) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: expected string or buffer\n\n\n :param s: String to validate as a CIDR notation ip address.\n :type s: str\n :returns: ``True`` if a valid CIDR address, ``False`` otherwise.\n :raises: TypeError\n \"\"\"\n if _CIDR_RE.match(s):\n ip, mask = s.split('/')\n if validate_ip(ip):\n if int(mask) > 128:\n return False\n else:\n return False\n return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cidr2block(cidr):\n if not validate_cidr(cidr):\n return None\n\n ip, prefix = cidr.split('/')\n prefix = int(prefix)\n ip = ip2long(ip)\n\n # keep left most prefix bits of ip\n shift = 128 - prefix\n block_start = ip >> shift << shift\n\n # expand right most 128 - prefix bits to 1\n mask = (1 << shift) - 1\n block_end = block_start | mask\n return (long2ip(block_start), long2ip(block_end))", "response": "Convert a CIDR notation ip address into a tuple containing the network\n block start and end addresses."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses the input samples and adds them to the job.", "response": "def parse_input_samples(job, inputs):\n \"\"\"\n Parses config file to pull sample information.\n Stores samples as tuples of (uuid, URL)\n\n :param JobFunctionWrappingJob job: passed by Toil automatically\n :param Namespace inputs: Stores input arguments (see main)\n \"\"\"\n job.fileStore.logToMaster('Parsing input samples and batching jobs')\n samples = []\n if inputs.config:\n with open(inputs.config, 'r') as f:\n for line in f.readlines():\n if not line.isspace():\n sample = line.strip().split(',')\n assert len(sample) == 2, 'Error: Config file is inappropriately formatted.'\n samples.append(sample)\n job.addChildJobFn(map_job, download_sample, samples, inputs)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef download_sample(job, sample, inputs):\n uuid, url = sample\n job.fileStore.logToMaster('Downloading sample: {}'.format(uuid))\n # Download sample\n tar_id = job.addChildJobFn(download_url_job, url, s3_key_path=inputs.ssec, disk='30G').rv()\n # Create copy of inputs for each sample\n sample_inputs = argparse.Namespace(**vars(inputs))\n sample_inputs.uuid = uuid\n sample_inputs.cores = multiprocessing.cpu_count()\n # Call children and follow-on jobs\n job.addFollowOnJobFn(process_sample, sample_inputs, tar_id, cores=2, disk='60G')", "response": "Download the sample from Toil and return the tar_id"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts sample.tar(.gz) into two fastq files. Due to edge conditions... BEWARE: HERE BE DRAGONS :param JobFunctionWrappingJob job: passed by Toil automatically :param Namespace inputs: Stores input arguments (see main) :param str tar_id: FileStore ID of sample tar", "response": "def process_sample(job, inputs, tar_id):\n \"\"\"\n Converts sample.tar(.gz) into two fastq files.\n Due to edge conditions... BEWARE: HERE BE DRAGONS\n\n :param JobFunctionWrappingJob job: passed by Toil automatically\n :param Namespace inputs: Stores input arguments (see main)\n :param str tar_id: FileStore ID of sample tar\n \"\"\"\n job.fileStore.logToMaster('Processing sample into read pairs: {}'.format(inputs.uuid))\n work_dir = job.fileStore.getLocalTempDir()\n # I/O\n tar_path = job.fileStore.readGlobalFile(tar_id, os.path.join(work_dir, 'sample.tar'))\n # Untar File and concat\n subprocess.check_call(['tar', '-xvf', tar_path, '-C', work_dir])\n os.remove(os.path.join(work_dir, 'sample.tar'))\n # Grab files from tarball\n fastqs = []\n for root, subdir, files in os.walk(work_dir):\n fastqs.extend([os.path.join(root, x) for x in files])\n # Check for read 1 and read 2 files\n r1 = sorted([x for x in fastqs if 'R1' in x])\n r2 = sorted([x for x in fastqs if 'R2' in x])\n if not r1 or not r2:\n # Check if using a different standard\n r1 = sorted([x for x in fastqs if '_1' in x])\n r2 = sorted([x for x in fastqs if '_2' in x])\n # Prune file name matches from each list\n if len(r1) > len(r2):\n r1 = [x for x in r1 if x not in r2]\n elif len(r2) > len(r1):\n r2 = [x for x in r2 if x not in r1]\n # Flag if data is single-ended\n assert r1 and r2, 'This pipeline does not support single-ended data. R1: {}\\nR2:{}'.format(r1, r2)\n command = 'zcat' if r1[0].endswith('gz') and r2[0].endswith('gz') else 'cat'\n with open(os.path.join(work_dir, 'R1.fastq'), 'w') as f1:\n p1 = subprocess.Popen([command] + r1, stdout=f1)\n with open(os.path.join(work_dir, 'R2.fastq'), 'w') as f2:\n p2 = subprocess.Popen([command] + r2, stdout=f2)\n p1.wait()\n p2.wait()\n # Write to fileStore\n r1_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1.fastq'))\n r2_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R2.fastq'))\n job.fileStore.deleteGlobalFile(tar_id)\n # Start cutadapt step\n job.addChildJobFn(cutadapt, inputs, r1_id, r2_id, disk='60G').rv()"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfilters out adapters that may be left in the RNA-seq files :param JobFunctionWrappingJob job: passed by Toil automatically :param Namespace inputs: Stores input arguments (see main) :param str r1_id: FileStore ID of read 1 fastq :param str r2_id: FileStore ID of read 2 fastq", "response": "def cutadapt(job, inputs, r1_id, r2_id):\n \"\"\"\n Filters out adapters that may be left in the RNA-seq files\n\n :param JobFunctionWrappingJob job: passed by Toil automatically\n :param Namespace inputs: Stores input arguments (see main)\n :param str r1_id: FileStore ID of read 1 fastq\n :param str r2_id: FileStore ID of read 2 fastq\n \"\"\"\n job.fileStore.logToMaster('Running CutAdapt: {}'.format(inputs.uuid))\n work_dir = job.fileStore.getLocalTempDir()\n inputs.improper_pair = None\n # Retrieve files\n job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, 'R1.fastq'))\n job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, 'R2.fastq'))\n # Cutadapt parameters\n parameters = ['-a', inputs.fwd_3pr_adapter,\n '-m', '35',\n '-A', inputs.rev_3pr_adapter,\n '-o', '/data/R1_cutadapt.fastq',\n '-p', '/data/R2_cutadapt.fastq',\n '/data/R1.fastq', '/data/R2.fastq']\n # Call: CutAdapt\n base_docker_call = 'docker run --log-driver=none --rm -v {}:/data'.format(work_dir).split()\n if inputs.sudo:\n base_docker_call = ['sudo'] + base_docker_call\n tool = 'quay.io/ucsc_cgl/cutadapt:1.9--6bd44edd2b8f8f17e25c5a268fedaab65fa851d2'\n p = subprocess.Popen(base_docker_call + [tool] + parameters, stderr=subprocess.PIPE, stdout=subprocess.PIPE)\n stdout, stderr = p.communicate()\n if p.returncode != 0:\n if 'improperly paired' in stderr:\n inputs.improper_pair = True\n shutil.move(os.path.join(work_dir, 'R1.fastq'), os.path.join(work_dir, 'R1_cutadapt.fastq'))\n shutil.move(os.path.join(work_dir, 'R2.fastq'), os.path.join(work_dir, 'R2_cutadapt.fastq'))\n # Write to fileStore\n if inputs.improper_pair:\n r1_cutadapt = r1_id\n r2_cutadapt = r2_id\n else:\n r1_cutadapt = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R1_cutadapt.fastq'))\n r2_cutadapt = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'R2_cutadapt.fastq'))\n job.fileStore.deleteGlobalFile(r1_id)\n job.fileStore.deleteGlobalFile(r2_id)\n # start STAR\n cores = min(inputs.cores, 16)\n job.addChildJobFn(star, inputs, r1_cutadapt, r2_cutadapt, cores=cores, disk='100G', memory='40G').rv()"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef star(job, inputs, r1_cutadapt, r2_cutadapt):\n job.fileStore.logToMaster('Aligning with STAR: {}'.format(inputs.uuid))\n work_dir = job.fileStore.getLocalTempDir()\n cores = min(inputs.cores, 16)\n # Retrieve files\n job.fileStore.readGlobalFile(r1_cutadapt, os.path.join(work_dir, 'R1_cutadapt.fastq'))\n job.fileStore.readGlobalFile(r2_cutadapt, os.path.join(work_dir, 'R2_cutadapt.fastq'))\n # Get starIndex\n download_url(job=job, url=inputs.star_index, work_dir=work_dir, name='starIndex.tar.gz')\n subprocess.check_call(['tar', '-xvf', os.path.join(work_dir, 'starIndex.tar.gz'), '-C', work_dir])\n # Parameters\n parameters = ['--runThreadN', str(cores),\n '--genomeDir', '/data/starIndex',\n '--outFileNamePrefix', 'rna',\n '--outSAMtype', 'BAM', 'SortedByCoordinate',\n '--outSAMunmapped', 'Within',\n '--quantMode', 'TranscriptomeSAM',\n '--outSAMattributes', 'NH', 'HI', 'AS', 'NM', 'MD',\n '--outFilterType', 'BySJout',\n '--outFilterMultimapNmax', '20',\n '--outFilterMismatchNmax', '999',\n '--outFilterMismatchNoverReadLmax', '0.04',\n '--alignIntronMin', '20',\n '--alignIntronMax', '1000000',\n '--alignMatesGapMax', '1000000',\n '--alignSJoverhangMin', '8',\n '--alignSJDBoverhangMin', '1',\n '--sjdbScore', '1',\n '--readFilesIn', '/data/R1_cutadapt.fastq', '/data/R2_cutadapt.fastq']\n # Call: STAR Map\n docker_call(job=job, tool='quay.io/ucsc_cgl/star:2.4.2a--bcbd5122b69ff6ac4ef61958e47bde94001cfe80',\n work_dir=work_dir, parameters=parameters)\n # Call Samtools Index\n index_command = ['index', '/data/rnaAligned.sortedByCoord.out.bam']\n docker_call(job=job, work_dir=work_dir, parameters=index_command,\n tool='quay.io/ucsc_cgl/samtools:1.3--256539928ea162949d8a65ca5c79a72ef557ce7c')\n # fileStore\n bam_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rnaAligned.sortedByCoord.out.bam'))\n bai_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'rnaAligned.sortedByCoord.out.bam.bai'))\n job.fileStore.deleteGlobalFile(r1_cutadapt)\n job.fileStore.deleteGlobalFile(r2_cutadapt)\n # Launch children and follow-on\n vcqc_id = job.addChildJobFn(variant_calling_and_qc, inputs, bam_id, bai_id, cores=2, disk='30G').rv()\n spladder_id = job.addChildJobFn(spladder, inputs, bam_id, bai_id, disk='30G').rv()\n job.addFollowOnJobFn(consolidate_output_tarballs, inputs, vcqc_id, spladder_id, disk='30G')", "response": "This function aligns fastqs to BAM via STAR"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nperform variant calling with samtools nad QC with CheckBias", "response": "def variant_calling_and_qc(job, inputs, bam_id, bai_id):\n \"\"\"\n Perform variant calling with samtools nad QC with CheckBias\n\n :param JobFunctionWrappingJob job: passed by Toil automatically\n :param Namespace inputs: Stores input arguments (see main)\n :param str bam_id: FileStore ID of bam\n :param str bai_id: FileStore ID of bam index file\n :return: FileStore ID of qc tarball\n :rtype: str\n \"\"\"\n job.fileStore.logToMaster('Variant calling and QC: {}'.format(inputs.uuid))\n work_dir = job.fileStore.getLocalTempDir()\n # Pull in alignment.bam from fileStore\n job.fileStore.readGlobalFile(bam_id, os.path.join(work_dir, 'alignment.bam'))\n job.fileStore.readGlobalFile(bai_id, os.path.join(work_dir, 'alignment.bam.bai'))\n # Download input files\n input_info = [(inputs.genome, 'genome.fa'), (inputs.positions, 'positions.tsv'),\n (inputs.genome_index, 'genome.fa.fai'), (inputs.gtf, 'annotation.gtf'),\n (inputs.gtf_m53, 'annotation.m53')]\n for url, fname in input_info:\n download_url(job=job, url=url, work_dir=work_dir, name=fname)\n\n # Part 1: Variant Calling\n variant_command = ['mpileup',\n '-f', 'genome.fa',\n '-l', 'positions.tsv',\n '-v', 'alignment.bam',\n '-t', 'DP,SP,INFO/AD,INFO/ADF,INFO/ADR,INFO/DPR,SP',\n '-o', '/data/output.vcf.gz']\n docker_call(job=job, work_dir=work_dir, parameters=variant_command,\n tool='quay.io/ucsc_cgl/samtools:1.3--256539928ea162949d8a65ca5c79a72ef557ce7c')\n\n # Part 2: QC\n qc_command = ['-o', 'qc',\n '-n', 'alignment.bam',\n '-a', 'annotation.gtf',\n '-m', 'annotation.m53']\n docker_call(job=job, work_dir=work_dir, parameters=qc_command,\n tool='jvivian/checkbias:612f129--b08a1fb6526a620bbb0304b08356f2ae7c3c0ec3')\n # Write output to fileStore and return ids\n output_tsv = glob(os.path.join(work_dir, '*counts.tsv*'))[0]\n output_vcf = os.path.join(work_dir, 'output.vcf.gz')\n tarball_files('vcqc.tar.gz', file_paths=[output_tsv, output_vcf], output_dir=work_dir)\n return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'vcqc.tar.gz'))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef spladder(job, inputs, bam_id, bai_id):\n job.fileStore.logToMaster('SplAdder: {}'.format(inputs.uuid))\n work_dir = job.fileStore.getLocalTempDir()\n # Pull in alignment.bam from fileStore\n job.fileStore.readGlobalFile(bam_id, os.path.join(work_dir, 'alignment.bam'))\n job.fileStore.readGlobalFile(bai_id, os.path.join(work_dir, 'alignment.bam.bai'))\n # Download input file\n download_url(job=job, url=inputs.gtf, work_dir=work_dir, name='annotation.gtf')\n download_url(job=job, url=inputs.gtf_pickle, work_dir=work_dir, name='annotation.gtf.pickle')\n # Call Spladder\n command = ['--insert_ir=y',\n '--insert_es=y',\n '--insert_ni=y',\n '--remove_se=n',\n '--validate_sg=n',\n '-b', 'alignment.bam',\n '-o ', '/data',\n '-a', 'annotation.gtf',\n '-v', 'y',\n '-c', '3',\n '-M', 'single',\n '-T', 'n',\n '-n', '50',\n '-P', 'y',\n '-p', 'n',\n '--sparse_bam', 'y']\n docker_call(job=job, work_dir=work_dir, parameters=command, sudo=inputs.sudo, tool='jvivian/spladder:1.0')\n # Write output to fileStore and return ids\n output_pickle = os.path.join(work_dir, ' ', 'spladder', 'genes_graph_conf3.alignment.pickle')\n if not os.path.exists(output_pickle):\n matches = []\n for root, dirnames, filenames in os.walk(work_dir):\n for filename in fnmatch.filter(filenames, '*genes_graph*'):\n matches.append(os.path.join(root, filename))\n if matches:\n output_pickle = matches[0]\n else:\n raise RuntimeError(\"Couldn't find genes file!\")\n output_filt = os.path.join(work_dir, 'alignment.filt.hdf5')\n output = os.path.join(work_dir, 'alignment.hdf5')\n print os.listdir(work_dir)\n tarball_files('spladder.tar.gz', file_paths=[output_pickle, output_filt, output], output_dir=work_dir)\n return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'spladder.tar.gz'))", "response": "Run SplAdder to detect alternative splicing events"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncombine the contents of separate tarballs into one.", "response": "def consolidate_output_tarballs(job, inputs, vcqc_id, spladder_id):\n \"\"\"\n Combine the contents of separate tarballs into one.\n\n :param JobFunctionWrappingJob job: passed by Toil automatically\n :param Namespace inputs: Stores input arguments (see main)\n :param str vcqc_id: FileStore ID of variant calling and QC tarball\n :param str spladder_id: FileStore ID of spladder tarball\n \"\"\"\n job.fileStore.logToMaster('Consolidating files and uploading: {}'.format(inputs.uuid))\n work_dir = job.fileStore.getLocalTempDir()\n # Retrieve IDs\n uuid = inputs.uuid\n # Unpack IDs\n # Retrieve output file paths to consolidate\n vcqc_tar = job.fileStore.readGlobalFile(vcqc_id, os.path.join(work_dir, 'vcqc.tar.gz'))\n spladder_tar = job.fileStore.readGlobalFile(spladder_id, os.path.join(work_dir, 'spladder.tar.gz'))\n # I/O\n fname = uuid + '.tar.gz' if not inputs.improper_pair else 'IMPROPER_PAIR' + uuid + '.tar.gz'\n out_tar = os.path.join(work_dir, fname)\n # Consolidate separate tarballs into one\n with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:\n for tar in [vcqc_tar, spladder_tar]:\n with tarfile.open(tar, 'r') as f_in:\n for tarinfo in f_in:\n with closing(f_in.extractfile(tarinfo)) as f_in_file:\n if tar == vcqc_tar:\n tarinfo.name = os.path.join(uuid, 'variants_and_qc', os.path.basename(tarinfo.name))\n else:\n tarinfo.name = os.path.join(uuid, 'spladder', os.path.basename(tarinfo.name))\n f_out.addfile(tarinfo, fileobj=f_in_file)\n # Move to output directory\n if inputs.output_dir:\n mkdir_p(inputs.output_dir)\n shutil.copy(out_tar, os.path.join(inputs.output_dir, os.path.basename(out_tar)))\n # Upload to S3\n if inputs.output_s3_dir:\n out_id = job.fileStore.writeGlobalFile(out_tar)\n job.addChildJobFn(s3am_upload_job, file_id=out_id, s3_dir=inputs.output_s3_dir,\n file_name=fname, key_path=inputs.ssec, cores=inputs.cores)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef main():\n # Define Parser object and add to toil\n url_prefix = 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/'\n parser = argparse.ArgumentParser(description=main.__doc__, formatter_class=argparse.RawTextHelpFormatter)\n parser.add_argument('--config', required=True,\n help='Path to configuration file for samples, one per line. UUID,URL_to_bamfile. '\n 'The URL may be a standard \"http://\", a \"file://\", or \"s3:///\"')\n parser.add_argument('--gtf', help='URL to annotation GTF file',\n default=url_prefix + 'rnaseq_cgl/gencode.v23.annotation.gtf')\n parser.add_argument('--gtf-pickle', help='Pickled GTF file',\n default=url_prefix + 'spladder/gencode.v23.annotation.gtf.pickle')\n parser.add_argument('--gtf-m53', help='M53 preprocessing annotation table',\n default=url_prefix + 'spladder/gencode.v23.annotation.gtf.m53')\n parser.add_argument('--positions', help='URL to SNP positions over genes file (TSV)',\n default=url_prefix + 'spladder/positions_fixed.tsv')\n parser.add_argument('--genome', help='URL to Genome fasta',\n default=url_prefix + 'rnaseq_cgl/hg38_no_alt.fa')\n parser.add_argument('--genome-index', help='Index file (fai) of genome',\n default=url_prefix + 'spladder/hg38_no_alt.fa.fai')\n parser.add_argument('--ssec', default=None, help='Path to master key used for downloading encrypted files.')\n parser.add_argument('--output-s3-dir', default=None, help='S3 Directory of the form: s3://bucket/directory')\n parser.add_argument('--output-dir', default=None, help='full path where final results will be output')\n parser.add_argument('--sudo', action='store_true', default=False,\n help='Set flag if sudo is required to run Docker.')\n parser.add_argument('--star-index', help='URL to download STAR Index built from HG38/gencodev23 annotation.',\n default=url_prefix + 'rnaseq_cgl/starIndex_hg38_no_alt.tar.gz')\n parser.add_argument('--fwd-3pr-adapter', help=\"Sequence for the FWD 3' Read Adapter.\", default='AGATCGGAAGAG')\n parser.add_argument('--rev-3pr-adapter', help=\"Sequence for the REV 3' Read Adapter.\", default='AGATCGGAAGAG')\n Job.Runner.addToilOptions(parser)\n args = parser.parse_args()\n # Sanity Checks\n if args.config:\n assert os.path.isfile(args.config), 'Config not found at: {}'.format(args.config)\n if args.ssec:\n assert os.path.isfile(args.ssec), 'Encryption key not found at: {}'.format(args.config)\n if args.output_s3_dir:\n assert args.output_s3_dir.startswith('s3://'), 'Wrong format for output s3 directory'\n # Program checks\n for program in ['curl', 'docker']:\n assert which(program), 'Program \"{}\" must be installed on every node.'.format(program)\n\n Job.Runner.startToil(Job.wrapJobFn(parse_input_samples, args), args)", "response": "This function is the main function of the toil pipeline. It will aligns reads and performs alternative splicing analysis."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef validate_ip(s):\n if _DOTTED_QUAD_RE.match(s):\n quads = s.split('.')\n for q in quads:\n if int(q) > 255:\n return False\n return True\n return False", "response": "Validate a dotted - quad ip address."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validate_netmask(s):\n if validate_ip(s):\n # Convert to binary string, strip '0b' prefix, 0 pad to 32 bits\n mask = bin(ip2network(s))[2:].zfill(32)\n # all left most bits must be 1, all right most must be 0\n seen0 = False\n for c in mask:\n if '1' == c:\n if seen0:\n return False\n else:\n seen0 = True\n return True\n else:\n return False", "response": "Validate that a dotted - quad ip address is a valid netmask."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validate_subnet(s):\n if isinstance(s, basestring):\n if '/' in s:\n start, mask = s.split('/', 2)\n return validate_ip(start) and validate_netmask(mask)\n else:\n return False\n raise TypeError(\"expected string or unicode\")", "response": "Validate a dotted - quad ip address including a netmask."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ip2long(ip):\n if not validate_ip(ip):\n return None\n quads = ip.split('.')\n if len(quads) == 1:\n # only a network quad\n quads = quads + [0, 0, 0]\n elif len(quads) < 4:\n # partial form, last supplied quad is host address, rest is network\n host = quads[-1:]\n quads = quads[:-1] + [0, ] * (4 - len(quads)) + host\n\n lngip = 0\n for q in quads:\n lngip = (lngip << 8) | int(q)\n return lngip", "response": "Convert a dotted - quad ip address to a network byte order 32 - bit integer."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconvert a dotted - quad ip address to base network number.", "response": "def ip2network(ip):\n \"\"\"Convert a dotted-quad ip to base network number.\n\n This differs from :func:`ip2long` in that partial addresses as treated as\n all network instead of network plus host (eg. '127.1' expands to\n '127.1.0.0')\n\n :param ip: dotted-quad ip address (eg. \u2018127.0.0.1\u2019).\n :type ip: str\n :returns: Network byte order 32-bit integer or `None` if ip is invalid.\n \"\"\"\n if not validate_ip(ip):\n return None\n quads = ip.split('.')\n netw = 0\n for i in range(4):\n netw = (netw << 8) | int(len(quads) > i and quads[i] or 0)\n return netw"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert a network byte order 32 - bit integer to a dotted quad ip.", "response": "def long2ip(l):\n \"\"\"Convert a network byte order 32-bit integer to a dotted quad ip\n address.\n\n\n >>> long2ip(2130706433)\n '127.0.0.1'\n >>> long2ip(MIN_IP)\n '0.0.0.0'\n >>> long2ip(MAX_IP)\n '255.255.255.255'\n >>> long2ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: unsupported operand type(s) for >>: 'NoneType' and 'int'\n >>> long2ip(-1) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: expected int between 0 and 4294967295 inclusive\n >>> long2ip(374297346592387463875) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: expected int between 0 and 4294967295 inclusive\n >>> long2ip(MAX_IP + 1) #doctest: +IGNORE_EXCEPTION_DETAIL\n Traceback (most recent call last):\n ...\n TypeError: expected int between 0 and 4294967295 inclusive\n\n\n :param l: Network byte order 32-bit integer.\n :type l: int\n :returns: Dotted-quad ip address (eg. '127.0.0.1').\n :raises: TypeError\n \"\"\"\n if MAX_IP < l or l < MIN_IP:\n raise TypeError(\n \"expected int between %d and %d inclusive\" % (MIN_IP, MAX_IP))\n return '%d.%d.%d.%d' % (\n l >> 24 & 255, l >> 16 & 255, l >> 8 & 255, l & 255)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cidr2block(cidr):\n if not validate_cidr(cidr):\n return None\n\n ip, prefix = cidr.split('/')\n prefix = int(prefix)\n\n # convert dotted-quad ip to base network number\n network = ip2network(ip)\n\n return _block_from_ip_and_prefix(network, prefix)", "response": "Convert a CIDR notation ip address into a tuple containing the start and end addresses."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef subnet2block(subnet):\n if not validate_subnet(subnet):\n return None\n\n ip, netmask = subnet.split('/')\n prefix = netmask2prefix(netmask)\n\n # convert dotted-quad ip to base network number\n network = ip2network(ip)\n\n return _block_from_ip_and_prefix(network, prefix)", "response": "Convert a dotted - quad ip address including a netmask into a tuple of start and end addresses."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _block_from_ip_and_prefix(ip, prefix):\n # keep left most prefix bits of ip\n shift = 32 - prefix\n block_start = ip >> shift << shift\n\n # expand right most 32 - prefix bits to 1\n mask = (1 << shift) - 1\n block_end = block_start | mask\n return (long2ip(block_start), long2ip(block_end))", "response": "Create a tuple of start end dotted - quad addresses from the given ip and prefix length."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndownloading shared files from the given samples in the given pipeline.", "response": "def download_shared_files(job, samples, config):\n \"\"\"\n Downloads files shared by all samples in the pipeline\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param Namespace config: Argparse Namespace object containing argument inputs\n :param list[list] samples: A nested list of samples containing sample information\n \"\"\"\n job.fileStore.logToMaster('Downloaded shared files')\n file_names = ['reference', 'phase', 'mills', 'dbsnp', 'cosmic']\n urls = [config.reference, config.phase, config.mills, config.dbsnp, config.cosmic]\n for name, url in zip(file_names, urls):\n if url:\n vars(config)[name] = job.addChildJobFn(download_url_job, url=url).rv()\n job.addFollowOnJobFn(reference_preprocessing, samples, config)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nspawn the jobs that create index and dict file for reference :param JobFunctionWrappingJob job: passed automatically by Toil :param Namespace config: Argparse Namespace object containing argument inputs :param list[list] samples: A nested list of samples containing sample information", "response": "def reference_preprocessing(job, samples, config):\n \"\"\"\n Spawn the jobs that create index and dict file for reference\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param Namespace config: Argparse Namespace object containing argument inputs\n :param list[list] samples: A nested list of samples containing sample information\n \"\"\"\n job.fileStore.logToMaster('Processed reference files')\n config.fai = job.addChildJobFn(run_samtools_faidx, config.reference).rv()\n config.dict = job.addChildJobFn(run_picard_create_sequence_dictionary, config.reference).rv()\n job.addFollowOnJobFn(map_job, download_sample, samples, config)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef download_sample(job, sample, config):\n # Create copy of config that is sample specific\n config = argparse.Namespace(**vars(config))\n uuid, normal_url, tumor_url = sample\n job.fileStore.logToMaster('Downloaded sample: ' + uuid)\n config.uuid = uuid\n config.normal = normal_url\n config.tumor = tumor_url\n config.cores = min(config.maxCores, int(multiprocessing.cpu_count()))\n disk = '1G' if config.ci_test else '20G'\n # Download sample bams and launch pipeline\n config.normal_bam = job.addChildJobFn(download_url_job, url=config.normal, s3_key_path=config.ssec,\n cghub_key_path=config.gtkey, disk=disk).rv()\n config.tumor_bam = job.addChildJobFn(download_url_job, url=config.tumor, s3_key_path=config.ssec,\n cghub_key_path=config.gtkey, disk=disk).rv()\n job.addFollowOnJobFn(index_bams, config)", "response": "Download sample and store sample specific attributes"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef index_bams(job, config):\n job.fileStore.logToMaster('Indexed sample BAMS: ' + config.uuid)\n disk = '1G' if config.ci_test else '20G'\n config.normal_bai = job.addChildJobFn(run_samtools_index, config.normal_bam, cores=1, disk=disk).rv()\n config.tumor_bai = job.addChildJobFn(run_samtools_index, config.tumor_bam, cores=1, disk=disk).rv()\n job.addFollowOnJobFn(preprocessing_declaration, config)", "response": "This function is used to index the sample BAMs in the cluster."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef preprocessing_declaration(job, config):\n if config.preprocessing:\n job.fileStore.logToMaster('Ran preprocessing: ' + config.uuid)\n disk = '1G' if config.ci_test else '20G'\n mem = '2G' if config.ci_test else '10G'\n processed_normal = job.wrapJobFn(run_gatk_preprocessing, config.normal_bam, config.normal_bai,\n config.reference, config.dict, config.fai, config.phase, config.mills,\n config.dbsnp, mem, cores=1, memory=mem, disk=disk)\n processed_tumor = job.wrapJobFn(run_gatk_preprocessing, config.tumor_bam, config.tumor_bai,\n config.reference, config.dict, config.fai, config.phase, config.mills,\n config.dbsnp, mem, cores=1, memory=mem, disk=disk)\n static_workflow = job.wrapJobFn(static_workflow_declaration, config, processed_normal.rv(0),\n processed_normal.rv(1), processed_tumor.rv(0), processed_tumor.rv(1))\n job.addChild(processed_normal)\n job.addChild(processed_tumor)\n job.addFollowOn(static_workflow)\n else:\n job.addFollowOnJobFn(static_workflow_declaration, config, config.normal_bam, config.normal_bai,\n config.tumor_bam, config.tumor_bai)", "response": "This function is used to define the job that will run the preprocessing of the current language."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncombining the contents of separate tarball outputs into one.", "response": "def consolidate_output(job, config, mutect, pindel, muse):\n \"\"\"\n Combine the contents of separate tarball outputs into one via streaming\n\n :param JobFunctionWrappingJob job: passed automatically by Toil\n :param Namespace config: Argparse Namespace object containing argument inputs\n :param str mutect: MuTect tarball FileStoreID\n :param str pindel: Pindel tarball FileStoreID\n :param str muse: MuSe tarball FileStoreID\n \"\"\"\n work_dir = job.fileStore.getLocalTempDir()\n mutect_tar, pindel_tar, muse_tar = None, None, None\n if mutect:\n mutect_tar = job.fileStore.readGlobalFile(mutect, os.path.join(work_dir, 'mutect.tar.gz'))\n if pindel:\n pindel_tar = job.fileStore.readGlobalFile(pindel, os.path.join(work_dir, 'pindel.tar.gz'))\n if muse:\n muse_tar = job.fileStore.readGlobalFile(muse, os.path.join(work_dir, 'muse.tar.gz'))\n out_tar = os.path.join(work_dir, config.uuid + '.tar.gz')\n # Consolidate separate tarballs into one as streams (avoids unnecessary untaring)\n tar_list = [x for x in [mutect_tar, pindel_tar, muse_tar] if x is not None]\n with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:\n for tar in tar_list:\n with tarfile.open(tar, 'r') as f_in:\n for tarinfo in f_in:\n with closing(f_in.extractfile(tarinfo)) as f_in_file:\n if tar is mutect_tar:\n tarinfo.name = os.path.join(config.uuid, 'mutect', os.path.basename(tarinfo.name))\n elif tar is pindel_tar:\n tarinfo.name = os.path.join(config.uuid, 'pindel', os.path.basename(tarinfo.name))\n else:\n tarinfo.name = os.path.join(config.uuid, 'muse', os.path.basename(tarinfo.name))\n f_out.addfile(tarinfo, fileobj=f_in_file)\n # Move to output location\n if urlparse(config.output_dir).scheme == 's3':\n job.fileStore.logToMaster('Uploading {} to S3: {}'.format(config.uuid, config.output_dir))\n s3am_upload(job=job, fpath=out_tar, s3_dir=config.output_dir, num_cores=config.cores)\n else:\n job.fileStore.logToMaster('Moving {} to output dir: {}'.format(config.uuid, config.output_dir))\n mkdir_p(config.output_dir)\n copy_files(file_paths=[out_tar], output_dir=config.output_dir)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nparses the manifest file and returns a list of samples and their attributes as defined in the manifest file.", "response": "def parse_manifest(path_to_manifest):\n \"\"\"\n Parses samples, specified in either a manifest or listed with --samples\n\n :param str path_to_manifest: Path to configuration file\n :return: Samples and their attributes as defined in the manifest\n :rtype: list[list]\n \"\"\"\n samples = []\n with open(path_to_manifest, 'r') as f:\n for line in f.readlines():\n if not line.isspace() and not line.startswith('#'):\n sample = line.strip().split('\\t')\n require(len(sample) == 3, 'Bad manifest format! '\n 'Expected 3 tab separated columns, got: {}'.format(sample))\n uuid, normal, tumor = sample\n for url in [normal, tumor]:\n require(urlparse(url).scheme and urlparse(url), 'Invalid URL passed for {}'.format(url))\n samples.append(sample)\n return samples"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef download_reference_files(job, inputs, samples):\n # Create dictionary to store FileStoreIDs of shared input files\n shared_ids = {}\n urls = [('amb', inputs.amb), ('ann', inputs.ann), ('bwt', inputs.bwt),\n ('pac', inputs.pac), ('sa', inputs.sa)]\n # Alt file is optional and can only be provided, not generated\n if inputs.alt:\n urls.append(('alt', inputs.alt))\n # Download reference\n download_ref = job.wrapJobFn(download_url_job, inputs.ref, disk='3G') # Human genomes are typically ~3G\n job.addChild(download_ref)\n shared_ids['ref'] = download_ref.rv()\n # If FAI is provided, download it. Otherwise, generate it\n if inputs.fai:\n shared_ids['fai'] = job.addChildJobFn(download_url_job, inputs.fai).rv()\n else:\n faidx = job.wrapJobFn(run_samtools_faidx, download_ref.rv())\n shared_ids['fai'] = download_ref.addChild(faidx).rv()\n # If all BWA index files are provided, download them. Otherwise, generate them\n if all(x[1] for x in urls):\n for name, url in urls:\n shared_ids[name] = job.addChildJobFn(download_url_job, url).rv()\n else:\n job.fileStore.logToMaster('BWA index files not provided, creating now')\n bwa_index = job.wrapJobFn(run_bwa_index, download_ref.rv())\n download_ref.addChild(bwa_index)\n for x, name in enumerate(['amb', 'ann', 'bwt', 'pac', 'sa']):\n shared_ids[name] = bwa_index.rv(x)\n\n # Map_job distributes one sample in samples to the downlaod_sample_and_align function\n job.addFollowOnJobFn(map_job, download_sample_and_align, samples, inputs, shared_ids)", "response": "Downloads shared files that are used by all samples for alignment and generates them if they are not provided."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef download_sample_and_align(job, sample, inputs, ids):\n uuid, urls = sample\n r1_url, r2_url = urls if len(urls) == 2 else (urls[0], None)\n job.fileStore.logToMaster('Downloaded sample: {0}. R1 {1}\\nR2 {2}\\nStarting BWA Run'.format(uuid, r1_url, r2_url))\n # Read fastq samples from file store\n ids['r1'] = job.addChildJobFn(download_url_job, r1_url, s3_key_path=inputs.ssec, disk=inputs.file_size).rv()\n if r2_url:\n ids['r2'] = job.addChildJobFn(download_url_job, r2_url, s3_key_path=inputs.ssec, disk=inputs.file_size).rv()\n else:\n ids['r2'] = None\n # Create config for bwakit\n inputs.cores = min(inputs.maxCores, multiprocessing.cpu_count())\n inputs.uuid = uuid\n config = dict(**vars(inputs)) # Create config as a copy of inputs since it has values we want\n config.update(ids) # Overwrite attributes with the FileStoreIDs from ids\n config = argparse.Namespace(**config)\n # Define and wire job functions\n bam_id = job.wrapJobFn(run_bwakit, config, sort=inputs.sort, trim=inputs.trim,\n disk=inputs.file_size, cores=inputs.cores)\n job.addFollowOn(bam_id)\n output_name = uuid + '.bam' + str(inputs.suffix) if inputs.suffix else uuid + '.bam'\n if urlparse(inputs.output_dir).scheme == 's3':\n bam_id.addChildJobFn(s3am_upload_job, file_id=bam_id.rv(), file_name=output_name, s3_dir=inputs.output_dir,\n s3_key_path=inputs.ssec, cores=inputs.cores, disk=inputs.file_size)\n else:\n mkdir_p(inputs.ouput_dir)\n bam_id.addChildJobFn(copy_file_job, name=output_name, file_id=bam_id.rv(), output_dir=inputs.output_dir,\n disk=inputs.file_size)", "response": "Downloads the sample and runs BWA -kit on it"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing manifest file to list of UUIDs and URLs.", "response": "def parse_manifest(manifest_path):\n \"\"\"\n Parse manifest file\n\n :param str manifest_path: Path to manifest file\n :return: samples\n :rtype: list[str, list]\n \"\"\"\n samples = []\n with open(manifest_path, 'r') as f:\n for line in f:\n if not line.isspace() and not line.startswith('#'):\n sample = line.strip().split('\\t')\n require(2 <= len(sample) <= 3, 'Bad manifest format! '\n 'Expected UUID\\tURL1\\t[URL2] (tab separated), got: {}'.format(sample))\n uuid = sample[0]\n urls = sample[1:]\n for url in urls:\n require(urlparse(url).scheme and urlparse(url), 'Invalid URL passed for {}'.format(url))\n samples.append([uuid, urls])\n return samples"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef main():\n # Define Parser object and add to Toil\n parser = argparse.ArgumentParser(description=main.__doc__, formatter_class=argparse.RawTextHelpFormatter)\n subparsers = parser.add_subparsers(dest='command')\n # Generate subparsers\n subparsers.add_parser('generate-config', help='Generates an editable config in the current working directory.')\n subparsers.add_parser('generate-manifest', help='Generates an editable manifest in the current working directory.')\n subparsers.add_parser('generate', help='Generates a config and manifest in the current working directory.')\n # Run subparser\n parser_run = subparsers.add_parser('run', help='Runs the BWA alignment pipeline')\n group = parser_run.add_mutually_exclusive_group()\n parser_run.add_argument('--config', default='config-toil-bwa.yaml', type=str,\n help='Path to the (filled in) config file, generated with \"generate-config\".')\n group.add_argument('--manifest', default='manifest-toil-bwa.tsv', type=str,\n help='Path to the (filled in) manifest file, generated with \"generate-manifest\". '\n '\\nDefault value: \"%(default)s\".')\n group.add_argument('--sample', nargs='+', action=required_length(2, 3),\n help='Space delimited sample UUID and fastq files in the format: uuid url1 [url2].')\n # Print docstring help if no arguments provided\n if len(sys.argv) == 1:\n parser.print_help()\n sys.exit(1)\n Job.Runner.addToilOptions(parser_run)\n args = parser.parse_args()\n # Parse subparsers related to generation of config and manifest\n cwd = os.getcwd()\n if args.command == 'generate-config' or args.command == 'generate':\n generate_file(os.path.join(cwd, 'config-toil-bwa.yaml'), generate_config)\n if args.command == 'generate-manifest' or args.command == 'generate':\n generate_file(os.path.join(cwd, 'manifest-toil-bwa.tsv'), generate_manifest)\n # Pipeline execution\n elif args.command == 'run':\n require(os.path.exists(args.config), '{} not found. Please run generate-config'.format(args.config))\n if not args.sample:\n args.sample = None\n require(os.path.exists(args.manifest), '{} not found and no sample provided. '\n 'Please run \"generate-manifest\"'.format(args.manifest))\n # Parse config\n parsed_config = {x.replace('-', '_'): y for x, y in yaml.load(open(args.config).read()).iteritems()}\n config = argparse.Namespace(**parsed_config)\n config.maxCores = int(args.maxCores) if args.maxCores else sys.maxint\n samples = [args.sample[0], args.sample[1:]] if args.sample else parse_manifest(args.manifest)\n # Sanity checks\n require(config.ref, 'Missing URL for reference file: {}'.format(config.ref))\n require(config.output_dir, 'No output location specified: {}'.format(config.output_dir))\n # Launch Pipeline\n Job.Runner.startToil(Job.wrapJobFn(download_reference_files, config, samples), args)", "response": "This function is the main function of the BWA alignment pipeline. It is used to generate and run the alignment pipeline."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert an address string to a long.", "response": "def _address2long(address):\n \"\"\"\n Convert an address string to a long.\n \"\"\"\n parsed = ipv4.ip2long(address)\n if parsed is None:\n parsed = ipv6.ip2long(address)\n return parsed"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the 0 - based index of item in this IpRange.", "response": "def index(self, item):\n \"\"\"\n Return the 0-based position of `item` in this IpRange.\n\n\n >>> r = IpRange('127.0.0.1', '127.255.255.255')\n >>> r.index('127.0.0.1')\n 0\n >>> r.index('127.255.255.255')\n 16777214\n >>> r.index('10.0.0.1')\n Traceback (most recent call last):\n ...\n ValueError: 10.0.0.1 is not in range\n\n\n :param item: Dotted-quad ip address.\n :type item: str\n :returns: Index of ip address in range\n \"\"\"\n item = self._cast(item)\n offset = item - self.startIp\n if offset >= 0 and offset < self._len:\n return offset\n raise ValueError('%s is not in range' % self._ipver.long2ip(item))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndetaches the daemon process.", "response": "def _detach_process():\n \"\"\"\n Detach daemon process.\n\n Forks the current process into a parent and a detached child. The\n child process resides in its own process group, has no controlling\n terminal attached and is cleaned up by the init process.\n\n Returns ``True`` for the parent and ``False`` for the child.\n \"\"\"\n # To detach from our process group we need to call ``setsid``. We\n # can only do that if we aren't a process group leader. Therefore\n # we fork once, which makes sure that the new child process is not\n # a process group leader.\n pid = os.fork()\n if pid > 0:\n # Parent process\n # Use waitpid to \"collect\" the child process and avoid Zombies\n os.waitpid(pid, 0)\n return True\n os.setsid()\n # We now fork a second time and let the second's fork parent exit.\n # This makes the second fork's child process an orphan. Orphans are\n # cleaned up by the init process, so we won't end up with a zombie.\n # In addition, the second fork's child is no longer a session\n # leader and can therefore never acquire a controlling terminal.\n pid = os.fork()\n if pid > 0:\n os._exit(os.EX_OK)\n return False"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _block(predicate, timeout):\n if timeout:\n if timeout is True:\n timeout = float('Inf')\n timeout = time.time() + timeout\n while not predicate() and time.time() < timeout:\n time.sleep(0.1)\n return predicate()", "response": "Block until a predicate returns a true value."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread the PID of the process owning the lock.", "response": "def read_pid(self):\n \"\"\"\n Return the PID of the process owning the lock.\n\n Returns ``None`` if no lock is present.\n \"\"\"\n try:\n with open(self._path, 'r') as f:\n s = f.read().strip()\n if not s:\n return None\n return int(s)\n except IOError as e:\n if e.errno == errno.ENOENT:\n return None\n raise"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_logger_file_handles(self):\n handles = []\n for handler in self.logger.handlers:\n # The following code works for logging's SysLogHandler,\n # StreamHandler, SocketHandler, and their subclasses.\n for attr in ['sock', 'socket', 'stream']:\n try:\n handle = getattr(handler, attr)\n if handle:\n handles.append(handle)\n break\n except AttributeError:\n continue\n return handles", "response": "Find the file handles used by our logger s handlers."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef is_running(self):\n pid = self.get_pid()\n if pid is None:\n return False\n # The PID file may still exist even if the daemon isn't running,\n # for example if it has crashed.\n try:\n os.kill(pid, 0)\n except OSError as e:\n if e.errno == errno.ESRCH:\n # In this case the PID file shouldn't have existed in\n # the first place, so we remove it\n self.pid_file.release()\n return False\n # We may also get an exception if we're not allowed to use\n # kill on the process, but that means that the process does\n # exist, which is all we care about here.\n return True", "response": "Check if the daemon is running."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_signal_event(self, s):\n '''\n Get the event for a signal.\n\n Checks if the signal has been enabled and raises a\n ``ValueError`` if not.\n '''\n try:\n return self._signal_events[int(s)]\n except KeyError:\n raise ValueError('Signal {} has not been enabled'.format(s))", "response": "Get the event for a signal."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef send_signal(self, s):\n self._get_signal_event(s) # Check if signal has been enabled\n pid = self.get_pid()\n if not pid:\n raise ValueError('Daemon is not running.')\n os.kill(pid, s)", "response": "Send a signal to the daemon process."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends a SIGTERM signal to the daemon process requesting it to terminate.", "response": "def stop(self, block=False):\n \"\"\"\n Tell the daemon process to stop.\n\n Sends the SIGTERM signal to the daemon process, requesting it\n to terminate.\n\n If ``block`` is true then the call blocks until the daemon\n process has exited. This may take some time since the daemon\n process will complete its on-going backup activities before\n shutting down. ``block`` can either be ``True`` (in which case\n it blocks indefinitely) or a timeout in seconds.\n\n The return value is ``True`` if the daemon process has been\n stopped and ``False`` otherwise.\n\n .. versionadded:: 0.3\n The ``block`` parameter\n \"\"\"\n self.send_signal(signal.SIGTERM)\n return _block(lambda: not self.is_running(), block)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nkill the daemon process.", "response": "def kill(self, block=False):\n \"\"\"\n Kill the daemon process.\n\n Sends the SIGKILL signal to the daemon process, killing it. You\n probably want to try :py:meth:`stop` first.\n\n If ``block`` is true then the call blocks until the daemon\n process has exited. ``block`` can either be ``True`` (in which\n case it blocks indefinitely) or a timeout in seconds.\n\n Returns ``True`` if the daemon process has (already) exited and\n ``False`` otherwise.\n\n The PID file is always removed, whether the process has already\n exited or not. Note that this means that subsequent calls to\n :py:meth:`is_running` and :py:meth:`get_pid` will behave as if\n the process has exited. If you need to be sure that the process\n has already exited, set ``block`` to ``True``.\n\n .. versionadded:: 0.5.1\n The ``block`` parameter\n \"\"\"\n pid = self.get_pid()\n if not pid:\n raise ValueError('Daemon is not running.')\n try:\n os.kill(pid, signal.SIGKILL)\n return _block(lambda: not self.is_running(), block)\n except OSError as e:\n if e.errno == errno.ESRCH:\n raise ValueError('Daemon is not running.')\n raise\n finally:\n self.pid_file.release()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef start(self, block=False):\n pid = self.get_pid()\n if pid:\n raise ValueError('Daemon is already running at PID %d.' % pid)\n\n # The default is to place the PID file into ``/var/run``. This\n # requires root privileges. Since not having these is a common\n # problem we check a priori whether we can create the lock file.\n try:\n self.pid_file.acquire()\n finally:\n self.pid_file.release()\n\n # Clear previously received SIGTERMs. This must be done before\n # the calling process returns so that the calling process can\n # call ``stop`` directly after ``start`` returns without the\n # signal being lost.\n self.clear_signal(signal.SIGTERM)\n\n if _detach_process():\n # Calling process returns\n return _block(lambda: self.is_running(), block)\n # Daemon process continues here\n self._debug('Daemon has detached')\n\n def on_signal(s, frame):\n self._debug('Received signal {}'.format(s))\n self._signal_events[int(s)].set()\n\n def runner():\n try:\n # We acquire the PID as late as possible, since its\n # existence is used to verify whether the service\n # is running.\n self.pid_file.acquire()\n self._debug('PID file has been acquired')\n self._debug('Calling `run`')\n self.run()\n self._debug('`run` returned without exception')\n except Exception as e:\n self.logger.exception(e)\n except SystemExit:\n self._debug('`run` called `sys.exit`')\n try:\n self.pid_file.release()\n self._debug('PID file has been released')\n except Exception as e:\n self.logger.exception(e)\n os._exit(os.EX_OK) # FIXME: This seems redundant\n\n try:\n setproctitle.setproctitle(self.name)\n self._debug('Process title has been set')\n files_preserve = (self.files_preserve +\n self._get_logger_file_handles())\n signal_map = {s: on_signal for s in self._signal_events}\n signal_map.update({\n signal.SIGTTIN: None,\n signal.SIGTTOU: None,\n signal.SIGTSTP: None,\n })\n with DaemonContext(\n detach_process=False,\n signal_map=signal_map,\n files_preserve=files_preserve):\n self._debug('Daemon context has been established')\n\n # Python's signal handling mechanism only forwards signals to\n # the main thread and only when that thread is doing something\n # (e.g. not when it's waiting for a lock, etc.). If we use the\n # main thread for the ``run`` method this means that we cannot\n # use the synchronization devices from ``threading`` for\n # communicating the reception of SIGTERM to ``run``. Hence we\n # use a separate thread for ``run`` and make sure that the\n # main loop receives signals. See\n # https://bugs.python.org/issue1167930\n thread = threading.Thread(target=runner)\n thread.start()\n while thread.is_alive():\n time.sleep(1)\n except Exception as e:\n self.logger.exception(e)\n\n # We need to shutdown the daemon process at this point, because\n # otherwise it will continue executing from after the original\n # call to ``start``.\n os._exit(os.EX_OK)", "response": "Start the daemon process."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create(opts):\n if opts['--address'] and is_boot2docker():\n raise DatacatsError('Cannot specify address on boot2docker.')\n return create_environment(\n environment_dir=opts['ENVIRONMENT_DIR'],\n port=opts['PORT'],\n create_skin=not opts['--bare'],\n start_web=not opts['--image-only'],\n create_sysadmin=not opts['--no-sysadmin'],\n site_name=opts['--site'],\n ckan_version=opts['--ckan'],\n address=opts['--address'],\n log_syslog=opts['--syslog'],\n datapusher=not opts['--no-datapusher'],\n site_url=opts['--site-url'],\n interactive=opts['--interactive'],\n init_db=not opts['--no-init-db'],\n )", "response": "Create a new environment with the given options."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef reset(environment, opts):\n # pylint: disable=unused-argument\n if not opts['--yes']:\n y_or_n_prompt('Reset will remove all data related to the '\n 'site {} and recreate the database'.format(opts['--site']))\n\n print 'Resetting...'\n environment.stop_supporting_containers()\n environment.stop_ckan()\n clean_pyc(environment)\n # Save the port.\n saved_port = environment.port\n environment.purge_data([opts['--site']], never_delete=True)\n init({\n 'ENVIRONMENT_DIR': opts['ENVIRONMENT'],\n '--site': opts['--site'],\n 'PORT': saved_port,\n '--syslog': None,\n '--address': None,\n '--image-only': False,\n '--interactive': opts['--interactive'],\n '--no-init-db': False,\n '--no-sysadmin': opts['--no-sysadmin'],\n '--site-url': None\n }, no_install=True)", "response": "Resets a site to the default state."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize a purged environment or copied environment directory", "response": "def init(opts, no_install=False, quiet=False):\n \"\"\"Initialize a purged environment or copied environment directory\n\nUsage:\n datacats init [-in] [--syslog] [-s NAME] [--address=IP] [--interactive]\n [--site-url SITE_URL] [ENVIRONMENT_DIR [PORT]] [--no-init-db]\n\nOptions:\n --address=IP Address to listen on (Linux-only)\n --interactive Don't detach from the web container\n -i --image-only Create the environment but don't start containers\n --no-init-db Don't initialize the database. Useful for importing other CKANs\n -n --no-sysadmin Don't prompt for an initial sysadmin user account\n -s --site=NAME Pick a site to initialize [default: primary]\n --site-url SITE_URL The site_url to use in API responses (e.g. http://example.org:{port}/)\n --syslog Log to the syslog\n\nENVIRONMENT_DIR is an existing datacats environment directory. Defaults to '.'\n\"\"\"\n if opts['--address'] and is_boot2docker():\n raise DatacatsError('Cannot specify address on boot2docker.')\n environment_dir = opts['ENVIRONMENT_DIR']\n port = opts['PORT']\n address = opts['--address']\n start_web = not opts['--image-only']\n create_sysadmin = not opts['--no-sysadmin']\n site_name = opts['--site']\n site_url = opts['--site-url']\n interactive = opts['--interactive']\n init_db = not opts['--no-init-db']\n\n environment_dir = abspath(environment_dir or '.')\n log_syslog = opts['--syslog']\n\n environment = Environment.load(environment_dir, site_name)\n if address:\n environment.address = address\n if port:\n environment.port = int(port)\n if site_url:\n environment.site_url = site_url\n\n try:\n if environment.sites and site_name in environment.sites:\n raise DatacatsError('Site named {0} already exists.'\n .format(site_name))\n # There are a couple of steps we can/must skip if we're making a sub-site only\n making_full_environment = not environment.data_exists()\n\n if not quiet:\n write('Creating environment {0}/{1} '\n 'from existing environment directory \"{0}\"'\n .format(environment.name, environment.site_name))\n steps = [\n lambda: environment.create_directories(create_project_dir=False)] + ([\n environment.save,\n environment.create_virtualenv\n ] if making_full_environment else []) + [\n environment.save_site,\n environment.start_supporting_containers,\n environment.fix_storage_permissions,\n ]\n\n for fn in steps:\n fn()\n if not quiet:\n write('.')\n if not quiet:\n write('\\n')\n except:\n if not quiet:\n print\n raise\n\n return finish_init(environment, start_web, create_sysadmin,\n log_syslog=log_syslog, do_install=not no_install,\n quiet=quiet, site_url=site_url, interactive=interactive,\n init_db=init_db)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfinish the create and init process.", "response": "def finish_init(environment, start_web, create_sysadmin, log_syslog=False,\n do_install=True, quiet=False, site_url=None, interactive=False,\n init_db=True):\n \"\"\"\n Common parts of create and init: Install, init db, start site, sysadmin\n \"\"\"\n if not init_db:\n start_web = False\n create_sysadmin = False\n\n if do_install:\n install_all(environment, False, verbose=False, quiet=quiet)\n\n if init_db:\n if not quiet:\n write('Initializing database')\n environment.install_postgis_sql()\n environment.ckan_db_init()\n if not quiet:\n write('\\n')\n\n if site_url:\n try:\n site_url = site_url.format(address=environment.address, port=environment.port)\n environment.site_url = site_url\n environment.save_site(False)\n except (KeyError, IndexError, ValueError) as e:\n raise DatacatsError('Could not parse site_url: {}'.format(e))\n\n if start_web:\n environment.start_ckan(log_syslog=log_syslog)\n if not quiet and not interactive:\n write('Starting web server at {0} ...\\n'.format(\n environment.web_address()))\n\n if create_sysadmin:\n try:\n adminpw = confirm_password()\n environment.create_admin_set_password(adminpw)\n except KeyboardInterrupt:\n print\n\n if not start_web:\n environment.stop_supporting_containers()"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsaves user profile settings into user profile directory", "response": "def save(self):\n \"\"\"\n Save profile settings into user profile directory\n \"\"\"\n config = self.profiledir + '/config'\n if not isdir(self.profiledir):\n makedirs(self.profiledir)\n\n cp = SafeConfigParser()\n\n cp.add_section('ssh')\n cp.set('ssh', 'private_key', self.ssh_private_key)\n cp.set('ssh', 'public_key', self.ssh_public_key)\n\n with open(config, 'w') as cfile:\n cp.write(cfile)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates a new ssh private and public key", "response": "def generate_ssh_key(self):\n \"\"\"\n Generate a new ssh private and public key\n \"\"\"\n web_command(\n command=[\"ssh-keygen\", \"-q\", \"-t\", \"rsa\", \"-N\", \"\", \"-C\",\n \"datacats generated {0}@{1}\".format(\n getuser(), gethostname()),\n \"-f\", \"/output/id_rsa\"],\n rw={self.profiledir: '/output'},\n )"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create(self, environment, target_name):\n remote_server_command(\n [\"ssh\", environment.deploy_target, \"create\", target_name],\n environment, self,\n clean_up=True,\n )", "response": "Sends create project command to the remote server"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset admin password for the current user", "response": "def admin_password(self, environment, target_name, password):\n \"\"\"\n Return True if password was set successfully\n \"\"\"\n try:\n remote_server_command(\n [\"ssh\", environment.deploy_target,\n \"admin_password\", target_name, password],\n environment, self,\n clean_up=True\n )\n return True\n except WebCommandError:\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndeploys the data to the remote server.", "response": "def deploy(self, environment, target_name, stream_output=None):\n \"\"\"\n Return True if deployment was successful\n \"\"\"\n try:\n remote_server_command(\n [\n \"rsync\", \"-lrv\", \"--safe-links\", \"--munge-links\",\n \"--delete\", \"--inplace\", \"--chmod=ugo=rwX\",\n \"--exclude=.datacats-environment\",\n \"--exclude=.git\",\n \"/project/.\",\n environment.deploy_target + ':' + target_name\n ],\n environment, self,\n include_project_dir=True,\n stream_output=stream_output,\n clean_up=True,\n )\n except WebCommandError as e:\n raise DatacatsError(\n \"Unable to deploy `{0}` to remote server for some reason:\\n\"\n \" datacats was not able to copy data to the remote server\"\n .format((target_name,)),\n parent_exception=e\n )\n\n try:\n remote_server_command(\n [\n \"ssh\", environment.deploy_target, \"install\", target_name,\n ],\n environment, self,\n clean_up=True,\n )\n return True\n except WebCommandError as e:\n raise DatacatsError(\n \"Unable to deploy `{0}` to remote server for some reason:\\n\"\n \"datacats copied data to the server but failed to register\\n\"\n \"(or `install`) the new catalog\"\n .format((target_name,)),\n parent_exception=e\n )"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef num_batches(n, batch_size):\n b = n // batch_size\n if n % batch_size > 0:\n b += 1\n return b", "response": "Compute the number of mini - batches required to cover a data set of size batch_size."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef num_indices_generated(self):\n if self.repeats == -1:\n return np.inf\n else:\n return self.length * self.repeats", "response": "Returns the number of indices that would be generated by this sampler."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngenerates an iterator that generates mini - batches of sample in - order.", "response": "def in_order_indices_batch_iterator(self, batch_size):\n \"\"\"\n Create an iterator that generates in-order mini-batches of sample\n indices. The batches will have `batch_size` elements, with the\n exception of the final batch which will have less if there are not\n enough samples left to fill it.\n\n The generated mini-batches indices take the form of 1D NumPy integer\n arrays.\n\n Parameters\n ----------\n batch_size: int\n Mini-batch size\n\n Returns\n -------\n iterator\n An iterator that generates mini-batches in the form of 1D NumPy\n integer arrays.\n \"\"\"\n if self.repeats == 1:\n for i in range(0, self.length, batch_size):\n yield np.arange(i, min(i + batch_size, self.length))\n else:\n repeats = self.repeats\n i = 0\n while True:\n j = i + batch_size\n if j <= self.length:\n # Within size of data\n yield np.arange(i, j)\n i = j\n elif j <= self.length * 2:\n # One restart is required\n # Reduce the number of remaining repeats\n if repeats != -1:\n repeats -= 1\n if repeats == 0:\n # Finished; emit remaining elements\n if i < self.length:\n yield np.arange(i, self.length)\n break\n\n # Wrap over\n # Compute number of elements required to make up\n # the batch\n k = batch_size - (self.length - i)\n yield np.append(np.arange(i, self.length),\n np.arange(0, k), axis=0)\n i = k\n else:\n # Multiple restarts required to fill the batch\n batch_ndx = np.arange(0)\n # i = 0\n while len(batch_ndx) < batch_size:\n # Wrap over\n k = min(batch_size - len(batch_ndx), self.length - i)\n batch_ndx = np.append(\n batch_ndx, np.arange(i, i + k), axis=0)\n i += k\n if i >= self.length:\n i -= self.length\n # Reduce the number of remaining repeats\n if repeats != -1:\n repeats -= 1\n if repeats == 0:\n break\n\n if len(batch_ndx) > 0:\n yield batch_ndx\n if repeats == 0:\n break"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef shuffled_indices_batch_iterator(self, batch_size, shuffle_rng):\n if self.repeats == 1:\n indices = shuffle_rng.permutation(self.length)\n for i in range(0, self.length, batch_size):\n yield indices[i:i + batch_size]\n else:\n repeats = self.repeats\n indices = shuffle_rng.permutation(self.length)\n i = 0\n while True:\n j = i + batch_size\n if j <= self.length:\n # Within size of data\n yield indices[i:j]\n i = j\n else:\n # Multiple restarts required to fill the batch\n batch_ndx = np.arange(0)\n while len(batch_ndx) < batch_size:\n # Wrap over\n k = min(batch_size - len(batch_ndx), self.length - i)\n batch_ndx = np.append(\n batch_ndx, indices[i:i + k], axis=0)\n i += k\n\n if i >= self.length:\n # Loop over; new permutation\n indices = shuffle_rng.permutation(self.length)\n i -= self.length\n # Reduce the number of remaining repeats\n if repeats != -1:\n repeats -= 1\n if repeats == 0:\n break\n\n if len(batch_ndx) > 0:\n yield batch_ndx\n if repeats == 0:\n break", "response": "Generates a batch of random integers from the internal memory."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef shuffled_indices_batch_iterator(self, batch_size, shuffle_rng):\n while True:\n yield shuffle_rng.choice(len(self.weights), size=(batch_size,),\n p=self.weights)", "response": "Create an iterator that generates randomly shuffled mini - batches of sample indices."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef class_balancing_sample_weights(y):\n h = np.bincount(y)\n cls_weight = 1.0 / (h.astype(float) * len(np.nonzero(h)[0]))\n cls_weight[np.isnan(cls_weight)] = 0.0\n sample_weight = cls_weight[y]\n return sample_weight", "response": "Compute the sample weight given an array of sample classes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate an iterator that generates randomly shuffled mini - batches of sample indices.", "response": "def shuffled_indices_batch_iterator(self, batch_size, shuffle_rng):\n \"\"\"\n Create an iterator that generates randomly shuffled mini-batches of\n sample indices. The batches will have `batch_size` elements.\n\n The generated mini-batches indices take the form of 1D NumPy integer\n arrays.\n\n Parameters\n ----------\n batch_size: int\n Mini-batch size\n shuffle_rng: a `numpy.random.RandomState` that will be used to\n randomise element order.\n\n Returns\n -------\n iterator\n An iterator that generates mini-batches in the form of 1D NumPy\n integer arrays.\n \"\"\"\n while True:\n yield shuffle_rng.choice(self.indices, size=(batch_size,),\n p=self.sub_weights)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef class_balancing_sampler(y, indices):\n weights = WeightedSampler.class_balancing_sample_weights(y[indices])\n return WeightedSubsetSampler(weights, indices=indices)", "response": "Constructs a WeightedSubsetSampler that compensates for class balancing of the samples drawn from the data y."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwait up to timeout seconds for a service at the specified url. Returns True if service becomes available False if timeout is reached.", "response": "def wait_for_service_available(container, url, timeout):\n \"\"\"\n Wait up to timeout seconds for service at host:port\n to start.\n\n Returns True if service becomes available, False if the\n container stops or raises ServiceTimeout if timeout is\n reached.\n \"\"\"\n start = time.time()\n remaining = timeout\n while True:\n remaining = start + timeout - time.time()\n if remaining < 0:\n raise ServiceTimeout\n try:\n response = get(url,\n timeout=min(remaining, REQUEST_TIMEOUT_SECONDS))\n if 500 <= response.status_code < 600:\n return False\n return True\n except (ConnectionError, Timeout):\n pass\n if not inspect_container(container)['State']['Running']:\n return False\n\n remaining = start + timeout - time.time()\n delay = max(0, min(RETRY_DELAY_SECONDS, remaining))\n time.sleep(delay)\n\n raise ServiceTimeout"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the path of the given file within the batchup data directory.", "response": "def get_data_path(filename):\n \"\"\"\n Get the path of the given file within the batchup data directory\n\n Parameters\n ----------\n filename: str\n The filename to locate within the batchup data directory\n\n Returns\n -------\n str\n The full path of the file\n \"\"\"\n if os.path.isabs(filename):\n return filename\n else:\n return os.path.join(get_data_dir(), filename)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndownloads a file from a given URL to a given path. If the path does not exist it will be created.", "response": "def download(path, source_url):\n \"\"\"\n Download a file to a given path from a given URL, if it does not exist.\n\n Parameters\n ----------\n path: str\n The (destination) path of the file on the local filesystem\n source_url: str\n The URL from which to download the file\n\n Returns\n -------\n str\n The path of the file\n \"\"\"\n dir_path = os.path.dirname(path)\n if not os.path.exists(dir_path):\n os.makedirs(dir_path)\n if not os.path.exists(path):\n print('Downloading {} to {}'.format(source_url, path))\n filename = source_url.split('/')[-1]\n\n def _progress(count, block_size, total_size):\n sys.stdout.write('\\rDownloading {} {:.2%}'.format(\n filename, float(count * block_size) / float(total_size)))\n sys.stdout.flush()\n try:\n urlretrieve(source_url, path, reporthook=_progress)\n except:\n sys.stdout.write('\\r')\n # Exception; remove any partially downloaded file and re-raise\n if os.path.exists(path):\n os.remove(path)\n raise\n sys.stdout.write('\\r')\n return path"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncomputes the SHA - 256 hash of the file at the given path", "response": "def compute_sha256(path):\n \"\"\"\n Compute the SHA-256 hash of the file at the given path\n\n Parameters\n ----------\n path: str\n The path of the file\n\n Returns\n -------\n str\n The SHA-256 HEX digest\n \"\"\"\n hasher = hashlib.sha256()\n with open(path, 'rb') as f:\n # 10MB chunks\n for chunk in iter(lambda: f.read(10 * 1024 * 1024), b''):\n hasher.update(chunk)\n return hasher.hexdigest()"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nverify the integrity of a file.", "response": "def verify_file(path, sha256):\n \"\"\"\n Verify the integrity of a file by checking its SHA-256 hash.\n If no digest is supplied, the digest is printed to the console.\n\n Closely follows the code in `torchvision.datasets.utils.check_integrity`\n\n Parameters\n ----------\n path: str\n The path of the file to check\n sha256: str\n The expected SHA-256 hex digest of the file, or `None` to print the\n digest of the file to the console\n\n Returns\n -------\n bool\n Indicates if the file passes the integrity check or not\n \"\"\"\n if not os.path.isfile(path):\n return False\n digest = compute_sha256(path)\n if sha256 is None:\n # No digest supplied; report it to the console so a develop can fill\n # it in\n print('SHA-256 of {}:'.format(path))\n print(' \"{}\"'.format(digest))\n else:\n if digest != sha256:\n return False\n return True"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndownloads a file to a given path from a given URL and verify it.", "response": "def download_and_verify(path, source_url, sha256):\n \"\"\"\n Download a file to a given path from a given URL, if it does not exist.\n After downloading it, verify it integrity by checking the SHA-256 hash.\n\n Parameters\n ----------\n path: str\n The (destination) path of the file on the local filesystem\n source_url: str\n The URL from which to download the file\n sha256: str\n The expected SHA-256 hex digest of the file, or `None` to print the\n digest of the file to the console\n\n Returns\n -------\n str or None\n The path of the file if successfully downloaded otherwise `None`\n \"\"\"\n if os.path.exists(path):\n # Already exists?\n # Nothing to do, except print the SHA-256 if necessary\n if sha256 is None:\n print('The SHA-256 of {} is \"{}\"'.format(\n path, compute_sha256(path)))\n return path\n\n # Compute the path of the unverified file\n unverified_path = path + '.unverified'\n for i in range(_MAX_DOWNLOAD_TRIES):\n # Download it\n try:\n unverified_path = download(unverified_path, source_url)\n except Exception as e:\n # Report failure\n print(\n 'Download of {} unsuccessful; error {}; '\n 'deleting and re-trying...'.format(source_url, e))\n # Delete so that we can retry\n if os.path.exists(unverified_path):\n os.remove(unverified_path)\n else:\n if os.path.exists(unverified_path):\n # Got something...\n if verify_file(unverified_path, sha256):\n # Success: rename the unverified file to the destination\n # filename\n os.rename(unverified_path, path)\n return path\n else:\n # Report failure\n print(\n 'Download of {} unsuccessful; verification failed; '\n 'deleting and re-trying...'.format(source_url))\n # Delete so that we can retry\n os.remove(unverified_path)\n\n print('Did not succeed in downloading {} (tried {} times)'.format(\n source_url, _MAX_DOWNLOAD_TRIES\n ))\n return None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef copy_and_verify(path, source_path, sha256):\n if os.path.exists(path):\n # Already exists?\n # Nothing to do, except print the SHA-256 if necessary\n if sha256 is None:\n print('The SHA-256 of {} is \"{}\"'.format(\n path, compute_sha256(path)))\n return path\n\n if not os.path.exists(source_path):\n return None\n\n # Compute the path of the unverified file\n unverified_path = path + '.unverified'\n # Copy it\n dir_path = os.path.dirname(path)\n if not os.path.exists(dir_path):\n os.makedirs(dir_path)\n shutil.copy(source_path, unverified_path)\n\n if os.path.exists(unverified_path):\n # Got something...\n if verify_file(unverified_path, sha256):\n # Success: rename the unverified file to the destination\n # filename\n os.rename(unverified_path, path)\n return path\n else:\n # Report failure\n print('SHA verification of file {} failed'.format(source_path))\n # Delete\n os.remove(unverified_path)\n return None", "response": "Copy a file to a given path from a given path if it does not exist and verify it."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a ckan extension template.", "response": "def ckan_extension_template(name, target):\n \"\"\"\n Create ckanext-(name) in target directory.\n \"\"\"\n setupdir = '{0}/ckanext-{1}theme'.format(target, name)\n extdir = setupdir + '/ckanext/{0}theme'.format(name)\n templatedir = extdir + '/templates/'\n staticdir = extdir + '/static/datacats'\n\n makedirs(templatedir + '/home/snippets')\n makedirs(staticdir)\n\n here = dirname(__file__)\n copyfile(here + '/images/chart.png', staticdir + '/chart.png')\n copyfile(here + '/images/datacats-footer.png',\n staticdir + '/datacats-footer.png')\n\n filecontents = [\n (setupdir + '/setup.py', SETUP_PY),\n (setupdir + '/.gitignore', DOT_GITIGNORE),\n (setupdir + '/ckanext/__init__.py', NAMESPACE_PACKAGE),\n (extdir + '/__init__.py', ''),\n (extdir + '/plugins.py', PLUGINS_PY),\n (templatedir + '/home/snippets/promoted.html', PROMOTED_SNIPPET),\n (templatedir + '/footer.html', FOOTER_HTML),\n ]\n\n for filename, content in filecontents:\n with open(filename, 'w') as f:\n f.write(content.replace('##name##', name))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef shell(environment, opts):\n environment.require_data()\n environment.start_supporting_containers()\n return environment.interactive_shell(\n opts['COMMAND'],\n detach=opts['--detach']\n )", "response": "Run a command or interactive shell within this environment"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef paster(opts):\n environment = Environment.load('.')\n environment.require_data()\n environment.start_supporting_containers()\n\n if not opts['COMMAND']:\n opts['COMMAND'] = ['--', 'help']\n\n assert opts['COMMAND'][0] == '--'\n return environment.interactive_shell(\n opts['COMMAND'][1:],\n paster=True,\n detach=opts['--detach']\n )", "response": "Run a paster command from the current directory"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef save_new_site(site_name, sitedir, srcdir, port, address, site_url,\n passwords):\n \"\"\"\n Add a site's configuration to the source dir and site dir\n \"\"\"\n cp = ConfigParser.SafeConfigParser()\n cp.read([srcdir + '/.datacats-environment'])\n\n section_name = 'site_' + site_name\n\n if not cp.has_section(section_name):\n cp.add_section(section_name)\n cp.set(section_name, 'port', str(port))\n if address:\n cp.set(section_name, 'address', address)\n\n if site_url:\n cp.set(section_name, 'site_url', site_url)\n\n with open(srcdir + '/.datacats-environment', 'w') as config:\n cp.write(config)\n\n # save passwords to datadir\n cp = ConfigParser.SafeConfigParser()\n\n cp.add_section('passwords')\n for n in sorted(passwords):\n cp.set('passwords', n.lower(), passwords[n])\n\n # Write to the sitedir so we maintain separate passwords.\n with open(sitedir + '/passwords.ini', 'w') as config:\n cp.write(config)", "response": "Save a new site to the source dir and site dir"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef save_new_environment(name, datadir, srcdir, ckan_version,\n deploy_target=None, always_prod=False):\n \"\"\"\n Save an environment's configuration to the source dir and data dir\n \"\"\"\n with open(datadir + '/.version', 'w') as f:\n f.write('2')\n\n cp = ConfigParser.SafeConfigParser()\n\n cp.read(srcdir + '/.datacats-environment')\n\n if not cp.has_section('datacats'):\n cp.add_section('datacats')\n cp.set('datacats', 'name', name)\n cp.set('datacats', 'ckan_version', ckan_version)\n\n if deploy_target:\n if not cp.has_section('deploy'):\n cp.add_section('deploy')\n cp.set('deploy', 'target', deploy_target)\n\n if always_prod:\n cp.set('datacats', 'always_prod', 'true')\n\n with open(srcdir + '/.datacats-environment', 'w') as config:\n cp.write(config)\n\n save_srcdir_location(datadir, srcdir)", "response": "Save a new environment to the source dir and data dir."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef find_environment_dirs(environment_name=None, data_only=False):\n docker.require_images()\n\n if environment_name is None:\n environment_name = '.'\n\n extension_dir = 'ckan'\n if validate.valid_name(environment_name) and path.isdir(\n path.expanduser('~/.datacats/' + environment_name)):\n # loading from a name\n datadir = path.expanduser('~/.datacats/' + environment_name)\n with open(datadir + '/project-dir') as pd:\n srcdir = pd.read()\n\n if not data_only and not path.exists(srcdir + '/.datacats-environment'):\n raise DatacatsError(\n 'Environment data found but environment directory is'\n ' missing. Try again from the new environment directory'\n ' location or remove the environment data with'\n ' \"datacats purge\"')\n\n return srcdir, extension_dir, datadir\n\n # loading from a path\n srcdir = path.abspath(environment_name)\n if not path.isdir(srcdir):\n raise DatacatsError('No environment found with that name')\n\n wd = srcdir\n oldwd = None\n while not path.exists(wd + '/.datacats-environment'):\n oldwd = wd\n wd, _ = path.split(wd)\n if wd == oldwd:\n raise DatacatsError(\n 'Environment not found in {0} or above'.format(srcdir))\n srcdir = wd\n\n if oldwd:\n _, extension_dir = path.split(oldwd)\n\n return srcdir, extension_dir, None", "response": "finds all directories in the specified environment"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_environment(srcdir, datadir=None, allow_old=False):\n cp = ConfigParser.SafeConfigParser()\n try:\n cp.read([srcdir + '/.datacats-environment'])\n except ConfigParser.Error:\n raise DatacatsError('Error reading environment information')\n\n name = cp.get('datacats', 'name')\n\n if datadir:\n # update the link in case user moved their srcdir\n save_srcdir_location(datadir, srcdir)\n else:\n datadir = path.expanduser('~/.datacats/' + name)\n # FIXME: check if datadir is sane, project-dir points back to srcdir\n\n if migrate.needs_format_conversion(datadir) and not allow_old:\n raise DatacatsError('This environment uses an old format. You must'\n ' migrate to the new format. To do so, use the'\n ' \"datacats migrate\" command.')\n\n if migrate.is_locked(datadir):\n raise DatacatsError('Migration in progress, cannot continue.\\n'\n 'If you interrupted a migration, you should'\n ' attempt manual recovery or contact us by'\n ' filing an issue at http://github.com/datacats/'\n 'datacats.\\nAs a last resort, you could delete'\n ' all your stored data and create a new environment'\n ' by running \"datacats purge\" followed by'\n ' \"datacats init\".')\n\n # FIXME: consider doing data_complete check here\n\n ckan_version = cp.get('datacats', 'ckan_version')\n try:\n always_prod = cp.getboolean('datacats', 'always_prod')\n except ConfigParser.NoOptionError:\n always_prod = False\n\n try:\n extra_containers = cp.get('datacats', 'extra_containers').split(' ')\n except ConfigParser.NoOptionError:\n extra_containers = ()\n\n # if remote_server's custom ssh connection\n # address is defined,\n # we overwrite the default datacats.com one\n try:\n deploy_target = cp.get('deploy', 'remote_server_user') \\\n + \"@\" + cp.get('deploy', 'remote_server')\n except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):\n deploy_target = DEFAULT_REMOTE_SERVER_TARGET\n\n # if remote_server's ssh public key is given,\n # we overwrite the default datacats.com one\n try:\n remote_server_key = cp.get('deploy', 'remote_server_key')\n except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):\n remote_server_key = None\n\n return (datadir, name, ckan_version, always_prod, deploy_target,\n remote_server_key, extra_containers)", "response": "Load the configuration values for an environment."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads configuration values for a site.", "response": "def load_site(srcdir, datadir, site_name=None):\n \"\"\"\n Load configuration values for a site.\n\n Returns (port, address, site_url, passwords)\n \"\"\"\n if site_name is None:\n site_name = 'primary'\n if not validate.valid_name(site_name):\n raise DatacatsError('{} is not a valid site name.'.format(site_name))\n\n cp = ConfigParser.SafeConfigParser()\n try:\n cp.read([srcdir + '/.datacats-environment'])\n except ConfigParser.Error:\n raise DatacatsError('Error reading environment information')\n\n site_section = 'site_' + site_name\n try:\n port = cp.getint(site_section, 'port')\n except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):\n port = None\n try:\n address = cp.get(site_section, 'address')\n except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):\n address = None\n try:\n site_url = cp.get(site_section, 'site_url')\n except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):\n site_url = None\n\n passwords = {}\n cp = ConfigParser.SafeConfigParser()\n cp.read(datadir + '/sites/' + site_name + '/passwords.ini')\n try:\n pw_options = cp.options('passwords')\n except ConfigParser.NoSectionError:\n pw_options = []\n\n for n in pw_options:\n passwords[n.upper()] = cp.get('passwords', n)\n\n return port, address, site_url, passwords"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef new_environment_check(srcpath, site_name, ckan_version):\n docker.require_images()\n\n workdir, name = path.split(path.abspath(path.expanduser(srcpath)))\n\n if not validate.valid_name(name):\n raise DatacatsError('Please choose an environment name starting'\n ' with a letter and including only lowercase letters'\n ' and digits')\n if not path.isdir(workdir):\n raise DatacatsError('Parent directory for environment'\n ' does not exist')\n\n datadir = path.expanduser('~/.datacats/' + name)\n sitedir = datadir + '/sites/' + site_name\n # We track through the datadir to the target if we are just making a\n # site\n if path.isdir(datadir):\n with open(datadir + '/project-dir') as pd:\n srcdir = pd.read()\n else:\n srcdir = workdir + '/' + name\n\n if ckan_version not in SUPPORTED_PRELOADS:\n raise DatacatsError('''Datacats does not currently support CKAN version {}.\nVersions that are currently supported are: {}'''.format(ckan_version,\n ', '.join(SUPPORTED_PRELOADS)))\n\n preload_name = str(ckan_version)\n\n # Get all the versions from the tags\n downloaded_versions = [tag for tag in docker.get_tags('datacats/ckan')]\n\n if ckan_version not in downloaded_versions:\n retrying_pull_image('datacats/ckan:{}'.format(preload_name))\n\n if path.isdir(sitedir):\n raise DatacatsError('Site data directory {0} already exists'.format(\n sitedir))\n # This is the case where the data dir has been removed,\n if path.isdir(srcdir) and not path.isdir(datadir):\n raise DatacatsError('Environment directory exists, but data directory does not.\\n'\n 'If you simply want to recreate the data directory, run '\n '\"datacats init\" in the environment directory.')\n\n return name, datadir, srcdir", "response": "Check if a new environment or site can be created at the given path."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef data_complete(datadir, sitedir, get_container_name):\n if any(not path.isdir(sitedir + x)\n for x in ('/files', '/run', '/solr')):\n return False\n\n if docker.is_boot2docker():\n # Inspect returns None if the container doesn't exist.\n return all(docker.inspect_container(get_container_name(x))\n for x in ('pgdata', 'venv'))\n\n return path.isdir(datadir + '/venv') and path.isdir(sitedir + '/postgres')", "response": "Return True if the directories and containers we re expecting\n are present in datadir sitedir and containers\n Return False if the directories and containers we re expecting\n are present in datadir sitedir and containers\n Return False if the directories and containers we re expecting\n are present in datadir sitedir and containers\n "} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate expected directories in datadir sitedir and srcdir.", "response": "def create_directories(datadir, sitedir, srcdir=None):\n \"\"\"\n Create expected directories in datadir, sitedir\n and optionally srcdir\n \"\"\"\n # It's possible that the datadir already exists\n # (we're making a secondary site)\n if not path.isdir(datadir):\n os.makedirs(datadir, mode=0o700)\n try:\n # This should take care if the 'site' subdir if needed\n os.makedirs(sitedir, mode=0o700)\n except OSError:\n raise DatacatsError(\"Site already exists.\")\n\n # venv isn't site-specific, the rest are.\n if not docker.is_boot2docker():\n if not path.isdir(datadir + '/venv'):\n os.makedirs(datadir + '/venv')\n os.makedirs(sitedir + '/postgres')\n os.makedirs(sitedir + '/solr')\n os.makedirs(sitedir + '/files')\n os.makedirs(sitedir + '/run')\n\n if srcdir:\n os.makedirs(srcdir)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_virtualenv(srcdir, datadir, preload_image, get_container_name):\n try:\n if docker.is_boot2docker():\n docker.data_only_container(\n get_container_name('venv'),\n ['/usr/lib/ckan'],\n )\n img_id = docker.web_command(\n '/bin/mv /usr/lib/ckan/ /usr/lib/ckan_original',\n image=preload_image,\n commit=True,\n )\n docker.web_command(\n command='/bin/cp -a /usr/lib/ckan_original/. /usr/lib/ckan/.',\n volumes_from=get_container_name('venv'),\n image=img_id,\n )\n docker.remove_image(img_id)\n return\n\n docker.web_command(\n command='/bin/cp -a /usr/lib/ckan/. /usr/lib/ckan_target/.',\n rw={datadir + '/venv': '/usr/lib/ckan_target'},\n image=preload_image,\n )\n finally:\n rw = {datadir + '/venv': '/usr/lib/ckan'} if not docker.is_boot2docker() else {}\n volumes_from = get_container_name('venv') if docker.is_boot2docker() else None\n # fix venv permissions\n docker.web_command(\n command='/bin/chown -R --reference=/project /usr/lib/ckan',\n rw=rw,\n volumes_from=volumes_from,\n ro={srcdir: '/project'},\n )", "response": "Create a new virtual environment."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_source(srcdir, preload_image, datapusher=False):\n try:\n docker.web_command(\n command='/bin/cp -a /project/ckan /project_target/ckan',\n rw={srcdir: '/project_target'},\n image=preload_image)\n if datapusher:\n docker.web_command(\n command='/bin/cp -a /project/datapusher /project_target/datapusher',\n rw={srcdir: '/project_target'},\n image=preload_image)\n shutil.copy(\n srcdir + '/ckan/ckan/config/who.ini',\n srcdir)\n shutil.copy(\n srcdir + '/ckan/ckan/config/solr/schema.xml',\n srcdir)\n finally:\n # fix srcdir permissions\n docker.web_command(\n command='/bin/chown -R --reference=/project /project',\n rw={srcdir: '/project'},\n )", "response": "Create source file from source directory."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef start_supporting_containers(sitedir, srcdir, passwords,\n get_container_name, extra_containers, log_syslog=False):\n \"\"\"\n Start all supporting containers (containers required for CKAN to\n operate) if they aren't already running, along with some extra\n containers specified by the user\n \"\"\"\n if docker.is_boot2docker():\n docker.data_only_container(get_container_name('pgdata'),\n ['/var/lib/postgresql/data'])\n rw = {}\n volumes_from = get_container_name('pgdata')\n else:\n rw = {sitedir + '/postgres': '/var/lib/postgresql/data'}\n volumes_from = None\n\n running = set(containers_running(get_container_name))\n\n needed = set(extra_containers).union({'postgres', 'solr'})\n\n if not needed.issubset(running):\n stop_supporting_containers(get_container_name, extra_containers)\n\n # users are created when data dir is blank so we must pass\n # all the user passwords as environment vars\n # XXX: postgres entrypoint magic\n docker.run_container(\n name=get_container_name('postgres'),\n image='datacats/postgres',\n environment=passwords,\n rw=rw,\n volumes_from=volumes_from,\n log_syslog=log_syslog)\n\n docker.run_container(\n name=get_container_name('solr'),\n image='datacats/solr',\n rw={sitedir + '/solr': '/var/lib/solr'},\n ro={srcdir + '/schema.xml': '/etc/solr/conf/schema.xml'},\n log_syslog=log_syslog)\n\n for container in extra_containers:\n # We don't know a whole lot about the extra containers so we're just gonna have to\n # mount /project and /datadir r/o even if they're not needed for ease of\n # implementation.\n docker.run_container(\n name=get_container_name(container),\n image=EXTRA_IMAGE_MAPPING[container],\n ro={\n sitedir: '/datadir',\n srcdir: '/project'\n },\n log_syslog=log_syslog\n )", "response": "Start all the containers required for CKAN to operate on CKAN."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstopping postgres and solr containers along with any specified extra containers.", "response": "def stop_supporting_containers(get_container_name, extra_containers):\n \"\"\"\n Stop postgres and solr containers, along with any specified extra containers\n \"\"\"\n docker.remove_container(get_container_name('postgres'))\n docker.remove_container(get_container_name('solr'))\n for container in extra_containers:\n docker.remove_container(get_container_name(container))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef containers_running(get_container_name):\n running = []\n for n in ['web', 'postgres', 'solr', 'datapusher', 'redis']:\n info = docker.inspect_container(get_container_name(n))\n if info and not info['State']['Running']:\n running.append(n + '(halted)')\n elif info:\n running.append(n)\n return running", "response": "Return a list of containers that are running in this environment"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _load_sites(self):\n if not self.sites:\n self.sites = task.list_sites(self.datadir)\n return self.sites", "response": "Loads all of the sites from the datadir and stores them in self. sites."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef save_site(self, create=True):\n self._load_sites()\n if create:\n self.sites.append(self.site_name)\n\n task.save_new_site(self.site_name, self.sitedir, self.target, self.port,\n self.address, self.site_url, self.passwords)", "response": "Save environment settings in the directory that need to be saved\n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsaving environment settings into environment directory overwriting any existing configuration and discarding site config", "response": "def save(self):\n \"\"\"\n Save environment settings into environment directory, overwriting\n any existing configuration and discarding site config\n \"\"\"\n task.save_new_environment(self.name, self.datadir, self.target,\n self.ckan_version, self.deploy_target, self.always_prod)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef new(cls, path, ckan_version, site_name, **kwargs):\n if ckan_version == 'master':\n ckan_version = 'latest'\n name, datadir, srcdir = task.new_environment_check(path, site_name, ckan_version)\n environment = cls(name, srcdir, datadir, site_name, ckan_version, **kwargs)\n environment._generate_passwords()\n return environment", "response": "Create a new Environment object with settings for a new project."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load(cls, environment_name=None, site_name='primary', data_only=False, allow_old=False):\n srcdir, extension_dir, datadir = task.find_environment_dirs(\n environment_name, data_only)\n\n if datadir and data_only:\n return cls(environment_name, None, datadir, site_name)\n\n (datadir, name, ckan_version, always_prod, deploy_target,\n remote_server_key, extra_containers) = task.load_environment(srcdir, datadir, allow_old)\n\n if not allow_old:\n (port, address, site_url, passwords) = task.load_site(srcdir, datadir, site_name)\n else:\n (port, address, site_url, passwords) = (None, None, None, None)\n\n environment = cls(name, srcdir, datadir, site_name, ckan_version=ckan_version,\n port=port, deploy_target=deploy_target, site_url=site_url,\n always_prod=always_prod, address=address,\n extension_dir=extension_dir,\n remote_server_key=remote_server_key,\n extra_containers=extra_containers)\n\n if passwords:\n environment.passwords = passwords\n else:\n environment._generate_passwords()\n\n if not allow_old:\n environment._load_sites()\n return environment", "response": "Load an existing environment and site."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning True if all the expected datadir files are present in the container", "response": "def data_complete(self):\n \"\"\"\n Return True if all the expected datadir files are present\n \"\"\"\n return task.data_complete(self.datadir, self.sitedir,\n self._get_container_name)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef require_data(self):\n files = task.source_missing(self.target)\n if files:\n raise DatacatsError('Missing files in source directory:\\n' +\n '\\n'.join(files))\n if not self.data_exists():\n raise DatacatsError('Environment datadir missing. '\n 'Try \"datacats init\".')\n if not self.data_complete():\n raise DatacatsError('Environment datadir damaged or volumes '\n 'missing. '\n 'To reset and discard all data use '\n '\"datacats reset\"')", "response": "Checks if the data directory exists and if not raises a DatacatsError"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate the directories for the current project.", "response": "def create_directories(self, create_project_dir=True):\n \"\"\"\n Call once for new projects to create the initial project directories.\n \"\"\"\n return task.create_directories(self.datadir, self.sitedir,\n self.target if create_project_dir else None)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\npopulate venv from preloaded image", "response": "def create_virtualenv(self):\n \"\"\"\n Populate venv from preloaded image\n \"\"\"\n return task.create_virtualenv(self.target, self.datadir,\n self._preload_image(), self._get_container_name)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef clean_virtualenv(self):\n self.user_run_script(\n script=scripts.get_script_path('clean_virtualenv.sh'),\n args=[],\n rw_venv=True,\n )", "response": "Clean our virtualenv so that new dependencies may be installed."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_source(self, datapusher=True):\n task.create_source(self.target, self._preload_image(), datapusher)", "response": "Create source image and copy it to target directory"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef start_supporting_containers(self, log_syslog=False):\n log_syslog = True if self.always_prod else log_syslog\n # in production we always use log_syslog driver (to aggregate all the logs)\n task.start_supporting_containers(\n self.sitedir,\n self.target,\n self.passwords,\n self._get_container_name,\n self.extra_containers,\n log_syslog=log_syslog\n )", "response": "Start all supporting containers."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a new development. ini file for the current locale.", "response": "def create_ckan_ini(self):\n \"\"\"\n Use make-config to generate an initial development.ini file\n \"\"\"\n self.run_command(\n command='/scripts/run_as_user.sh /usr/lib/ckan/bin/paster make-config'\n ' ckan /project/development.ini',\n rw_project=True,\n ro={scripts.get_script_path('run_as_user.sh'): '/scripts/run_as_user.sh'},\n )"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update_ckan_ini(self, skin=True):\n command = [\n '/usr/lib/ckan/bin/paster', '--plugin=ckan', 'config-tool',\n '/project/development.ini', '-e',\n 'sqlalchemy.url = postgresql://',\n 'ckan.datastore.read_url = postgresql://',\n 'ckan.datastore.write_url = postgresql://',\n 'ckan.datapusher.url = http://datapusher:8800',\n 'solr_url = http://solr:8080/solr',\n 'ckan.storage_path = /var/www/storage',\n 'ckan.plugins = datastore resource_proxy text_view ' +\n ('datapusher ' if exists(self.target + '/datapusher') else '')\n + 'recline_grid_view recline_graph_view'\n + (' {0}_theme'.format(self.name) if skin else ''),\n 'ckan.site_title = ' + self.name,\n 'ckan.site_logo =',\n 'ckan.auth.create_user_via_web = false',\n ]\n self.run_command(command=command, rw_project=True)", "response": "Update development. ini with our environment settings."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates an example ckan extension and install it", "response": "def create_install_template_skin(self):\n \"\"\"\n Create an example ckan extension for this environment and install it\n \"\"\"\n ckan_extension_template(self.name, self.target)\n self.install_package_develop('ckanext-' + self.name + 'theme')"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrunning db init to create all ckan tables", "response": "def ckan_db_init(self, retry_seconds=DB_INIT_RETRY_SECONDS):\n \"\"\"\n Run db init to create all ckan tables\n\n :param retry_seconds: how long to retry waiting for db to start\n \"\"\"\n # XXX workaround for not knowing how long we need to wait\n # for postgres to be ready. fix this by changing the postgres\n # entrypoint, or possibly running once with command=/bin/true\n started = time.time()\n while True:\n try:\n self.run_command(\n '/usr/lib/ckan/bin/paster --plugin=ckan db init '\n '-c /project/development.ini',\n db_links=True,\n clean_up=True,\n )\n break\n except WebCommandError:\n if started + retry_seconds > time.time():\n raise\n time.sleep(DB_INIT_RETRY_DELAY)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nstarts the apache server or paster.", "response": "def start_ckan(self, production=False, log_syslog=False, paster_reload=True,\n interactive=False):\n \"\"\"\n Start the apache server or paster serve\n\n :param log_syslog: A flag to redirect all container logs to host's syslog\n :param production: True for apache, False for paster serve + debug on\n :param paster_reload: Instruct paster to watch for file changes\n \"\"\"\n self.stop_ckan()\n\n address = self.address or '127.0.0.1'\n port = self.port\n # in prod we always use log_syslog driver\n log_syslog = True if self.always_prod else log_syslog\n\n production = production or self.always_prod\n # We only override the site URL with the docker URL on three conditions\n override_site_url = (self.address is None\n and not is_boot2docker()\n and not self.site_url)\n command = ['/scripts/web.sh', str(production), str(override_site_url), str(paster_reload)]\n\n # XXX nasty hack, remove this once we have a lessc command\n # for users (not just for building our preload image)\n if not production:\n css = self.target + '/ckan/ckan/public/base/css'\n if not exists(css + '/main.debug.css'):\n from shutil import copyfile\n copyfile(css + '/main.css', css + '/main.debug.css')\n\n ro = {\n self.target: '/project',\n scripts.get_script_path('datapusher.sh'): '/scripts/datapusher.sh'\n }\n\n if not is_boot2docker():\n ro[self.datadir + '/venv'] = '/usr/lib/ckan'\n\n datapusher = self.needs_datapusher()\n if datapusher:\n run_container(\n self._get_container_name('datapusher'),\n 'datacats/web',\n '/scripts/datapusher.sh',\n ro=ro,\n volumes_from=(self._get_container_name('venv') if is_boot2docker() else None),\n log_syslog=log_syslog)\n\n while True:\n self._create_run_ini(port, production)\n try:\n self._run_web_container(port, command, address, log_syslog=log_syslog,\n datapusher=datapusher, interactive=interactive)\n if not is_boot2docker():\n self.address = address\n except PortAllocatedError:\n port = self._next_port(port)\n continue\n break"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate run. ini in datadir with correct db passwords inserted", "response": "def _create_run_ini(self, port, production, output='development.ini',\n source='development.ini', override_site_url=True):\n \"\"\"\n Create run/development.ini in datadir with debug and site_url overridden\n and with correct db passwords inserted\n \"\"\"\n cp = SafeConfigParser()\n try:\n cp.read([self.target + '/' + source])\n except ConfigParserError:\n raise DatacatsError('Error reading development.ini')\n\n cp.set('DEFAULT', 'debug', 'false' if production else 'true')\n\n if self.site_url:\n site_url = self.site_url\n else:\n if is_boot2docker():\n web_address = socket.gethostbyname(docker_host())\n else:\n web_address = self.address\n site_url = 'http://{}:{}'.format(web_address, port)\n\n if override_site_url:\n cp.set('app:main', 'ckan.site_url', site_url)\n\n cp.set('app:main', 'sqlalchemy.url',\n 'postgresql://ckan:{0}@db:5432/ckan'\n .format(self.passwords['CKAN_PASSWORD']))\n cp.set('app:main', 'ckan.datastore.read_url',\n 'postgresql://ckan_datastore_readonly:{0}@db:5432/ckan_datastore'\n .format(self.passwords['DATASTORE_RO_PASSWORD']))\n cp.set('app:main', 'ckan.datastore.write_url',\n 'postgresql://ckan_datastore_readwrite:{0}@db:5432/ckan_datastore'\n .format(self.passwords['DATASTORE_RW_PASSWORD']))\n cp.set('app:main', 'solr_url', 'http://solr:8080/solr')\n cp.set('app:main', 'ckan.redis.url', 'http://redis:6379')\n cp.set('app:main', 'beaker.session.secret', self.passwords['BEAKER_SESSION_SECRET'])\n\n if not isdir(self.sitedir + '/run'):\n makedirs(self.sitedir + '/run') # upgrade old datadir\n with open(self.sitedir + '/run/' + output, 'w') as runini:\n cp.write(runini)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _run_web_container(self, port, command, address, log_syslog=False,\n datapusher=True, interactive=False):\n \"\"\"\n Start web container on port with command\n \"\"\"\n if is_boot2docker():\n ro = {}\n volumes_from = self._get_container_name('venv')\n else:\n ro = {self.datadir + '/venv': '/usr/lib/ckan'}\n volumes_from = None\n\n links = {\n self._get_container_name('solr'): 'solr',\n self._get_container_name('postgres'): 'db'\n }\n\n links.update({self._get_container_name(container): container\n for container in self.extra_containers})\n\n if datapusher:\n if 'datapusher' not in self.containers_running():\n raise DatacatsError(container_logs(self._get_container_name('datapusher'), \"all\",\n False, False))\n links[self._get_container_name('datapusher')] = 'datapusher'\n\n ro = dict({\n self.target: '/project/',\n scripts.get_script_path('web.sh'): '/scripts/web.sh',\n scripts.get_script_path('adjust_devini.py'): '/scripts/adjust_devini.py'},\n **ro)\n rw = {\n self.sitedir + '/files': '/var/www/storage',\n self.sitedir + '/run/development.ini': '/project/development.ini'\n }\n try:\n if not interactive:\n run_container(\n name=self._get_container_name('web'),\n image='datacats/web',\n rw=rw,\n ro=ro,\n links=links,\n volumes_from=volumes_from,\n command=command,\n port_bindings={\n 5000: port if is_boot2docker() else (address, port)},\n log_syslog=log_syslog\n )\n else:\n # FIXME: share more code with interactive_shell\n if is_boot2docker():\n switches = ['--volumes-from',\n self._get_container_name('pgdata'), '--volumes-from',\n self._get_container_name('venv')]\n else:\n switches = []\n switches += ['--volume={}:{}:ro'.format(vol, ro[vol]) for vol in ro]\n switches += ['--volume={}:{}'.format(vol, rw[vol]) for vol in rw]\n links = ['--link={}:{}'.format(link, links[link]) for link in links]\n args = ['docker', 'run', '-it', '--name', self._get_container_name('web'),\n '-p', '{}:5000'.format(port) if is_boot2docker()\n else '{}:{}:5000'.format(address, port)] + \\\n switches + links + ['datacats/web', ] + command\n subprocess.call(args)\n except APIError as e:\n if '409' in str(e):\n raise DatacatsError('Web container already running. '\n 'Please stop_web before running.')\n else:\n raise", "response": "Start a web container on port with command"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwaits for the web server to become available or raise DatacatsError", "response": "def wait_for_web_available(self):\n \"\"\"\n Wait for the web server to become available or raise DatacatsError\n if it fails to start.\n \"\"\"\n try:\n if not wait_for_service_available(\n self._get_container_name('web'),\n self.web_address(),\n WEB_START_TIMEOUT_SECONDS):\n raise DatacatsError('Error while starting web container:\\n' +\n container_logs(self._get_container_name('web'), \"all\",\n False, None))\n except ServiceTimeout:\n raise DatacatsError('Timeout while starting web container. Logs:' +\n container_logs(self._get_container_name('web'), \"all\", False, None))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a port number from 5000 - 5999 based on the environment name", "response": "def _choose_port(self):\n \"\"\"\n Return a port number from 5000-5999 based on the environment name\n to be used as a default when the user hasn't selected one.\n \"\"\"\n # instead of random let's base it on the name chosen (and the site name)\n return 5000 + unpack('Q',\n sha((self.name + self.site_name)\n .decode('ascii')).digest()[:8])[0] % 1000"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _next_port(self, port):\n port = 5000 + (port + 1) % 1000\n if port == self.port:\n raise DatacatsError('Too many instances running')\n return port", "response": "Return another port from the 5000 - 5999 range"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef stop_ckan(self):\n remove_container(self._get_container_name('web'), force=True)\n remove_container(self._get_container_name('datapusher'), force=True)", "response": "Stop and remove the web container\n "} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _current_web_port(self):\n info = inspect_container(self._get_container_name('web'))\n if info is None:\n return None\n try:\n if not info['State']['Running']:\n return None\n return info['NetworkSettings']['Ports']['5000/tcp'][0]['HostPort']\n except TypeError:\n return None", "response": "Return the port number for the current web container or None if the container is not running"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_extra_container(self, container, error_on_exists=False):\n if container in self.extra_containers:\n if error_on_exists:\n raise DatacatsError('{} is already added as an extra container.'.format(container))\n else:\n return\n\n self.extra_containers.append(container)\n\n cp = SafeConfigParser()\n cp.read(self.target + '/.datacats-environment')\n\n cp.set('datacats', 'extra_containers', ' '.join(self.extra_containers))\n\n with open(self.target + '/.datacats-environment', 'w') as f:\n cp.write(f)", "response": "Add a container as an extra container."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the url of the web server or None if not running", "response": "def web_address(self):\n \"\"\"\n Return the url of the web server or None if not running\n \"\"\"\n port = self._current_web_port()\n address = self.address or '127.0.0.1'\n if port is None:\n return None\n return 'http://{0}:{1}/'.format(\n address if address and not is_boot2docker() else docker_host(),\n port)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating admin account with given password", "response": "def create_admin_set_password(self, password):\n \"\"\"\n create 'admin' account with given password\n \"\"\"\n with open(self.sitedir + '/run/admin.json', 'w') as out:\n json.dump({\n 'name': 'admin',\n 'email': 'none',\n 'password': password,\n 'sysadmin': True},\n out)\n self.user_run_script(\n script=scripts.get_script_path('update_add_admin.sh'),\n args=[],\n db_links=True,\n ro={\n self.sitedir + '/run/admin.json': '/input/admin.json'\n },\n )\n remove(self.sitedir + '/run/admin.json')"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef interactive_shell(self, command=None, paster=False, detach=False):\n if not exists(self.target + '/.bash_profile'):\n # this file is required for activating the virtualenv\n self.create_bash_profile()\n\n if not command:\n command = []\n use_tty = sys.stdin.isatty() and sys.stdout.isatty()\n\n background = environ.get('CIRCLECI', False) or detach\n\n if is_boot2docker():\n venv_volumes = ['--volumes-from', self._get_container_name('venv')]\n else:\n venv_volumes = ['-v', self.datadir + '/venv:/usr/lib/ckan:rw']\n\n self._create_run_ini(self.port, production=False, output='run.ini')\n self._create_run_ini(self.port, production=True, output='test.ini',\n source='ckan/test-core.ini', override_site_url=False)\n\n script = scripts.get_script_path('shell.sh')\n if paster:\n script = scripts.get_script_path('paster.sh')\n if command and command != ['help'] and command != ['--help']:\n command += ['--config=/project/development.ini']\n command = [self.extension_dir] + command\n\n proxy_settings = self._proxy_settings()\n if proxy_settings:\n venv_volumes += ['-v',\n self.sitedir + '/run/proxy-environment:/etc/environment:ro']\n\n links = {self._get_container_name('solr'): 'solr',\n self._get_container_name('postgres'): 'db'}\n\n links.update({self._get_container_name(container): container for container\n in self.extra_containers})\n\n link_params = []\n\n for link in links:\n link_params.append('--link')\n link_params.append(link + ':' + links[link])\n\n if 'datapusher' in self.containers_running():\n link_params.append('--link')\n link_params.append(self._get_container_name('datapusher') + ':datapusher')\n\n # FIXME: consider switching this to dockerpty\n # using subprocess for docker client's interactive session\n return subprocess.call([\n DOCKER_EXE, 'run',\n ] + (['--rm'] if not background else []) + [\n '-t' if use_tty else '',\n '-d' if detach else '-i',\n ] + venv_volumes + [\n '-v', self.target + ':/project:rw',\n '-v', self.sitedir + '/files:/var/www/storage:rw',\n '-v', script + ':/scripts/shell.sh:ro',\n '-v', scripts.get_script_path('paster_cd.sh') + ':/scripts/paster_cd.sh:ro',\n '-v', self.sitedir + '/run/run.ini:/project/development.ini:ro',\n '-v', self.sitedir +\n '/run/test.ini:/project/ckan/test-core.ini:ro'] +\n link_params +\n ['--hostname', self.name,\n 'datacats/web', '/scripts/shell.sh'] + command)", "response": "launch interactive shell with all writable volumes"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef install_package_requirements(self, psrc, stream_output=None):\n package = self.target + '/' + psrc\n assert isdir(package), package\n reqname = '/requirements.txt'\n if not exists(package + reqname):\n reqname = '/pip-requirements.txt'\n if not exists(package + reqname):\n return\n return self.user_run_script(\n script=scripts.get_script_path('install_reqs.sh'),\n args=['/project/' + psrc + reqname],\n rw_venv=True,\n rw_project=True,\n stream_output=stream_output\n )", "response": "Install from requirements. txt file found in environment directory"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nremoving uploaded files solr index and venv directories", "response": "def purge_data(self, which_sites=None, never_delete=False):\n \"\"\"\n Remove uploaded files, postgres db, solr index, venv\n \"\"\"\n # Default to the set of all sites\n if not exists(self.datadir + '/.version'):\n format_version = 1\n else:\n with open(self.datadir + '/.version') as f:\n format_version = int(f.read().strip())\n\n if format_version == 1:\n print 'WARNING: Defaulting to old purge for version 1.'\n datadirs = ['files', 'solr']\n if is_boot2docker():\n remove_container('datacats_pgdata_{}'.format(self.name))\n remove_container('datacats_venv_{}'.format(self.name))\n else:\n datadirs += ['postgres', 'venv']\n\n web_command(\n command=['/scripts/purge.sh']\n + ['/project/data/' + d for d in datadirs],\n ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},\n rw={self.datadir: '/project/data'},\n )\n shutil.rmtree(self.datadir)\n elif format_version == 2:\n if not which_sites:\n which_sites = self.sites\n\n datadirs = []\n boot2docker = is_boot2docker()\n\n if which_sites:\n if self.target:\n cp = SafeConfigParser()\n cp.read([self.target + '/.datacats-environment'])\n\n for site in which_sites:\n if boot2docker:\n remove_container(self._get_container_name('pgdata'))\n else:\n datadirs += [site + '/postgres']\n # Always rm the site dir & solr & files\n datadirs += [site, site + '/files', site + '/solr']\n if self.target:\n cp.remove_section('site_' + site)\n self.sites.remove(site)\n\n if self.target:\n with open(self.target + '/.datacats-environment', 'w') as conf:\n cp.write(conf)\n\n datadirs = ['sites/' + datadir for datadir in datadirs]\n\n if not self.sites and not never_delete:\n datadirs.append('venv')\n\n web_command(\n command=['/scripts/purge.sh']\n + ['/project/data/' + d for d in datadirs],\n ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},\n rw={self.datadir: '/project/data'},\n )\n\n if not self.sites and not never_delete:\n shutil.rmtree(self.datadir)\n else:\n raise DatacatsError('Unknown format version {}'.format(format_version))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef logs(self, container, tail='all', follow=False, timestamps=False):\n return container_logs(\n self._get_container_name(container),\n tail,\n follow,\n timestamps)", "response": "Returns a generator of log entries for the specified container."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a dictionary of environment variables that define the proxy settings for the current ro mount.", "response": "def _proxy_settings(self):\n \"\"\"\n Create/replace ~/.datacats/run/proxy-environment and return\n entry for ro mount for containers\n \"\"\"\n if not ('https_proxy' in environ or 'HTTPS_PROXY' in environ\n or 'http_proxy' in environ or 'HTTP_PROXY' in environ):\n return {}\n https_proxy = environ.get('https_proxy')\n if https_proxy is None:\n https_proxy = environ.get('HTTPS_PROXY')\n http_proxy = environ.get('http_proxy')\n if http_proxy is None:\n http_proxy = environ.get('HTTP_PROXY')\n no_proxy = environ.get('no_proxy')\n if no_proxy is None:\n no_proxy = environ.get('NO_PROXY', '')\n no_proxy = no_proxy + ',solr,db'\n\n out = [\n 'PATH=\"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:'\n '/bin:/usr/games:/usr/local/games\"\\n']\n if https_proxy is not None:\n out.append('https_proxy=' + posix_quote(https_proxy) + '\\n')\n out.append('HTTPS_PROXY=' + posix_quote(https_proxy) + '\\n')\n if http_proxy is not None:\n out.append('http_proxy=' + posix_quote(http_proxy) + '\\n')\n out.append('HTTP_PROXY=' + posix_quote(http_proxy) + '\\n')\n if no_proxy is not None:\n out.append('no_proxy=' + posix_quote(no_proxy) + '\\n')\n out.append('NO_PROXY=' + posix_quote(no_proxy) + '\\n')\n\n with open(self.sitedir + '/run/proxy-environment', 'w') as f:\n f.write(\"\".join(out))\n\n return {self.sitedir + '/run/proxy-environment': '/etc/environment'}"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the full name of a container of the specified type.", "response": "def _get_container_name(self, container_type):\n \"\"\"\n Gets the full name of a container of the type specified.\n Currently the supported types are:\n - 'venv'\n - 'postgres'\n - 'solr'\n - 'web'\n - 'pgdata'\n - 'lessc'\n - 'datapusher'\n - 'redis'\n The name will be formatted appropriately with any prefixes and postfixes\n needed.\n\n :param container_type: The type of container name to generate (see above).\n \"\"\"\n if container_type in ['venv']:\n return 'datacats_{}_{}'.format(container_type, self.name)\n else:\n return 'datacats_{}_{}_{}'.format(container_type, self.name, self.site_name)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrecompile less files in an environment. Usage: datacats less [ENVIRONMENT] ENVIRONMENT may be an environment name or a path to an environment directory. Default: '.'", "response": "def less(environment, opts):\n # pylint: disable=unused-argument\n \"\"\"Recompiles less files in an environment.\n\nUsage:\n datacats less [ENVIRONMENT]\n\nENVIRONMENT may be an environment name or a path to an environment directory.\nDefault: '.'\n\"\"\"\n require_extra_image(LESSC_IMAGE)\n\n print 'Converting .less files to .css...'\n for log in environment.compile_less():\n print log"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef fetch_and_convert_dataset(source_files, target_filename):\n if not isinstance(target_filename, six.string_types) and \\\n not callable(target_filename):\n raise TypeError(\n 'target_filename must either be a string or be callable (it is '\n 'a {})'.format(type(target_filename)))\n\n for src in source_files:\n if not isinstance(src, AbstractSourceFile):\n raise TypeError('source_files should contain'\n '`AbstractSourceFile` instances, '\n 'not {}'.format(type(src)))\n\n def decorate_fetcher(convert_function):\n def fetch(**kwargs):\n target_fn = path_string(target_filename)\n target_path = config.get_data_path(target_fn)\n\n # If the target file does not exist, we need to acquire the\n # source files and convert them\n if not os.path.exists(target_path):\n # Acquire the source files\n source_paths = []\n for src in source_files:\n p = src.acquire(**kwargs)\n if p is not None:\n if p in source_paths:\n raise ValueError(\n 'Duplicate source file {}'.format(p))\n source_paths.append(p)\n else:\n print('Failed to acquire {}'.format(src))\n return None\n\n # Got the source files\n # Convert\n converted_path = convert_function(source_paths, target_path)\n\n # If successful, delete the source files\n if converted_path is not None:\n for src in source_files:\n src.clean_up()\n\n return converted_path\n else:\n # Target file already exists\n return target_path\n\n fetch.__name__ = convert_function.__name__\n\n return fetch\n\n return decorate_fetcher", "response": "This function is used to fetch the USPS dataset from the BatchUp dataset and convert it into a dataset file that BatchUp can use."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndeletes the cache for a dataset.", "response": "def delete_dataset_cache(*filenames):\n \"\"\"\n Delete the cache (converted files) for a dataset.\n\n Parameters\n ----------\n filenames: str\n Filenames of files to delete\n \"\"\"\n for filename in filenames:\n filename = path_string(filename)\n path = config.get_data_path(filename)\n if os.path.exists(path):\n os.remove(path)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndownload the file and return its path.", "response": "def acquire(self, **kwargs):\n \"\"\"\n Download the file and return its path\n\n Returns\n -------\n str or None\n The path of the file in BatchUp's temporary directory or None if\n the download failed.\n \"\"\"\n return config.download_data(self.temp_filename, self.url,\n self.sha256)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef acquire(self, **kwargs):\n if self.source_path is None:\n source_path = kwargs[self.arg_name]\n else:\n source_path = self.source_path\n return config.copy_data(self.temp_filename, source_path, self.sha256)", "response": "Copy the file and return its path."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef acquire(self, **kwargs):\n path = path_string(self.path)\n if os.path.exists(path):\n if config.verify_file(path, self.sha256):\n return path\n return None", "response": "Acquire the file and return its path."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _retry_func(func, param, num, retry_notif, error_msg):\n for retry_num in range(num):\n if retry_num:\n retry_notif(param, retry_num)\n try:\n func(param)\n return\n except DatacatsError:\n pass\n\n raise DatacatsError(error_msg)", "response": "Retry a function func num times and return the result of func."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nretrieves a result from executing a task.", "response": "def retrieve(self):\n \"\"\"\n Retrieve a result from executing a task. Note that tasks are executed\n in order and that if the next task has not yet completed, this call\n will block until the result is available.\n\n Returns\n -------\n A result from the result buffer.\n \"\"\"\n if len(self.__result_buffer) > 0:\n res = self.__result_buffer.popleft()\n value = res.get()\n else:\n return None\n\n self.__populate_buffer()\n\n return value"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef install(environment, opts):\n environment.require_data()\n install_all(environment, opts['--clean'], verbose=not opts['--quiet'],\n packages=opts['PACKAGE'])\n\n for site in environment.sites:\n environment = Environment.load(environment.name, site)\n if 'web' in environment.containers_running():\n # FIXME: reload without changing debug setting?\n manage.reload_(environment, {\n '--address': opts['--address'],\n '--background': False,\n '--no-watch': False,\n '--production': False,\n 'PORT': None,\n '--syslog': False,\n '--site-url': None,\n '--interactive': False\n })", "response": "Install or reinstall Python packages within this environment"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmigrate an environment to a given revision of the datadir format.", "response": "def migrate(opts):\n \"\"\"Migrate an environment to a given revision of the datadir format.\n\nUsage:\n datacats migrate [-y] [-r VERSION] [ENVIRONMENT_DIR]\n\nOptions:\n -r --revision=VERSION The version of the datadir format you want\n to convert to [default: 2]\n -y --yes Answer yes to all questions.\n\nDefaults to '.' if ENVIRONMENT_DIR isn't specified.\n\"\"\"\n try:\n version = int(opts['--revision'])\n except:\n raise DatacatsError('--revision parameter must be an integer.')\n\n always_yes = opts['--yes']\n\n if 'ENVIRONMENT_DIR' not in opts or not opts['ENVIRONMENT_DIR']:\n cwd = getcwd()\n # Get the dirname\n opts['ENVIRONMENT_DIR'] = split(cwd if cwd[-1] != '/' else cwd[:-1])[1]\n\n datadir = expanduser('~/.datacats/' + opts['ENVIRONMENT_DIR'])\n if needs_format_conversion(datadir, version):\n convert_environment(datadir, version, always_yes)\n print 'Successfully converted datadir {} to format version {}'.format(datadir, version)\n else:\n print 'datadir {} is already at version {}.'.format(datadir, version)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndeploy an environment to production DataCats. com", "response": "def deploy(environment, opts, profile):\n \"\"\"Deploy environment to production DataCats.com cloud service\n\nUsage:\n datacats deploy [--create] [ENVIRONMENT [TARGET_NAME]]\n\nOptions:\n --create Create a new environment on DataCats.com instead\n of updating an existing environment\n\nENVIRONMENT may be an environment name or a path to a environment directory.\nDefault: '.'\n\nTARGET_NAME is the name of the environment on DataCats.com. Defaults to\nthe environment name.\n\"\"\"\n\n target_name = opts['TARGET_NAME']\n if target_name is None:\n target_name = environment.name\n\n if not valid_deploy_name(target_name):\n raise DatacatsError(\" `{target_name}` target name for deployment can't be accepted.\\n\"\n \"Can't have http://{target_name}.datacats.io for your datcat URL\\n\"\n \"Please choose a target name at least 5 characters long,\\n\"\n \"and containing only lowercase letters and numbers\\n\"\n .format(target_name=target_name))\n\n if opts['--create']:\n profile.create(environment, target_name)\n\n profile.deploy(environment, target_name, stdout)\n print \"Deployed source to http://{0}.datacats.io\".format(target_name)\n\n if opts['--create']:\n try:\n pw = confirm_password()\n profile.admin_password(environment, target_name, pw)\n except KeyboardInterrupt:\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _trim_batch(batch, length):\n if isinstance(batch, tuple):\n return tuple([_trim_batch(b, length) for b in batch])\n else:\n return batch[:length]", "response": "Trim the mini - batch to the size length."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef batch_map_concat(func, batch_iter, progress_iter_func=None,\n n_batches=None, prepend_args=None):\n \"\"\"\n Apply a function to all the samples that are accessed as mini-batches\n obtained from an iterator.\n Returns the per-sample results.\n\n The function `func` should return the result for each sample in the\n mini-batch as an array. To return multiple results (e.g. loss and errors)\n return a tuple of arrays (e.g. `(loss_array, error_array)`)\n\n `batch_iter` must be an iterator that generates mini-batches that\n contain samples\n\n Parameters\n ----------\n func: callable `func(*batch) -> results`\n The function to call on each mini-batch. Note that the results\n must be `None`, a tuple or a NumPy array\n batch_iter: data set iterator\n Iterator that generates mini-batches of data\n progress_iter_func: [optional] callable\n `progress_iter_func(iterator, total=total, leave=leave)`\n A `tqdm` style function that will be passed the iterator that\n generates training batches along with the total number of batches\n and `False` for the `leave` parameter. By passing either\n `tqdm.tqdm` or `tqdm.tqdm_notebook` as this argument you can have\n the training loop display a progress bar.\n n_batches: [optional] integer\n Process at most this number of batches before returning.\n prepend_args: [optional] tuple\n Arguments to prepend to the arguments passed to `func`\n\n Returns\n -------\n tuple\n The per-sample sum of the results of the function `func` e.g.\n `(batch_A, batch_B, ...)`\n Returns an empty tuple if there were 0 samples in the data set.\n\n Examples\n --------\n In these examples we will demonstrate the use of `batch_map` to apply\n a function (e.g. a Theano function that runs on the GPU) to samples\n in a data set. We construct an iterator that generates mini-batches from\n the data set and pass it to `batch_map` along with the function that\n we wish to apply. The function will receive the batches and process them.\n\n Define a function to apply to samples:\n >>> def sqr_sum(x):\n ... # Ensure that we receive batches of the expected size:\n ... assert len(x) in {5, 2}\n ... return (x ** 2).sum(axis=1)\n\n Construct data to process and create a data source:\n >>> X = np.random.normal(size=(7, 10))\n >>> ds = ArrayDataSource([X])\n\n Apply the function defined above:\n >>> batch_iter = ds.batch_iterator(batch_size=5)\n >>> X_sqr_sum = batch_map_concat(sqr_sum, batch_iter)\n >>> assert np.allclose(X_sqr_sum[0], (X ** 2).sum(axis=1))\n\n There are also cases where we wish to limit the number of batches that\n will be processed:\n - when the iterator generates an infinite number of samples\n - when the data set is huge and we wish to show results as we go\n Use the `n_batches` argument to limit the number of batches to process:\n >>> X_large = np.random.normal(size=(100, 10))\n >>> ds_large = ArrayDataSource([X_large])\n >>> iter_large = ds_large.batch_iterator(batch_size=5)\n >>> for i in range(10):\n ... partial_result = batch_map_concat(sqr_sum, iter_large, n_batches=2)\n ... # Should have 10 samples per partial result\n ... assert len(partial_result[0]) == 10\n ... j = i * 10\n ... assert np.allclose(partial_result[0],\n ... (X_large[j:j + 10]**2).sum(axis=1))\n \"\"\"\n # Accumulator for results and number of samples\n results = []\n\n # If `progress_iter_func` is not `None`, apply it\n if progress_iter_func is not None:\n batch_iter = progress_iter_func(batch_iter, total=n_batches,\n leave=False)\n\n # Apply `func` to each batch\n n_processed = 0\n for batch in batch_iter:\n # Apply on batch and check the type of the results\n if prepend_args is not None:\n batch_results = func(*(prepend_args + tuple(batch)))\n else:\n batch_results = func(*batch)\n if batch_results is None:\n pass\n elif isinstance(batch_results, np.ndarray):\n batch_results = (batch_results,)\n elif isinstance(batch_results, tuple):\n pass\n else:\n raise TypeError(\n 'Batch function should return a tuple of results, a '\n 'single result as a NumPy array, or None, '\n 'not {}'.format(type(batch_results)))\n\n # Accumulate training results\n if batch_results is not None:\n results.append(batch_results)\n\n n_processed += 1\n if n_batches is not None and n_processed >= n_batches:\n break\n\n # Concatenate result arrays\n if len(results) > 0:\n results = zip(*results)\n results = tuple([np.concatenate(list(r), axis=0) for r in results])\n return results\n else:\n return None", "response": "This function applies a function to all the samples that are accessed as mini - batches and returns the sum of the results for each mini - batch."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef batch_map_mean(func, batch_iter, progress_iter_func=None, sum_axis=None,\n n_batches=None, prepend_args=None):\n \"\"\"\n Apply a function to all the samples that are accessed as mini-batches\n obtained from an iterator.\n Returns the across-samples mean of the results returned by `func`\n\n The `sum_axis` arguments tells `mean_batch_map` how to process the\n results of `func` before accumulating them:\n - If `sum_axis` is `None`, `func` should return the\n across-samples SUM of the results of operating on the mini-batch the\n sum of the values for the samples, e.g. for loss and error it should\n return `(sum([loss0, loss1, ... lossN]), sum([err0, err1, ... errN]))`\n - Otherwise, `sum_axis` should specify the axis or axes over which\n the the batch results should be summed, e.g. if `func` returns a\n per-sample loss and error in two arrays\n `[[loss0, loss1, ... lossN], [err0, err1, ... errN]`, give `sum_axis`\n a value of `0` to sum over axis 0 to get the per-batch loss and error.\n These results will be accumulated and divided by the number of samples\n at the end to get the mean.\n\n Parameters\n ----------\n func: callable `func(*batch) -> results`\n The function to call on each mini-batch. Note that the results\n must be `None`, a tuple or a NumPy array\n batch_iter: data set iterator\n Iterator that generates mini-batches of data\n progress_iter_func: [optional] callable\n `progress_iter_func(iterator, total=total, leave=leave)`\n A `tqdm` style function that will be passed the iterator that\n generates training batches along with the total number of batches\n and `False` for the `leave` parameter. By passing either\n `tqdm.tqdm` or `tqdm.tqdm_notebook` as this argument you can have\n the training loop display a progress bar.\n sum_axis: (default=`None`) int, tuple of ints or None\n If an integer or a tuple of integers, the results returned by `func`\n will be summed across this axis / these axes before being accumulated;\n e.g. if `func` returns an array of per-sample losses, with axis 0\n being the sample dimension, passing a value of `0` as `sum_axis`\n will cause these results to be summed along axis 0 to get the\n per-batch sum before accumulating the losses. The total summed loss\n will be divided by the number of samples at the end in order to\n compute the mean loss.\n n_batches: [optional] integer that specifies the number of mini-batches\n to process before returning\n prepend_args: [optional] tuple\n Arguments to prepend to the arguments passed to `func`\n\n Returns\n -------\n tuple\n The sum of the results of the function `fn` divided by the number of\n samples processed, e.g.\n `(sum(outA_per_batch) / n_samples,\n sum(outB_per_batch) / n_samples,\n ...)`\n\n Examples\n --------\n The following examples will demonstrate the use of `mean_batch_map`\n to compute binary cross entropy loss over a data set.\n A few variants will be demonstrated:\n - the default behaviour in which the function being applied should\n return the sum over the batch sample axis\n - having the function return per sample results and maving\n `mean_batch_map` perform the sum operation. This is easier to\n understand but less efficient as a Theano function would have to\n move more data back from the GPU.\n - limiting the number of batches that will be processed in order to get\n partial results when dealing with a large data set\n\n Define a function to compute the per-sample binary cross entropy\n loss:\n >>> def binary_crossentropy_loss(pred, target):\n ... e = -target * np.log(pred) - (1 - target) * np.log(1 - pred)\n ... return e.mean(axis=1)\n\n Now define a function that computes the *SUM* of the binary cross\n entropy losses over the sample axis (axis 0), as the default\n behaviour of `mean_batch_map` will sum them up and divide by the\n number of samples at the end:\n >>> def binary_crossentropy_loss_sum(pred, target):\n ... return binary_crossentropy_loss(pred, target).sum()\n\n Construct prediction and target data\n >>> pred = np.random.uniform(0.1, 0.9, size=(7, 10))\n >>> tgt = np.random.uniform(0.1, 0.9, size=(7, 10))\n >>> ds = ArrayDataSource([pred, tgt])\n\n Apply the loss sum function defined above:\n >>> batch_iter = ds.batch_iterator(batch_size=5)\n >>> loss = batch_map_mean(binary_crossentropy_loss_sum, batch_iter)\n >>> assert np.allclose(\n ... loss, binary_crossentropy_loss(pred, tgt).mean())\n\n Have `mean_batch_map` sum over axis 0:\n >>> batch_iter = ds.batch_iterator(batch_size=5)\n >>> loss = batch_map_mean(binary_crossentropy_loss, batch_iter,\n ... sum_axis=0)\n >>> assert np.allclose(\n ... loss, binary_crossentropy_loss(pred, tgt).mean())\n\n Construct a large data set and use `batch\n >>> pred_large = np.random.uniform(0.1, 0.9, size=(100, 10))\n >>> tgt_large = np.random.uniform(0.1, 0.9, size=(100, 10))\n >>> ds_large = ArrayDataSource([pred_large, tgt_large])\n >>> iter_large = ds_large.batch_iterator(batch_size=5)\n >>> for i in range(10):\n ... partial_loss = batch_map_mean(binary_crossentropy_loss_sum,\n ... iter_large, n_batches=2)\n ... j = i * 10\n ... assert np.allclose(\n ... partial_loss, binary_crossentropy_loss(\n ... pred_large[j:j + 10], tgt_large[j:j + 10]).mean())\n \"\"\"\n # Accumulator for results and number of samples\n results_accum = None\n n_samples_accum = 0\n\n # If `progress_iter_func` is not `None`, apply it\n if progress_iter_func is not None:\n batch_iter = progress_iter_func(batch_iter, total=n_batches,\n leave=False)\n\n # Train on each batch\n n_processed = 0\n for batch in batch_iter:\n # Get number of samples in batch; can vary\n batch_n = _length_of_batch(batch)\n\n # Apply on batch and check the type of the results\n if prepend_args is not None:\n batch_results = func(*(prepend_args + tuple(batch)))\n else:\n batch_results = func(*batch)\n if batch_results is None:\n pass\n elif isinstance(batch_results, (np.ndarray, float)):\n batch_results = (batch_results,)\n elif isinstance(batch_results, tuple):\n pass\n else:\n raise TypeError(\n 'Batch function should return a tuple of results, a '\n 'single result as a NumPy array or float, or None, '\n 'not {}'.format(type(batch_results)))\n\n # Accumulate results and number of samples\n if results_accum is None:\n # Initialise the accumulator to the batch results if `func`\n # returns summed results or if it returned None;\n # don't attempt to iterate over None and sum each item\n if batch_results is None:\n pass\n elif sum_axis is None:\n results_accum = list(batch_results)\n else:\n results_accum = [br.sum(axis=sum_axis) for br in batch_results]\n else:\n if batch_results is not None:\n for i in range(len(results_accum)):\n br = batch_results[i]\n if sum_axis is not None:\n br = br.sum(axis=sum_axis)\n results_accum[i] += br\n n_samples_accum += batch_n\n\n n_processed += 1\n if n_batches is not None and n_processed >= n_batches:\n break\n\n # Divide by the number of training examples used to compute mean\n if results_accum is not None:\n results_accum = tuple([np.array(r).astype(float) / n_samples_accum\n for r in results_accum])\n\n return results_accum", "response": "Returns the mean of the results of a function across all the mini - batches of a single tree node."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncoerces an object into a data source.", "response": "def coerce_data_source(x):\n \"\"\"\n Helper function to coerce an object into a data source, selecting the\n appropriate data source class for the given object. If `x` is already\n a data source it is returned as is.\n\n Parameters\n ----------\n x: any\n The object to coerce. If `x` is a data source, it is returned as is.\n If it is a list or tuple of array-like objects they will be wrapped\n in an `ArrayDataSource` that will be returned. If `x` is an iterator\n it will be wrapped in an `IteratorDataSource`. If it is a callable\n it will be wrapped in a `CallableDataSource`.\n\n Returns\n -------\n `x` coerced into a data source\n\n Raises\n ------\n `TypeError` if `x` is not a data souce, a list or tuple of array-like\n objects, an iterator or a callable.\n \"\"\"\n if isinstance(x, AbstractDataSource):\n return x\n elif isinstance(x, (list, tuple)):\n # Sequence of array-likes\n items = []\n for item in x:\n if _is_array_like(item):\n items.append(item)\n else:\n raise TypeError(\n 'Cannot convert x to a data source; x is a sequence and '\n 'one of the elements is not an array-like object, rather '\n 'a {}'.format(type(item)))\n if len(items) == 0:\n raise ValueError('Cannot convert x to a data source; x is an '\n 'empty sequence')\n return ArrayDataSource(items)\n elif isinstance(x, collections.Iterator):\n return IteratorDataSource(x)\n elif callable(x):\n return CallableDataSource(x)\n else:\n raise TypeError('Cannot convert x to a data source; can only handle '\n 'iterators, callables, non-empty sequences of '\n 'array-like objects; cannot '\n 'handle {}'.format(type(x)))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngather a batch of samples by indices applying the mapping to the indices passed to the sample_by_indices_nomapping method.", "response": "def samples_by_indices(self, indices):\n \"\"\"\n Gather a batch of samples by indices, applying the mapping\n described by the (optional) `indices` array passed to the\n constructor.\n\n Parameters\n ----------\n indices: 1D-array of ints or slice\n The samples to retrieve\n\n Returns\n -------\n list of arrays\n A mini-batch in the form of a list of NumPy arrays\n \"\"\"\n indices = self.sampler.map_indices(indices)\n return self.samples_by_indices_nomapping(indices)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef batch_indices_iterator(self, batch_size, shuffle=None, **kwargs):\n shuffle_rng = self._get_shuffle_rng(shuffle)\n if shuffle_rng is not None:\n return self.sampler.shuffled_indices_batch_iterator(\n batch_size, shuffle_rng)\n else:\n return self.sampler.in_order_indices_batch_iterator(batch_size)", "response": "Create an iterator that generates mini - batches of sample indices."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef batch_iterator(self, batch_size, shuffle=None, **kwargs):\n for batch_ndx in self.batch_indices_iterator(\n batch_size, shuffle=shuffle, **kwargs):\n yield self.samples_by_indices_nomapping(batch_ndx)", "response": "Create an iterator that generates mini - batches extracted from the data source."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef samples_by_indices_nomapping(self, indices):\n batch = tuple([d[indices] for d in self.data])\n if self.include_indices:\n if isinstance(indices, slice):\n indices = np.arange(indices.start, indices.stop,\n indices.step)\n return (indices,) + batch\n else:\n return batch", "response": "Gather a batch of samples by indices without applying any index\n mapping."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef num_samples(self, **kwargs):\n if self.num_samples_fn is None:\n return None\n elif callable(self.num_samples_fn):\n return self.num_samples_fn(**kwargs)\n else:\n return self.num_samples_fn", "response": "Returns the number of samples in this data source."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef samples_by_indices_nomapping(self, indices):\n if not self._random_access:\n raise TypeError('samples_by_indices_nomapping method not '\n 'supported as one or more of the underlying '\n 'data sources does not support random access')\n if len(indices) != len(self.datasets):\n raise ValueError(\n 'length mis-match: indices has {} items, self has {} data '\n 'sources, should be equal'.format(len(indices),\n len(self.datasets)))\n batch = tuple([ds.samples_by_indices_nomapping(ndx)\n for ds, ndx in zip(self.datasets, indices)])\n return self._prepare_batch(batch)", "response": "Gather a batch of samples by indices without applying any index\n mapping."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate an iterator that generates mini - batches indices for each entry in the dataset.", "response": "def batch_indices_iterator(self, batch_size, **kwargs):\n \"\"\"\n Create an iterator that generates mini-batch sample indices\n\n The generated mini-batches indices take the form of nested lists of\n either:\n - 1D NumPy integer arrays\n - slices\n\n The list nesting structure with match that of the tree of data sources\n rooted at `self`\n\n Parameters\n ----------\n batch_size: int\n Mini-batch size\n\n Returns\n -------\n iterator\n An iterator that generates items that are nested lists of slices\n or 1D NumPy integer arrays.\n \"\"\"\n if not self._random_access:\n raise TypeError('batch_indices_iterator method not supported as '\n 'one or more of the underlying data sources '\n 'does not support random access')\n iterators = [d.batch_indices_iterator(batch_size, **kwargs)\n for d in self.datasets]\n\n for batch in six.moves.zip(*iterators):\n yield self._prepare_index_batch(batch)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngathers a batch of samples by indices without applying any index mapping.", "response": "def samples_by_indices_nomapping(self, indices):\n \"\"\"\n Gather a batch of samples by indices *without* applying any index\n mapping.\n\n Parameters\n ----------\n indices: a tuple of the form `(dataset_index, sample_indices)`\n The `dataset_index` identifies the dataset from which to draw\n samples while `sample_indices` identifies the samples to draw\n from it.\n\n Returns\n -------\n nested list of arrays\n A mini-batch\n \"\"\"\n if not self._random_access:\n raise TypeError('samples_by_indices_nomapping method not '\n 'supported as one or more of the underlying '\n 'data sources does not support random access')\n if not isinstance(indices, tuple):\n raise TypeError('indices should be a tuple, not a {}'.format(\n type(indices)\n ))\n dataset_index, sample_indices = indices\n ds = self.datasets[dataset_index]\n return ds.samples_by_indices_nomapping(sample_indices)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate an iterator that generates mini - batches indices for each entry in the tree of data sources.", "response": "def batch_indices_iterator(self, batch_size, shuffle=None, **kwargs):\n \"\"\"\n Create an iterator that generates mini-batch sample indices\n\n The generated mini-batches indices take the form of nested lists of\n either:\n - 1D NumPy integer arrays\n - slices\n\n The list nesting structure with match that of the tree of data sources\n rooted at `self`\n\n Parameters\n ----------\n batch_size: int\n Mini-batch size\n shuffle: `numpy.random.RandomState` or `True` or `None`\n Used to randomise element order. If `None`, elements will be\n extracted in order. If it is a `RandomState` instance, that\n RNG will be used to shuffle elements. If it is `True`, NumPy's\n default RNG will be used.\n\n Returns\n -------\n iterator\n An iterator that generates items that are nested lists of slices\n or 1D NumPy integer arrays.\n \"\"\"\n if not self._random_access:\n raise TypeError('batch_indices_iterator method not supported as '\n 'one or more of the underlying data sources '\n 'does not support random access')\n shuffle_rng = self._get_shuffle_rng(shuffle)\n iterators = [d.batch_indices_iterator(batch_size,\n shuffle=shuffle_rng, **kwargs)\n for d in self.datasets]\n return self._ds_iterator(batch_size, iterators, shuffle_rng, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ngather a batch of samples by indices without applying any index mapping.", "response": "def samples_by_indices_nomapping(self, indices):\n \"\"\"\n Gather a batch of samples by indices *without* applying any index\n mapping.\n\n Parameters\n ----------\n indices: 1D-array of ints or slice\n An index array or a slice that selects the samples to retrieve\n\n Returns\n -------\n nested list of arrays\n A mini-batch\n \"\"\"\n if not self._random_access:\n raise TypeError('samples_by_indices_nomapping method not '\n 'supported as one or more of the underlying '\n 'data sources does not support random access')\n batch = self.source.samples_by_indices_nomapping(indices)\n return self.fn(*batch)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef samples_by_indices(self, indices):\n if not self._random_access:\n raise TypeError('samples_by_indices method not supported as one '\n 'or more of the underlying data sources does '\n 'not support random access')\n batch = self.source.samples_by_indices(indices)\n return self.fn(*batch)", "response": "Gather a batch of samples by indices applying any index\n mapping defined by the underlying data sources."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef batch_indices_iterator(self, batch_size, **kwargs):\n if not self._random_access:\n raise TypeError('batch_indices_iterator method not supported as '\n 'one or more of the underlying data sources '\n 'does not support random access')\n return self.source.batch_indices_iterator(batch_size, **kwargs)", "response": "Create an iterator that generates mini - batches indices for each entry in the source tree."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\npurge environment database and uploaded files", "response": "def purge(opts):\n \"\"\"Purge environment database and uploaded files\n\nUsage:\n datacats purge [-s NAME | --delete-environment] [-y] [ENVIRONMENT]\n\nOptions:\n --delete-environment Delete environment directory as well as its data, as\n well as the data for **all** sites.\n -s --site=NAME Specify a site to be purge [default: primary]\n -y --yes Respond yes to all prompts (i.e. force)\n\nENVIRONMENT may be an environment name or a path to an environment directory.\nDefault: '.'\n\"\"\"\n old = False\n try:\n environment = Environment.load(opts['ENVIRONMENT'], opts['--site'])\n except DatacatsError:\n environment = Environment.load(opts['ENVIRONMENT'], opts['--site'], data_only=True)\n if get_format_version(environment.datadir) == 1:\n old = True\n environment = Environment.load(opts['ENVIRONMENT'], opts['--site'], allow_old=True)\n\n # We need a valid site if they don't want to blow away everything.\n if not opts['--delete-environment'] and not old:\n environment.require_valid_site()\n\n sites = [opts['--site']] if not opts['--delete-environment'] else environment.sites\n\n if not opts['--yes']:\n y_or_n_prompt('datacats purge will delete all stored data')\n\n environment.stop_ckan()\n environment.stop_supporting_containers()\n\n environment.purge_data(sites)\n\n if opts['--delete-environment']:\n if environment.target:\n rmtree(environment.target)\n else:\n DatacatsError((\"Unable to find the environment source\"\n \" directory so that it can be deleted.\\n\"\n \"Chances are it's because it already does not exist\"))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pretty_print(self):\n print colored.blue(\"-\" * 40)\n print colored.red(\"datacats: problem was encountered:\")\n print self.message\n print colored.blue(\"-\" * 40)", "response": "Print the error message to stdout with colors and borders"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef generate_password():\n chars = uppercase + lowercase + digits\n return ''.join(SystemRandom().choice(chars) for x in xrange(16))", "response": "Generate a 16 - character alphanumeric random string generated by the\n operating system s secure pseudo random number generator."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks connectivity of a node.", "response": "def _machine_check_connectivity():\n \"\"\"\n This method calls to docker-machine on the command line and\n makes sure that it is up and ready.\n\n Potential improvements to be made:\n - Support multiple machine names (run a `docker-machine ls` and then\n see which machines are active. Use a priority list)\n \"\"\"\n with open(devnull, 'w') as devnull_f:\n try:\n status = subprocess.check_output(\n ['docker-machine', 'status', 'dev'],\n stderr=devnull_f).strip()\n if status == 'Stopped':\n raise DatacatsError('Please start your docker-machine '\n 'VM with \"docker-machine start dev\"')\n\n # XXX HACK: This exists because of\n # http://github.com/datacats/datacats/issues/63,\n # as a temporary fix.\n if 'tls' in _docker_kwargs:\n # It will print out messages to the user otherwise.\n _docker_kwargs['tls'].assert_hostname = False\n except subprocess.CalledProcessError:\n raise DatacatsError('Please create a docker-machine with '\n '\"docker-machine start dev\"')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert ro and rw dicts to docker - py s", "response": "def ro_rw_to_binds(ro, rw):\n \"\"\"\n ro and rw {localdir: binddir} dicts to docker-py's\n {localdir: {'bind': binddir, 'ro': T/F}} binds dicts\n \"\"\"\n out = {}\n if ro:\n for localdir, binddir in ro.iteritems():\n out[localdir] = {'bind': binddir, 'ro': True}\n if rw:\n for localdir, binddir in rw.iteritems():\n out[localdir] = {'bind': binddir, 'ro': False}\n return out"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef web_command(command, ro=None, rw=None, links=None,\n image='datacats/web', volumes_from=None, commit=False,\n clean_up=False, stream_output=None, entrypoint=None):\n \"\"\"\n Run a single command in a web image optionally preloaded with the ckan\n source and virtual envrionment.\n\n :param command: command to execute\n :param ro: {localdir: binddir} dict for read-only volumes\n :param rw: {localdir: binddir} dict for read-write volumes\n :param links: links passed to start\n :param image: docker image name to use\n :param volumes_from:\n :param commit: True to create a new image based on result\n :param clean_up: True to remove container even on error\n :param stream_output: file to write stderr+stdout from command\n :param entrypoint: override entrypoint (script that runs command)\n\n :returns: image id if commit=True\n \"\"\"\n binds = ro_rw_to_binds(ro, rw)\n c = _get_docker().create_container(\n image=image,\n command=command,\n volumes=binds_to_volumes(binds),\n detach=False,\n host_config=_get_docker().create_host_config(binds=binds, volumes_from=volumes_from, links=links),\n entrypoint=entrypoint)\n _get_docker().start(\n container=c['Id'],\n )\n if stream_output:\n for output in _get_docker().attach(\n c['Id'], stdout=True, stderr=True, stream=True):\n stream_output.write(output)\n if _get_docker().wait(c['Id']):\n # Before the (potential) cleanup, grab the logs!\n logs = _get_docker().logs(c['Id'])\n\n if clean_up:\n remove_container(c['Id'])\n raise WebCommandError(command, c['Id'][:12], logs)\n if commit:\n rval = _get_docker().commit(c['Id'])\n if not remove_container(c['Id']):\n # circle ci doesn't let us remove containers, quiet the warnings\n if not environ.get('CIRCLECI', False):\n warn('failed to remove container: {0}'.format(c['Id']))\n if commit:\n return rval['Id']", "response": "Execute a command in a web image optionally preloaded with the ckan\n source and virtual envrionment."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef remote_server_command(command, environment, user_profile, **kwargs):\n\n if environment.remote_server_key:\n temp = tempfile.NamedTemporaryFile(mode=\"wb\")\n temp.write(environment.remote_server_key)\n temp.seek(0)\n known_hosts = temp.name\n else:\n known_hosts = get_script_path('known_hosts')\n\n binds = {\n user_profile.profiledir + '/id_rsa': '/root/.ssh/id_rsa',\n known_hosts: '/root/.ssh/known_hosts',\n get_script_path('ssh_config'): '/etc/ssh/ssh_config'\n }\n\n if kwargs.get(\"include_project_dir\", None):\n binds[environment.target] = '/project'\n del kwargs[\"include_project_dir\"]\n\n kwargs[\"ro\"] = binds\n try:\n web_command(command, **kwargs)\n except WebCommandError as e:\n e.user_description = 'Sending a command to remote server failed'\n raise e", "response": "This function wraps web_command function to create a new container that can be used to connect to a remote server and run commands on that server."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwraps for docker create_container start calls", "response": "def run_container(name, image, command=None, environment=None,\n ro=None, rw=None, links=None, detach=True, volumes_from=None,\n port_bindings=None, log_syslog=False):\n \"\"\"\n Wrapper for docker create_container, start calls\n\n :param log_syslog: bool flag to redirect container's logs to host's syslog\n\n :returns: container info dict or None if container couldn't be created\n\n Raises PortAllocatedError if container couldn't start on the\n requested port.\n \"\"\"\n binds = ro_rw_to_binds(ro, rw)\n log_config = LogConfig(type=LogConfig.types.JSON)\n if log_syslog:\n log_config = LogConfig(\n type=LogConfig.types.SYSLOG,\n config={'syslog-tag': name})\n host_config = _get_docker().create_host_config(binds=binds, log_config=log_config, links=links, volumes_from=volumes_from, port_bindings=port_bindings)\n\n c = _get_docker().create_container(\n name=name,\n image=image,\n command=command,\n environment=environment,\n volumes=binds_to_volumes(binds),\n detach=detach,\n stdin_open=False,\n tty=False,\n ports=list(port_bindings) if port_bindings else None,\n host_config=host_config)\n try:\n _get_docker().start(\n container=c['Id'],\n )\n except APIError as e:\n if 'address already in use' in e.explanation:\n try:\n _get_docker().remove_container(name, force=True)\n except APIError:\n pass\n raise PortAllocatedError()\n raise\n return c"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwrapping for docker remove_container", "response": "def remove_container(name, force=False):\n \"\"\"\n Wrapper for docker remove_container\n\n :returns: True if container was found and removed\n \"\"\"\n\n try:\n if not force:\n _get_docker().stop(name)\n except APIError:\n pass\n try:\n _get_docker().remove_container(name, force=True)\n return True\n except APIError:\n return False"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef container_logs(name, tail, follow, timestamps):\n if follow:\n return _get_docker().attach(\n name,\n stdout=True,\n stderr=True,\n stream=True\n )\n\n return _docker.logs(\n name,\n stdout=True,\n stderr=True,\n tail=tail,\n timestamps=timestamps,\n )", "response": "Wrapper for docker logs attach commands."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a string representation of the logs from a container.", "response": "def collect_logs(name):\n \"\"\"\n Returns a string representation of the logs from a container.\n This is similar to container_logs but uses the `follow` option\n and flattens the logs into a string instead of a generator.\n\n :param name: The container name to grab logs for\n :return: A string representation of the logs\n \"\"\"\n logs = container_logs(name, \"all\", True, None)\n string = \"\"\n for s in logs:\n string += s\n return string"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns generator of pull status objects for the given image", "response": "def pull_stream(image):\n \"\"\"\n Return generator of pull status objects\n \"\"\"\n return (json.loads(s) for s in _get_docker().pull(image, stream=True))"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a data - only container.", "response": "def data_only_container(name, volumes):\n \"\"\"\n create \"data-only container\" if it doesn't already exist.\n\n We'd like to avoid these, but postgres + boot2docker make\n it difficult, see issue #5\n \"\"\"\n info = inspect_container(name)\n if info:\n return\n c = _get_docker().create_container(\n name=name,\n image='datacats/postgres', # any image will do\n command='true',\n volumes=volumes,\n detach=True)\n return c"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _subcommand_arguments(args):\n skip_site = False\n # Find subcommand without docopt so that subcommand options may appear\n # anywhere\n for i, a in enumerate(args):\n if skip_site:\n skip_site = False\n continue\n if a.startswith('-'):\n if a == '-s' or a == '--site':\n skip_site = True\n continue\n if a == 'help':\n return _subcommand_arguments(args[:i] + ['--help'] + args[i + 1:])\n if a not in COMMANDS:\n raise DatacatsError(\"\\'{0}\\' command is not recognized. \\n\"\n \"See \\'datacats help\\' for the list of available commands\".format(a))\n command = a\n break\n else:\n return None, args\n\n if command != 'shell' and command != 'paster':\n return command, args\n\n # shell requires the environment name, paster does not\n remaining_positional = 2 if command == 'shell' else 1\n\n # i is where the subcommand starts.\n # shell, paster are special: options might belong to the command being\n # find where the the inner command starts and insert a '--' before\n # so that we can separate inner options from ones we need to parse\n while i < len(args):\n a = args[i]\n if a.startswith('-'):\n if a == '-s' or a == '--site':\n # site name is coming\n i += 2\n continue\n i += 1\n continue\n if remaining_positional:\n remaining_positional -= 1\n i += 1\n continue\n return command, args[:i] + ['--'] + args[i:]\n\n return command, args", "response": "Return a tuple of subcommand and positional arguments for that subcommand"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef start(environment, opts):\n environment.require_data()\n\n if environment.fully_running():\n print 'Already running at {0}'.format(environment.web_address())\n return\n\n reload_(environment, opts)", "response": "Create containers and start serving"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreloads the environment source and configuration from apache and watch.", "response": "def reload_(environment, opts):\n \"\"\"Reload environment source and configuration\n\nUsage:\n datacats reload [-b] [-p|--no-watch] [--syslog] [-s NAME] [--site-url=SITE_URL]\n [-i] [--address=IP] [ENVIRONMENT [PORT]]\n datacats reload -r [-b] [--syslog] [-s NAME] [--address=IP] [--site-url=SITE_URL]\n [-i] [ENVIRONMENT]\n\nOptions:\n --address=IP Address to listen on (Linux-only)\n -i --interactive Calls out to docker via the command line, allowing\n for interactivity with the web image.\n --site-url=SITE_URL The site_url to use in API responses. Can use Python template syntax\n to insert the port and address (e.g. http://example.org:{port}/)\n -b --background Don't wait for response from web server\n --no-watch Do not automatically reload templates and .py files on change\n -p --production Reload with apache and debug=false\n -s --site=NAME Specify a site to reload [default: primary]\n --syslog Log to the syslog\n\nENVIRONMENT may be an environment name or a path to an environment directory.\nDefault: '.'\n\"\"\"\n if opts['--interactive']:\n # We can't wait for the server if we're tty'd\n opts['--background'] = True\n if opts['--address'] and is_boot2docker():\n raise DatacatsError('Cannot specify address on boot2docker.')\n environment.require_data()\n environment.stop_ckan()\n if opts['PORT'] or opts['--address'] or opts['--site-url']:\n if opts['PORT']:\n environment.port = int(opts['PORT'])\n if opts['--address']:\n environment.address = opts['--address']\n if opts['--site-url']:\n site_url = opts['--site-url']\n # TODO: Check it against a regex or use urlparse\n try:\n site_url = site_url.format(address=environment.address, port=environment.port)\n environment.site_url = site_url\n environment.save_site(False)\n except (KeyError, IndexError, ValueError) as e:\n raise DatacatsError('Could not parse site_url: {}'.format(e))\n environment.save()\n\n for container in environment.extra_containers:\n require_extra_image(EXTRA_IMAGE_MAPPING[container])\n\n environment.stop_supporting_containers()\n environment.start_supporting_containers()\n\n environment.start_ckan(\n production=opts['--production'],\n paster_reload=not opts['--no-watch'],\n log_syslog=opts['--syslog'],\n interactive=opts['--interactive'])\n write('Starting web server at {0} ...'.format(environment.web_address()))\n if opts['--background']:\n write('\\n')\n return\n\n try:\n environment.wait_for_web_available()\n finally:\n write('\\n')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndisplaying information about the environment and running containers.", "response": "def info(environment, opts):\n \"\"\"Display information about environment and running containers\n\nUsage:\n datacats info [-qr] [ENVIRONMENT]\n\nOptions:\n -q --quiet Echo only the web URL or nothing if not running\n\nENVIRONMENT may be an environment name or a path to an environment directory.\nDefault: '.'\n\"\"\"\n damaged = False\n sites = environment.sites\n if not environment.sites:\n sites = []\n damaged = True\n\n if opts['--quiet']:\n if damaged:\n raise DatacatsError('Damaged datadir: cannot get address.')\n for site in sites:\n environment.site_name = site\n print '{}: {}'.format(site, environment.web_address())\n return\n\n datadir = environment.datadir\n if not environment.data_exists():\n datadir = ''\n elif damaged:\n datadir += ' (damaged)'\n\n print 'Environment name: ' + environment.name\n print ' Environment dir: ' + environment.target\n print ' Data dir: ' + datadir\n print ' Sites: ' + ' '.join(environment.sites)\n\n for site in environment.sites:\n print\n environment.site_name = site\n print ' Site: ' + site\n print ' Containers: ' + ' '.join(environment.containers_running())\n\n sitedir = environment.sitedir + (' (damaged)' if not environment.data_complete() else '')\n print ' Site dir: ' + sitedir\n addr = environment.web_address()\n if addr:\n print ' Available at: ' + addr"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndisplays or follow container logs", "response": "def logs(environment, opts):\n \"\"\"Display or follow container logs\n\nUsage:\n datacats logs [--postgres | --solr | --datapusher] [-s NAME] [-tr] [--tail=LINES] [ENVIRONMENT]\n datacats logs -f [--postgres | --solr | --datapusher] [-s NAME] [-r] [ENVIRONMENT]\n\nOptions:\n --datapusher Show logs for datapusher instead of web logs\n --postgres Show postgres database logs instead of web logs\n -f --follow Follow logs instead of exiting immediately\n --solr Show solr search logs instead of web logs\n -t --timestamps Add timestamps to log lines\n -s --site=NAME Specify a site for logs if needed [default: primary]\n --tail=LINES Number of lines to show [default: all]\n\nENVIRONMENT may be an environment name or a path to an environment directory.\nDefault: '.'\n\"\"\"\n container = 'web'\n if opts['--solr']:\n container = 'solr'\n if opts['--postgres']:\n container = 'postgres'\n if opts['--datapusher']:\n container = 'datapusher'\n tail = opts['--tail']\n if tail != 'all':\n tail = int(tail)\n l = environment.logs(container, tail, opts['--follow'],\n opts['--timestamps'])\n if not opts['--follow']:\n print l\n return\n try:\n for message in l:\n write(message)\n except KeyboardInterrupt:\n print"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nopen a web browser window to this environment", "response": "def open_(environment, opts):\n # pylint: disable=unused-argument\n \"\"\"Open web browser window to this environment\n\nUsage:\n datacats open [-r] [-s NAME] [ENVIRONMENT]\n\nOptions:\n -s --site=NAME Choose a site to open [default: primary]\n\nENVIRONMENT may be an environment name or a path to an environment directory.\nDefault: '.'\n\"\"\"\n\n environment.require_data()\n addr = environment.web_address()\n if not addr:\n print \"Site not currently running\"\n else:\n webbrowser.open(addr)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncommand operating on environment datacats tweak", "response": "def tweak(environment, opts):\n \"\"\"Commands operating on environment data\n\nUsage:\n datacats tweak --install-postgis [ENVIRONMENT]\n datacats tweak --add-redis [ENVIRONMENT]\n datacats tweak --admin-password [ENVIRONMENT]\n\nOptions:\n --install-postgis Install postgis in ckan database\n --add-redis Adds redis next time this environment reloads\n -s --site=NAME Choose a site to tweak [default: primary]\n -p --admin-password Prompt to change the admin password\n\nENVIRONMENT may be an environment name or a path to an environment directory.\n\nDefault: '.'\n\"\"\"\n\n environment.require_data()\n if opts['--install-postgis']:\n print \"Installing postgis\"\n environment.install_postgis_sql()\n if opts['--add-redis']:\n # Let the user know if they are trying to add it and it is already there\n print ('Adding redis extra container... Please note that you will have '\n 'to reload your environment for these changes to take effect (\"datacats reload {}\")'\n .format(environment.name))\n environment.add_extra_container('redis', error_on_exists=True)\n if opts['--admin-password']:\n environment.create_admin_set_password(confirm_password())"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef convert_environment(datadir, version, always_yes):\n # Since we don't call either load() or new() we have to call require_images ourselves.\n require_images()\n\n inp = None\n old_version = _get_current_format(datadir)\n migration_func = migrations[(old_version, version)]\n\n if version > CURRENT_FORMAT_VERSION:\n raise DatacatsError('Cannot migrate to a version higher than the '\n 'current one.')\n if version < 1:\n raise DatacatsError('Datadir versioning starts at 1.')\n\n if not always_yes:\n while inp != 'y' and inp != 'n':\n inp = raw_input(migration_func.__doc__.format(version))\n\n if inp == 'n':\n sys.exit(1)\n\n lockfile = LockFile(path_join(datadir, '.migration_lock'))\n lockfile.acquire()\n\n try:\n # FIXME: If we wanted to, we could find a set of conversions which\n # would bring us up to the one we want if there's no direct path.\n # This isn't necessary with just two formats, but it may be useful\n # at 3.\n # Call the appropriate conversion function\n migration_func(datadir)\n finally:\n lockfile.release()", "response": "Convert an environment to a specific version."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfetching the history of a flight by its number. This method can be used to get the history of a flight route by the number. It checks the user authentication and returns the data accordingly. Args: flight_number (str): The flight number, e.g. AI101 page (int): Optional page number; for users who are on a plan with flightradar24 they can pass in higher page numbers to get more data limit (int): Optional limit on number of records returned Returns: A list of dicts with the data; one dict for each row of data from flightradar24 Example:: from pyflightdata import FlightData f=FlightData() #optional login f.login(myemail,mypassword) f.get_history_by_flight_number('AI101') f.get_history_by_flight_number('AI101',page=1,limit=10)", "response": "def get_history_by_flight_number(self, flight_number, page=1, limit=100):\n \"\"\"Fetch the history of a flight by its number.\n\n This method can be used to get the history of a flight route by the number.\n It checks the user authentication and returns the data accordingly.\n\n Args:\n flight_number (str): The flight number, e.g. AI101\n page (int): Optional page number; for users who are on a plan with flightradar24 they can pass in higher page numbers to get more data\n limit (int): Optional limit on number of records returned\n\n Returns:\n A list of dicts with the data; one dict for each row of data from flightradar24\n\n Example::\n\n from pyflightdata import FlightData\n f=FlightData()\n #optional login\n f.login(myemail,mypassword)\n f.get_history_by_flight_number('AI101')\n f.get_history_by_flight_number('AI101',page=1,limit=10)\n\n \"\"\"\n url = FLT_BASE.format(flight_number, str(self.AUTH_TOKEN), page, limit)\n return self._fr24.get_data(url)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_history_by_tail_number(self, tail_number, page=1, limit=100):\n url = REG_BASE.format(tail_number, str(self.AUTH_TOKEN), page, limit)\n return self._fr24.get_data(url, True)", "response": "This method is used to get the history of a particular aircraft by its tail number. It will login to flightradar24 and then get the data for that aircraft."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_airports(self, country):\n url = AIRPORT_BASE.format(country.replace(\" \", \"-\"))\n return self._fr24.get_airports_data(url)", "response": "Returns a list of all the airports for a given country"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_info_by_tail_number(self, tail_number, page=1, limit=100):\n url = REG_BASE.format(tail_number, str(self.AUTH_TOKEN), page, limit)\n return self._fr24.get_aircraft_data(url)", "response": "This method is used to get the details of a particular aircraft by its tail number. It will login to flightradar24 and then get the details of a particular aircraft by its tail number."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the fleet for a particular airline.", "response": "def get_fleet(self, airline_key):\n \"\"\"Get the fleet for a particular airline.\n\n Given a airline code form the get_airlines() method output, this method returns the fleet for the airline.\n\n Args:\n airline_key (str): The code for the airline on flightradar24\n\n Returns:\n A list of dicts, one for each aircraft in the airlines fleet\n\n Example::\n from pyflightdata import FlightData\n f=FlightData()\n #optional login\n f.login(myemail,mypassword)\n f.get_fleet('ai-aic')\n \"\"\"\n url = AIRLINE_FLEET_BASE.format(airline_key)\n return self._fr24.get_airline_fleet_data(url, self.AUTH_TOKEN != '')"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the flights for a particular airline.", "response": "def get_flights(self, search_key):\n \"\"\"Get the flights for a particular airline.\n\n Given a full or partial flight number string, this method returns the first 100 flights matching that string.\n\n Please note this method was different in earlier versions. The older versions took an airline code and returned all scheduled flights for that airline\n\n Args:\n search_key (str): Full or partial flight number for any airline e.g. MI47 to get all SilkAir flights starting with MI47\n\n Returns:\n A list of dicts, one for each scheduled flight in the airlines network\n\n Example::\n from pyflightdata import FlightData\n f=FlightData()\n #optional login\n f.login(myemail,mypassword)\n f.get_flights('MI47')\n \"\"\"\n # assume limit 100 to return first 100 of any wild card search\n url = AIRLINE_FLT_BASE.format(search_key, 100)\n return self._fr24.get_airline_flight_data(url)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget the flights for a particular origin and destination.", "response": "def get_flights_from_to(self, origin, destination):\n \"\"\"Get the flights for a particular origin and destination.\n\n Given an origin and destination this method returns the upcoming scheduled flights between these two points.\n The data returned has the airline, airport and schedule information - this is subject to change in future.\n\n Args:\n origin (str): The origin airport code\n destination (str): The destination airport code\n\n Returns:\n A list of dicts, one for each scheduled flight between the two points.\n\n Example::\n from pyflightdata import FlightData\n f=FlightData()\n #optional login\n f.login(myemail,mypassword)\n f.get_flights_from_to('SIN','HYD')\n \"\"\"\n # assume limit 100 to return first 100 of any wild card search\n url = AIRLINE_FLT_BASE_POINTS.format(origin, destination)\n return self._fr24.get_airline_flight_data(url, by_airports=True)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_airport_weather(self, iata, page=1, limit=100):\n url = AIRPORT_DATA_BASE.format(iata, str(self.AUTH_TOKEN), page, limit)\n weather = self._fr24.get_airport_weather(url)\n mi = weather['sky']['visibility']['mi']\n if (mi is not None) and (mi != \"None\"):\n mi = float(mi)\n km = mi * 1.6094\n weather['sky']['visibility']['km'] = km\n return weather", "response": "Retrieve the weather at an airport"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nretrieving the metar data at the current time of the airport.", "response": "def get_airport_metars(self, iata, page=1, limit=100):\n \"\"\"Retrieve the metar data at the current time\n\n Given the IATA code of an airport, this method returns the metar information.\n\n Args:\n iata (str): The IATA code for an airport, e.g. HYD\n page (int): Optional page number; for users who are on a plan with flightradar24 they can pass in higher page numbers to get more data\n limit (int): Optional limit on number of records returned\n\n Returns:\n The metar data for the airport\n\n Example::\n\n from pyflightdata import FlightData\n f=FlightData()\n #optional login\n f.login(myemail,mypassword)\n f.get_airport_metars('HYD')\n\n \"\"\"\n url = AIRPORT_DATA_BASE.format(iata, str(self.AUTH_TOKEN), page, limit)\n w = self._fr24.get_airport_weather(url)\n return w['metar']"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_airport_metars_hist(self, iata):\n url = AIRPORT_BASE.format(iata) + \"/weather\"\n return self._fr24.get_airport_metars_hist(url)", "response": "Retrieve the metar data for the airport."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_airport_stats(self, iata, page=1, limit=100):\n url = AIRPORT_DATA_BASE.format(iata, str(self.AUTH_TOKEN), page, limit)\n return self._fr24.get_airport_stats(url)", "response": "Retrieve the performance statistics at an airport"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nretrieving the details of an airport", "response": "def get_airport_details(self, iata, page=1, limit=100):\n \"\"\"Retrieve the details of an airport\n\n Given the IATA code of an airport, this method returns the detailed information like lat lon, full name, URL, codes etc.\n\n Args:\n iata (str): The IATA code for an airport, e.g. HYD\n page (int): Optional page number; for users who are on a plan with flightradar24 they can pass in higher page numbers to get more data\n limit (int): Optional limit on number of records returned\n\n Returns:\n A list of dicts with the data; one dict for each row of data from flightradar24\n\n Example::\n\n from pyflightdata import FlightData\n f=FlightData()\n #optional login\n f.login(myemail,mypassword)\n f.get_airport_details('HYD')\n f.get_airport_details('HYD',page=1,limit=10)\n\n \"\"\"\n url = AIRPORT_DATA_BASE.format(iata, str(self.AUTH_TOKEN), page, limit)\n details = self._fr24.get_airport_details(url)\n weather = self._fr24.get_airport_weather(url)\n # weather has more correct and standard elevation details in feet and meters\n details['position']['elevation'] = weather['elevation']\n return details"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfetches the images of a particular aircraft by its tail number. This method can be used to get the images of the aircraft. The images are in 3 sizes and you can use what suits your need. Args: tail_number (str): The tail number, e.g. VT-ANL page (int): Optional page number; for users who are on a plan with flightradar24 they can pass in higher page numbers to get more data limit (int): Optional limit on number of records returned Returns: A dict with the images of the aircraft in various sizes Example:: from pyflightdata import FlightData f=FlightData() #optional login f.login(myemail,mypassword) f.get_images_by_flight_number('VT-ANL') f.get_images_by_flight_number('VT-ANL',page=1,limit=10)", "response": "def get_images_by_tail_number(self, tail_number, page=1, limit=100):\n \"\"\"Fetch the images of a particular aircraft by its tail number.\n\n This method can be used to get the images of the aircraft. The images are in 3 sizes and you can use what suits your need.\n\n Args:\n tail_number (str): The tail number, e.g. VT-ANL\n page (int): Optional page number; for users who are on a plan with flightradar24 they can pass in higher page numbers to get more data\n limit (int): Optional limit on number of records returned\n\n Returns:\n A dict with the images of the aircraft in various sizes\n\n Example::\n\n from pyflightdata import FlightData\n f=FlightData()\n #optional login\n f.login(myemail,mypassword)\n f.get_images_by_flight_number('VT-ANL')\n f.get_images_by_flight_number('VT-ANL',page=1,limit=10)\n \"\"\"\n url = REG_BASE.format(tail_number, str(self.AUTH_TOKEN), page, limit)\n return self._fr24.get_aircraft_image_data(url)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef login(self, email, password):\n response = FlightData.session.post(\n url=LOGIN_URL,\n data={\n 'email': email,\n 'password': password,\n 'remember': 'true',\n 'type': 'web'\n },\n headers={\n 'Origin': 'https://www.flightradar24.com',\n 'Referer': 'https://www.flightradar24.com',\n 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0'\n }\n )\n response = self._fr24.json_loads_byteified(\n response.content) if response.status_code == 200 else None\n if response:\n token = response['userData']['subscriptionKey']\n self.AUTH_TOKEN = token", "response": "Login to the flightradar24 session"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef decode_metar(self, metar):\n try:\n from metar import Metar\n except:\n return \"Unable to parse metars. Please install parser from https://github.com/tomp/python-metar.\"\n m = Metar.Metar(metar)\n return m.string()", "response": "Simple method that decodes a given metar string into a readable format."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_auth_packet(self, username, password, client):\n pkt = client.CreateAuthPacket(code=AccessRequest,\n User_Name=username)\n pkt[\"User-Password\"] = pkt.PwCrypt(password)\n pkt[\"NAS-Identifier\"] = 'django-radius'\n for key, val in list(getattr(settings, 'RADIUS_ATTRIBUTES', {}).items()):\n pkt[key] = val\n return pkt", "response": "Get the pyrad authentication packet for the username and password and the pyrad client."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_client(self, server):\n return Client(\n server=server[0],\n authport=server[1],\n secret=server[2],\n dict=self._get_dictionary(),\n )", "response": "Get the pyrad client for a given server."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _perform_radius_auth(self, client, packet):\n try:\n reply = client.SendPacket(packet)\n except Timeout as e:\n logging.error(\"RADIUS timeout occurred contacting %s:%s\" % (\n client.server, client.authport))\n return False\n except Exception as e:\n logging.error(\"RADIUS error: %s\" % e)\n return False\n\n if reply.code == AccessReject:\n logging.warning(\"RADIUS access rejected for user '%s'\" % (\n packet['User-Name']))\n return False\n elif reply.code != AccessAccept:\n logging.error(\"RADIUS access error for user '%s' (code %s)\" % (\n packet['User-Name'], reply.code))\n return False\n\n logging.info(\"RADIUS access granted for user '%s'\" % (\n packet['User-Name']))\n return True", "response": "Perform the actual radius authentication by passing the given packet to the client which is bound to. Returns True or False depending on whether the user is authenticated otherwise False."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nauthenticates the given username and password against the RADIUS server.", "response": "def _radius_auth(self, server, username, password):\n \"\"\"\n Authenticate the given username/password against the RADIUS server\n described by `server`.\n \"\"\"\n client = self._get_client(server)\n packet = self._get_auth_packet(username, password, client)\n return self._perform_radius_auth(client, packet)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget the Django user with the given username or create one if it doesn t exist.", "response": "def get_django_user(self, username, password=None):\n \"\"\"\n Get the Django user with the given username, or create one if it\n doesn't already exist. If `password` is given, then set the user's\n password to that (regardless of whether the user was created or not).\n \"\"\"\n try:\n user = User.objects.get(username=username)\n except User.DoesNotExist:\n user = User(username=username)\n\n if password is not None:\n user.set_password(password)\n user.save()\n\n return user"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nauthenticates against the RADIUS server and return a User object or None.", "response": "def authenticate(self, request, username=None, password=None):\n \"\"\"\n Check credentials against RADIUS server and return a User object or\n None.\n \"\"\"\n if isinstance(username, basestring):\n username = username.encode('utf-8')\n\n if isinstance(password, basestring):\n password = password.encode('utf-8')\n\n server = self._get_server_from_settings()\n result = self._radius_auth(server, username, password)\n\n if result:\n return self.get_django_user(username, password)\n\n return None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef authenticate(self, request, username=None, password=None, realm=None):\n if isinstance(username, basestring):\n username = username.encode('utf-8')\n\n if isinstance(password, basestring):\n password = password.encode('utf-8')\n\n server = self.get_server(realm)\n\n if not server:\n return None\n\n result = self._radius_auth(server, username, password)\n\n if result:\n full_username = self.construct_full_username(username, realm)\n return self.get_django_user(full_username, password)\n\n return None", "response": "Authenticate against the RADIUS server identified by realm and return a User object or None."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncloses then moves the file to dst.", "response": "def move(self, dst):\n \"Closes then moves the file to dst.\"\n self.close()\n shutil.move(self.path, dst)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sigma_clipping(date, mag, err, threshold=3, iteration=1):\n\n # Check length.\n if (len(date) != len(mag)) \\\n or (len(date) != len(err)) \\\n or (len(mag) != len(err)):\n raise RuntimeError('The length of date, mag, and err must be same.')\n\n # By magnitudes\n for i in range(int(iteration)):\n mean = np.median(mag)\n std = np.std(mag)\n\n index = (mag >= mean - threshold*std) & (mag <= mean + threshold*std)\n date = date[index]\n mag = mag[index]\n err = err[index]\n\n return date, mag, err", "response": "Remove any fluctuated data points by magnitudes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a schema object from a spec.", "response": "def from_spec(spec):\n \"\"\"Return a schema object from a spec.\n\n A spec is either a string for a scalar type, or a list of 0 or 1 specs,\n or a dictionary with two elements: {'fields': { ... }, required: [...]}.\n \"\"\"\n if spec == '':\n return any_schema\n\n if framework.is_str(spec):\n # Scalar type\n if spec not in SCALAR_TYPES:\n raise exceptions.SchemaError('Not a valid schema type: %r' % spec)\n return ScalarSchema(spec)\n\n if framework.is_list(spec):\n return ListSchema(spec[0] if len(spec) else any_schema)\n\n if framework.is_tuple(spec):\n return TupleSchema(spec.get('fields', {}), spec.get('required', []))\n\n raise exceptions.SchemaError('Not valid schema spec; %r' % spec)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef validate(obj, schema):\n if not framework.EvaluationContext.current().validate:\n # Short circuit evaluation when disabled\n return obj\n\n # Validate returned object according to its own schema\n if hasattr(obj, 'tuple_schema'):\n obj.tuple_schema.validate(obj)\n # Validate object according to externally imposed schema\n if schema:\n schema.validate(obj)\n return obj", "response": "Validate an object according to its own AND an externally imposed schema."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef attach(obj, schema):\n\n # We have a silly exception for lists, since they have no 'attach_schema'\n # method, and I don't feel like making a subclass for List just to add it.\n # So, we recursively search the list for tuples and attach the schema in\n # there.\n if framework.is_list(obj) and isinstance(schema, ListSchema):\n for x in obj:\n attach(x, schema.element_schema)\n return\n\n # Otherwise, the object should be able to handle its own schema attachment.\n getattr(obj, 'attach_schema', nop)(schema)", "response": "Attach the given schema to the given object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_feature_set_all():\n\n features = get_feature_set()\n\n features.append('cusum')\n features.append('eta')\n features.append('n_points')\n features.append('period_SNR')\n features.append('period_log10FAP')\n features.append('period_uncertainty')\n features.append('weighted_mean')\n features.append('weighted_std')\n\n features.sort()\n\n return features", "response": "Returns a list of all features in the alphabetical order."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parameters(self):\n parameters = []\n for hl in self.hidden_layers:\n parameters.extend(hl.parameters)\n parameters.extend(self.top_layer.parameters)\n return parameters", "response": "A property that returns all of the model s parameters."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the parameters of the current object.", "response": "def parameters(self, value):\n \"\"\" Used to set all of the model's parameters to new values.\n\n **Parameters:**\n\n value : array_like\n New values for the model parameters. Must be of length\n ``self.n_parameters``.\n \"\"\"\n \n if len(value) != self.n_parameters:\n raise ValueError(\"Incorrect length of parameter vector. \"\n \"Model has %d parameters, but got %d\" %\n (self.n_parameters, len(value)))\n\n i = 0\n for hl in self.hidden_layers:\n hl.parameters = value[i:i + hl.n_parameters]\n i += hl.n_parameters\n\n self.top_layer.parameters = value[-self.top_layer.n_parameters:]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef checksum(self):\n \n m = md5()\n for hl in self.hidden_layers:\n m.update(str(hl.architecture))\n m.update(str(self.top_layer.architecture))\n return m.hexdigest()", "response": "Returns an MD5 digest of the model."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef evaluate(self, input_data, targets,\n return_cache=False, prediction=True):\n \"\"\" Evaluate the loss function without computing gradients.\n\n **Parameters:**\n\n input_data : GPUArray\n Data to evaluate\n\n targets: GPUArray\n Targets\n\n return_cache : bool, optional\n Whether to return intermediary variables from the\n computation and the hidden activations.\n\n prediction : bool, optional\n Whether to use prediction model. Only relevant when using\n dropout. If true, then weights are multiplied by\n 1 - dropout if the layer uses dropout.\n\n **Returns:**\n\n loss : float\n The value of the loss function.\n\n hidden_cache : list, only returned if ``return_cache == True``\n Cache as returned by :meth:`hebel.models.NeuralNet.feed_forward`.\n\n activations : list, only returned if ``return_cache == True``\n Hidden activations as returned by\n :meth:`hebel.models.NeuralNet.feed_forward`.\n \"\"\"\n\n # Forward pass\n activations, hidden_cache = self.feed_forward(\n input_data, return_cache=True, prediction=prediction)\n\n loss = self.top_layer.train_error(None,\n targets, average=False, cache=activations,\n prediction=prediction)\n\n for hl in self.hidden_layers:\n if hl.l1_penalty_weight: loss += hl.l1_penalty\n if hl.l2_penalty_weight: loss += hl.l2_penalty\n\n if self.top_layer.l1_penalty_weight: loss += self.top_layer.l1_penalty\n if self.top_layer.l2_penalty_weight: loss += self.top_layer.l2_penalty\n\n if not return_cache:\n return loss\n else:\n return loss, hidden_cache, activations", "response": "Evaluate the loss function without computing gradients."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef training_pass(self, input_data, targets):\n\n # Forward pass\n loss, hidden_cache, logistic_cache = self.evaluate(\n input_data, targets, return_cache=True, prediction=False)\n\n if not np.isfinite(loss):\n raise ValueError('Infinite activations!')\n\n # Backpropagation\n if self.hidden_layers:\n hidden_activations = hidden_cache[-1][0]\n else:\n hidden_activations = input_data\n\n df_top_layer = \\\n self.top_layer.backprop(hidden_activations, targets,\n cache=logistic_cache)\n gradients = list(df_top_layer[0][::-1])\n df_hidden = df_top_layer[1]\n\n if self.hidden_layers:\n hidden_inputs = [input_data] + [c[0] for c in hidden_cache[:-1]] \n for hl, hc, hi in \\\n zip(self.hidden_layers[::-1], hidden_cache[::-1],\n hidden_inputs[::-1]):\n g, df_hidden = hl.backprop(hi, df_hidden, cache=hc)\n gradients.extend(g[::-1])\n\n gradients.reverse()\n\n return loss, gradients", "response": "Perform a full forward and backward pass through the model."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrun data forward through the model.", "response": "def feed_forward(self, input_data, return_cache=False, prediction=True):\n \"\"\" Run data forward through the model.\n\n **Parameters:**\n\n input_data : GPUArray\n Data to run through the model.\n\n return_cache : bool, optional\n Whether to return the intermediary results.\n\n prediction : bool, optional\n Whether to run in prediction mode. Only relevant when\n using dropout. If true, weights are multiplied by 1 - dropout.\n If false, then half of hidden units are randomly dropped and\n the dropout mask is returned in case ``return_cache==True``.\n\n **Returns:**\n \n prediction : GPUArray\n Predictions from the model.\n\n cache : list of GPUArray, only returned if ``return_cache == True``\n Results of intermediary computations. \n \"\"\"\n\n hidden_cache = None # Create variable in case there are no hidden layers\n if self.hidden_layers:\n # Forward pass\n hidden_cache = []\n for i in range(len(self.hidden_layers)):\n hidden_activations = hidden_cache[i - 1][0] if i else input_data\n # Use dropout predict if previous layer has dropout\n hidden_cache.append(self.hidden_layers[i]\n .feed_forward(hidden_activations,\n prediction=prediction))\n\n hidden_activations = hidden_cache[-1][0]\n\n else:\n hidden_activations = input_data\n\n # Use dropout_predict if last hidden layer has dropout\n activations = \\\n self.top_layer.feed_forward(hidden_activations,\n prediction=False)\n\n if return_cache:\n return activations, hidden_cache\n return activations"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nderive not - period - based features.", "response": "def shallow_run(self):\n \"\"\"Derive not-period-based features.\"\"\"\n # Number of data points\n self.n_points = len(self.date)\n\n # Weight calculation.\n # All zero values.\n if not self.err.any():\n self.err = np.ones(len(self.mag)) * np.std(self.mag)\n # Some zero values.\n elif not self.err.all():\n np.putmask(self.err, self.err==0, np.median(self.err))\n\n self.weight = 1. / self.err\n self.weighted_sum = np.sum(self.weight)\n\n # Simple statistics, mean, median and std.\n self.mean = np.mean(self.mag)\n self.median = np.median(self.mag)\n self.std = np.std(self.mag)\n\n # Weighted mean and std.\n self.weighted_mean = np.sum(self.mag * self.weight) / self.weighted_sum\n self.weighted_std = np.sqrt(np.sum((self.mag - self.weighted_mean) ** 2 \\\n * self.weight) / self.weighted_sum)\n\n # Skewness and kurtosis.\n self.skewness = ss.skew(self.mag)\n self.kurtosis = ss.kurtosis(self.mag)\n\n # Normalization-test. Shapiro-Wilk test.\n shapiro = ss.shapiro(self.mag)\n self.shapiro_w = shapiro[0]\n # self.shapiro_log10p = np.log10(shapiro[1])\n\n # Percentile features.\n self.quartile31 = np.percentile(self.mag, 75) \\\n - np.percentile(self.mag, 25)\n\n # Stetson K.\n self.stetson_k = self.get_stetson_k(self.mag, self.median, self.err)\n\n # Ratio between higher and lower amplitude than average.\n self.hl_amp_ratio = self.half_mag_amplitude_ratio(\n self.mag, self.median, self.weight)\n # This second function's value is very similar with the above one.\n # self.hl_amp_ratio2 = self.half_mag_amplitude_ratio2(\n # self.mag, self.median)\n\n # Cusum\n self.cusum = self.get_cusum(self.mag)\n\n # Eta\n self.eta = self.get_eta(self.mag, self.weighted_std)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nderive period - based features.", "response": "def deep_run(self):\n \"\"\"Derive period-based features.\"\"\"\n # Lomb-Scargle period finding.\n self.get_period_LS(self.date, self.mag, self.n_threads, self.min_period)\n\n # Features based on a phase-folded light curve\n # such as Eta, slope-percentile, etc.\n # Should be called after the getPeriodLS() is called.\n\n # Created phased a folded light curve.\n # We use period * 2 to take eclipsing binaries into account.\n phase_folded_date = self.date % (self.period * 2.)\n sorted_index = np.argsort(phase_folded_date)\n\n folded_date = phase_folded_date[sorted_index]\n folded_mag = self.mag[sorted_index]\n\n # phase Eta\n self.phase_eta = self.get_eta(folded_mag, self.weighted_std)\n\n # Slope percentile.\n self.slope_per10, self.slope_per90 = \\\n self.slope_percentile(folded_date, folded_mag)\n\n # phase Cusum\n self.phase_cusum = self.get_cusum(folded_mag)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets uncertainty of a period. The uncertainty is defined as the half width of the frequencies around the peak, that becomes lower than average + standard deviation of the power spectrum. Since we may not have fine resolution around the peak, we do not assume it is gaussian. So, no scaling factor of 2.355 (= 2 * sqrt(2 * ln2)) is applied. Parameters ---------- fx : array_like An array of frequencies. fy : array_like An array of amplitudes. jmax : int An index at the peak frequency. fx_width : int, optional Width of power spectrum to calculate uncertainty. Returns ------- p_uncertain : float Period uncertainty.", "response": "def get_period_uncertainty(self, fx, fy, jmax, fx_width=100):\n \"\"\"\n Get uncertainty of a period.\n\n The uncertainty is defined as the half width of the frequencies\n around the peak, that becomes lower than average + standard deviation\n of the power spectrum.\n\n Since we may not have fine resolution around the peak,\n we do not assume it is gaussian. So, no scaling factor of\n 2.355 (= 2 * sqrt(2 * ln2)) is applied.\n\n Parameters\n ----------\n fx : array_like\n An array of frequencies.\n fy : array_like\n An array of amplitudes.\n jmax : int\n An index at the peak frequency.\n fx_width : int, optional\n Width of power spectrum to calculate uncertainty.\n\n Returns\n -------\n p_uncertain : float\n Period uncertainty.\n \"\"\"\n\n # Get subset\n start_index = jmax - fx_width\n end_index = jmax + fx_width\n if start_index < 0:\n start_index = 0\n if end_index > len(fx) - 1:\n end_index = len(fx) - 1\n\n fx_subset = fx[start_index:end_index]\n fy_subset = fy[start_index:end_index]\n fy_mean = np.median(fy_subset)\n fy_std = np.std(fy_subset)\n\n # Find peak\n max_index = np.argmax(fy_subset)\n\n # Find list whose powers become lower than average + std.\n index = np.where(fy_subset <= fy_mean + fy_std)[0]\n\n # Find the edge at left and right. This is the full width.\n left_index = index[(index < max_index)]\n if len(left_index) == 0:\n left_index = 0\n else:\n left_index = left_index[-1]\n right_index = index[(index > max_index)]\n if len(right_index) == 0:\n right_index = len(fy_subset) - 1\n else:\n right_index = right_index[0]\n\n # We assume the half of the full width is the period uncertainty.\n half_width = (1. / fx_subset[left_index]\n - 1. / fx_subset[right_index]) / 2.\n period_uncertainty = half_width\n\n return period_uncertainty"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the residuals of the Fourier Series.", "response": "def residuals(self, pars, x, y, order):\n \"\"\"\n Residual of Fourier Series.\n\n Parameters\n ----------\n pars : array_like\n Fourier series parameters.\n x : array_like\n An array of date.\n y : array_like\n An array of true values to fit.\n order : int\n An order of Fourier Series.\n \"\"\"\n\n return y - self.fourier_series(pars, x, order)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfunction to fit Fourier Series.", "response": "def fourier_series(self, pars, x, order):\n \"\"\"\n Function to fit Fourier Series.\n\n Parameters\n ----------\n x : array_like\n An array of date divided by period. It doesn't need to be sorted.\n pars : array_like\n Fourier series parameters.\n order : int\n An order of Fourier series.\n \"\"\"\n\n sum = pars[0]\n for i in range(order):\n sum += pars[i * 2 + 1] * np.sin(2 * np.pi * (i + 1) * x) \\\n + pars[i * 2 + 2] * np.cos(2 * np.pi * (i + 1) * x)\n\n return sum"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_stetson_k(self, mag, avg, err):\n\n residual = (mag - avg) / err\n stetson_k = np.sum(np.fabs(residual)) \\\n / np.sqrt(np.sum(residual * residual)) / np.sqrt(len(mag))\n\n return stetson_k", "response": "Returns the Stetson K feature."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning ratio of amplitude of higher and lower magnitudes than average.", "response": "def half_mag_amplitude_ratio(self, mag, avg, weight):\n \"\"\"\n Return ratio of amplitude of higher and lower magnitudes.\n\n\n A ratio of amplitude of higher and lower magnitudes than average,\n considering weights. This ratio, by definition, should be higher\n for EB than for others.\n\n Parameters\n ----------\n mag : array_like\n An array of magnitudes.\n avg : float\n An average value of magnitudes.\n weight : array_like\n An array of weight.\n\n Returns\n -------\n hl_ratio : float\n Ratio of amplitude of higher and lower magnitudes than average.\n \"\"\"\n\n # For lower (fainter) magnitude than average.\n index = np.where(mag > avg)\n lower_weight = weight[index]\n lower_weight_sum = np.sum(lower_weight)\n lower_mag = mag[index]\n lower_weighted_std = np.sum((lower_mag\n - avg) ** 2 * lower_weight) / \\\n lower_weight_sum\n\n # For higher (brighter) magnitude than average.\n index = np.where(mag <= avg)\n higher_weight = weight[index]\n higher_weight_sum = np.sum(higher_weight)\n higher_mag = mag[index]\n higher_weighted_std = np.sum((higher_mag\n - avg) ** 2 * higher_weight) / \\\n higher_weight_sum\n\n # Return ratio.\n return np.sqrt(lower_weighted_std / higher_weighted_std)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef half_mag_amplitude_ratio2(self, mag, avg):\n\n # For lower (fainter) magnitude than average.\n index = np.where(mag > avg)\n fainter_mag = mag[index]\n\n lower_sum = np.sum((fainter_mag - avg) ** 2) / len(fainter_mag)\n\n # For higher (brighter) magnitude than average.\n index = np.where(mag <= avg)\n brighter_mag = mag[index]\n\n higher_sum = np.sum((avg - brighter_mag) ** 2) / len(brighter_mag)\n\n # Return ratio.\n return np.sqrt(lower_sum / higher_sum)", "response": "Return ratio of amplitude of higher and lower magnitudes than average."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_eta(self, mag, std):\n\n diff = mag[1:] - mag[:len(mag) - 1]\n eta = np.sum(diff * diff) / (len(mag) - 1.) / std / std\n\n return eta", "response": "Returns the value of the Eta feature."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating 10 and 90% percentile of slope.", "response": "def slope_percentile(self, date, mag):\n \"\"\"\n Return 10% and 90% percentile of slope.\n\n Parameters\n ----------\n date : array_like\n An array of phase-folded date. Sorted.\n mag : array_like\n An array of phase-folded magnitudes. Sorted by date.\n\n Returns\n -------\n per_10 : float\n 10% percentile values of slope.\n per_90 : float\n 90% percentile values of slope.\n \"\"\"\n\n date_diff = date[1:] - date[:len(date) - 1]\n mag_diff = mag[1:] - mag[:len(mag) - 1]\n\n # Remove zero mag_diff.\n index = np.where(mag_diff != 0.)\n date_diff = date_diff[index]\n mag_diff = mag_diff[index]\n\n # Derive slope.\n slope = date_diff / mag_diff\n\n percentile_10 = np.percentile(slope, 10.)\n percentile_90 = np.percentile(slope, 90.)\n\n return percentile_10, percentile_90"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_cusum(self, mag):\n\n c = np.cumsum(mag - self.weighted_mean) / len(mag) / self.weighted_std\n\n return np.max(c) - np.min(c)", "response": "Returns the cumulative sum of the entry in the set of magnitudes."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning all features with its names and values.", "response": "def get_features2(self):\n \"\"\"\n Return all features with its names.\n\n Returns\n -------\n names : list\n Feature names.\n values : list\n Feature values\n \"\"\"\n\n feature_names = []\n feature_values = []\n\n # Get all the names of features.\n all_vars = vars(self)\n for name in all_vars.keys():\n # Omit input variables such as date, mag, err, etc.\n if not (name == 'date' or name == 'mag' or name == 'err'\n or name == 'n_threads' or name == 'min_period'):\n # Filter some other unnecessary features.\n if not (name == 'f' or name == 'f_phase'\n or name == 'period_log10FAP'\n or name == 'weight' or name == 'weighted_sum'\n or name == 'median' or name == 'mean' or name == 'std'):\n feature_names.append(name)\n\n # Sort by the names.\n # Sorting should be done to keep maintaining the same order of features.\n feature_names.sort()\n\n # Get feature values.\n for name in feature_names:\n feature_values.append(all_vars[name])\n\n return feature_names, feature_values"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn all features with its names.", "response": "def get_features_all(self):\n \"\"\"\n Return all features with its names.\n\n Regardless of being used for train and prediction. Sorted by the names.\n\n Returns\n -------\n all_features : OrderedDict\n Features dictionary.\n \"\"\"\n\n features = {}\n\n # Get all the names of features.\n all_vars = vars(self)\n for name in all_vars.keys():\n if name in feature_names_list_all:\n features[name] = all_vars[name]\n\n # Sort by the keys (i.e. feature names).\n features = OrderedDict(sorted(features.items(), key=lambda t: t[0]))\n\n return features"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef init(device_id=None, random_seed=None):\n\n if device_id is None:\n random_seed = _os.environ.get('CUDA_DEVICE')\n \n if random_seed is None:\n random_seed = _os.environ.get('RANDOM_SEED')\n\n global is_initialized\n if not is_initialized:\n is_initialized = True\n\n global context\n context.init_context(device_id)\n\n from pycuda import gpuarray, driver, curandom\n\n # Initialize memory pool\n global memory_pool\n memory_pool.init()\n\n # Initialize PRG\n global sampler\n sampler.set_seed(random_seed)\n\n # Initialize pycuda_ops\n from hebel import pycuda_ops\n pycuda_ops.init()", "response": "Initialize the CUDA context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context and CUBLAS context."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef inflate_context_tuple(ast_rootpath, root_env):\n with util.LogTime('inflate_context_tuple'):\n # We only need to look at tuple members going down.\n inflated = ast_rootpath[0].eval(root_env)\n current = inflated\n env = root_env\n try:\n for node in ast_rootpath[1:]:\n if is_tuple_member_node(node):\n assert framework.is_tuple(current)\n with util.LogTime('into tuple'):\n thunk, env = inflated.get_thunk_env(node.name)\n current = framework.eval(thunk, env)\n\n elif framework.is_list(current):\n with util.LogTime('eval thing'):\n current = framework.eval(node, env)\n\n if framework.is_tuple(current):\n inflated = current\n except (gcl.EvaluationError, ast.UnparseableAccess):\n # Eat evaluation error, probably means the rightmost tuplemember wasn't complete.\n # Return what we have so far.\n pass\n return inflated", "response": "Instantiate a tuple from a TupleNode."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef enumerate_scope(ast_rootpath, root_env=None, include_default_builtins=False):\n with util.LogTime('enumerate_scope'):\n scope = {}\n for node in reversed(ast_rootpath):\n if is_tuple_node(node):\n for member in node.members:\n if member.name not in scope:\n scope[member.name] = Completion(member.name, False, member.comment.as_string(), member.location)\n\n if include_default_builtins: # Backwards compat flag\n root_env = gcl.default_env\n\n if root_env:\n for k in root_env.keys():\n if k not in scope and not hide_from_autocomplete(root_env[k]):\n v = root_env[k]\n scope[k] = Completion(k, True, dedent(v.__doc__ or ''), None)\n\n return scope", "response": "Enumerates all keys that are in scope in a given tuple."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a dict of all completions that are declared in the given AST rootpath.", "response": "def find_deref_completions(ast_rootpath, root_env=gcl.default_env):\n \"\"\"Returns a dict of { name => Completions }.\"\"\"\n with util.LogTime('find_deref_completions'):\n tup = inflate_context_tuple(ast_rootpath, root_env)\n path = path_until(ast_rootpath, is_deref_node)\n if not path:\n return {}\n deref = path[-1]\n haystack = deref.haystack(tup.env(tup))\n if not hasattr(haystack, 'keys'):\n return {}\n return {n: get_completion(haystack, n) for n in haystack.keys()}"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef is_identifier_position(rootpath):\n if len(rootpath) >= 2 and is_tuple_member_node(rootpath[-2]) and is_identifier(rootpath[-1]):\n return True\n if len(rootpath) >= 1 and is_tuple_node(rootpath[-1]):\n # No deeper node than tuple? Must be identifier position, otherwise we'd have a TupleMemberNode.\n return True\n return False", "response": "Return whether the cursor is in identifier - position in a member declaration."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinds completions at the cursor.", "response": "def find_completions_at_cursor(ast_tree, filename, line, col, root_env=gcl.default_env):\n \"\"\"Find completions at the cursor.\n\n Return a dict of { name => Completion } objects.\n \"\"\"\n q = gcl.SourceQuery(filename, line, col - 1)\n rootpath = ast_tree.find_tokens(q)\n\n if is_identifier_position(rootpath):\n return find_inherited_key_completions(rootpath, root_env)\n\n try:\n ret = find_deref_completions(rootpath, root_env) or enumerate_scope(rootpath, root_env=root_env)\n assert isinstance(ret, dict)\n return ret\n except gcl.EvaluationError:\n # Probably an unbound value or something--just return an empty list\n return {}"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_inherited_key_completions(rootpath, root_env):\n tup = inflate_context_tuple(rootpath, root_env)\n if isinstance(tup, runtime.CompositeTuple):\n keys = set(k for t in tup.tuples[:-1] for k in t.keys())\n return {n: get_completion(tup, n) for n in keys}\n return {}", "response": "Return the completions of INHERITED tuples."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_value_at_cursor(ast_tree, filename, line, col, root_env=gcl.default_env):\n q = gcl.SourceQuery(filename, line, col)\n rootpath = ast_tree.find_tokens(q)\n rootpath = path_until(rootpath, is_thunk)\n\n if len(rootpath) <= 1:\n # Just the file tuple itself, or some non-thunk element at the top level\n return None\n\n tup = inflate_context_tuple(rootpath, root_env)\n try:\n if isinstance(rootpath[-1], ast.Inherit):\n # Special case handling of 'Inherit' nodes, show the value that's being\n # inherited.\n return tup[rootpath[-1].name]\n return rootpath[-1].eval(tup.env(tup))\n except gcl.EvaluationError as e:\n return e", "response": "Find the value of the object under the cursor."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_vec_to_mat(mat, vec, axis=None, inplace=False,\n target=None, substract=False):\n \"\"\" Add a vector to a matrix\n \"\"\"\n\n assert mat.flags.c_contiguous\n\n if axis is None:\n if vec.shape[0] == mat.shape[0]:\n axis = 0\n elif vec.shape[0] == mat.shape[1]:\n axis = 1\n else:\n raise ValueError('Vector length must be equal '\n 'to one side of the matrix')\n\n n, m = mat.shape\n\n block = (_compilation_constants['add_vec_block_size'],\n _compilation_constants['add_vec_block_size'], 1)\n gridx = ceil_div(n, block[0])\n gridy = ceil_div(m, block[1])\n grid = (gridx, gridy, 1)\n\n if inplace:\n target = mat\n elif target is None:\n target = gpuarray.empty_like(mat)\n\n if axis == 0:\n assert vec.shape[0] == mat.shape[0]\n add_col_vec_kernel.prepared_call(\n grid, block,\n mat.gpudata,\n vec.gpudata,\n target.gpudata,\n np.uint32(n),\n np.uint32(m),\n np.int32(substract))\n elif axis == 1:\n assert vec.shape[0] == mat.shape[1]\n add_row_vec_kernel.prepared_call(\n grid, block,\n mat.gpudata,\n vec.gpudata,\n target.gpudata,\n np.uint32(n),\n np.uint32(m),\n np.int32(substract))\n return target", "response": "Add a vector to a matrix."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef vector_normalize(mat, max_vec_norm=1.):\n assert mat.flags.c_contiguous\n n, m = mat.shape\n\n vector_normalize_kernel.prepared_call(\n (m, 1, 1), (32, 1, 1),\n mat.gpudata,\n np.float32(max_vec_norm),\n np.int32(m),\n np.int32(n))", "response": "Normalize each column vector in mat to length max_vec_norm."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef preprocess(string):\n\n split = string.split('${')\n\n rval = [split[0]]\n\n for candidate in split[1:]:\n subsplit = candidate.split('}')\n\n if len(subsplit) < 2:\n raise ValueError('Open ${ not followed by } before ' \\\n + 'end of string or next ${ in \"' \\\n + string + '\"')\n\n varname = subsplit[0]\n\n if varname == 'PYLEARN2_TRAIN_FILE_NAME':\n warnings.warn(\"PYLEARN2_TRAIN_FILE_NAME is deprecated and may be \"\n \"removed from the library on or after Oct 22, 2013. Switch\"\n \" to PYLEARN2_TRAIN_FILE_FULL_STEM\")\n\n try:\n val = os.environ[varname]\n except KeyError:\n if varname == 'PYLEARN2_DATA_PATH':\n raise NoDataPathError()\n if varname == 'PYLEARN2_VIEWER_COMMAND':\n raise EnvironmentVariableError(environment_variable_essay)\n\n raise ValueError('Unrecognized environment variable \"' + varname\n + '\". Did you mean ' + match(varname, os.environ.keys())\n + '?')\n\n rval.append(val)\n\n rval.append('}'.join(subsplit[1:]))\n\n rval = ''.join(rval)\n\n return rval", "response": "Preprocesses a string by replacing the VARNAME with the value of the environment variable PYLEARN2_TRAIN_FILE_FULL_STEM."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef tokenize_by_number(s):\n\n r = find_number(s)\n\n if r == None:\n return [ s ]\n else:\n tokens = []\n if r[0] > 0:\n tokens.append(s[0:r[0]])\n tokens.append( float(s[r[0]:r[1]]) )\n if r[1] < len(s):\n tokens.extend(tokenize_by_number(s[r[1]:]))\n return tokens\n assert False", "response": "splits a string into a list of tokens\nAttributeNames each is either a string containing no numbers\nAttributeNames or a float"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwronging a mispelling candidates: a set of correct words returns a guess of which candidate is the right one This should be used with a small number of candidates and a high potential edit distance. ie, use it to correct a wrong filename in a directory, wrong class name in a module, etc. Don't use it to correct small typos of freeform natural language words.", "response": "def match(wrong, candidates):\n \"\"\"\n wrong: a mispelling\n candidates: a set of correct words\n\n returns a guess of which candidate is the right one\n\n This should be used with a small number of candidates and a high potential\n edit distance.\n ie, use it to correct a wrong filename in a directory, wrong class name\n in a module, etc. Don't use it to correct small typos of freeform natural\n language words.\n \"\"\"\n\n assert len(candidates) > 0\n\n # Current implementation tries all candidates and outputs the one\n # with the min score\n # Could try to do something smarter\n\n def score(w1,w2):\n # Current implementation returns negative dot product of\n # the two words mapped into a feature space by mapping phi\n # w -> [ phi(w1), .1 phi(first letter of w), .1 phi(last letter of w) ]\n # Could try to do something smarter\n\n w1 = w1.lower()\n w2 = w2.lower()\n\n def phi(w):\n # Current feature mapping is to the vector of counts of\n # all letters and two-letter sequences\n # Could try to do something smarter\n rval = {}\n\n for i in xrange(len(w)):\n l = w[i]\n rval[l] = rval.get(l,0.) + 1.\n if i < len(w)-1:\n b = w[i:i+2]\n rval[b] = rval.get(b,0.) + 1.\n\n return rval\n\n d1 = phi(w1)\n d2 = phi(w2)\n\n def mul(d1, d2):\n rval = 0\n\n for key in set(d1).union(d2):\n rval += d1.get(key,0) * d2.get(key,0)\n\n return rval\n\n tot_score = mul(phi(w1),phi(w2)) / float(len(w1)*len(w2)) + \\\n 0.1 * mul(phi(w1[0:1]), phi(w2[0:1])) + \\\n 0.1 * mul(phi(w1[-1:]), phi(w2[-1:]))\n\n return tot_score\n\n scored_candidates = [ (-score(wrong, candidate), candidate)\n for candidate in candidates ]\n\n scored_candidates.sort()\n\n return scored_candidates[0][1]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef censor_non_alphanum(s):\n\n def censor(ch):\n if (ch >= 'A' and ch <= 'z') or (ch >= '0' and ch <= '9'):\n return ch\n return '*'\n\n return ''.join([censor(ch) for ch in s])", "response": "Returns a string with all non - alphanumeric characters replaced with *"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck if a given period is possibly an alias.", "response": "def is_period_alias(period):\n \"\"\"\n Check if a given period is possibly an alias.\n\n Parameters\n ----------\n period : float\n A period to test if it is a possible alias or not.\n\n Returns\n -------\n is_alias : boolean\n True if the given period is in a range of period alias.\n \"\"\"\n\n # Based on the period vs periodSN plot of EROS-2 dataset (Kim+ 2014).\n # Period alias occurs mostly at ~1 and ~30.\n # Check each 1, 2, 3, 4, 5 factors.\n for i in range(1, 6):\n # One-day and one-month alias\n if (.99 / float(i)) < period < (1.004 / float(i)):\n return True\n if (1.03 / float(i)) < period < (1.04 / float(i)):\n return True\n if (29.2 / float(i)) < period < (29.9 / float(i)):\n return True\n\n # From candidates from the two fields 01, 08.\n # All of them are close to one day (or sidereal) alias.\n if (0.96465 / float(i)) < period < (0.96485 / float(i)):\n return True\n if (0.96725 / float(i)) < period < (0.96745 / float(i)):\n return True\n if (0.98190 / float(i)) < period < (0.98230 / float(i)):\n return True\n if (1.01034 / float(i)) < period < (1.01076 / float(i)):\n return True\n if (1.01568 / float(i)) < period < (1.01604 / float(i)):\n return True\n if (1.01718 / float(i)) < period < (1.01742 / float(i)):\n return True\n\n # From the all candidates from the entire LMC fields.\n # Some of these could be overlapped with the above cuts.\n if (0.50776 / float(i)) < period < (0.50861 / float(i)):\n return True\n if (0.96434 / float(i)) < period < (0.9652 / float(i)):\n return True\n if (0.96688 / float(i)) < period < (0.96731 / float(i)):\n return True\n if (1.0722 / float(i)) < period < (1.0729 / float(i)):\n return True\n if (27.1 / float(i)) < period < (27.5 / float(i)):\n return True\n\n # Not in the range of any alias.\n return False"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nserializing object to a file denoted by filepath.", "response": "def save(filepath, obj, on_overwrite = 'ignore'):\n \"\"\"\n Serialize `object` to a file denoted by `filepath`.\n\n Parameters\n ----------\n filepath : str\n A filename. If the suffix is `.joblib` and joblib can be\n imported, `joblib.dump` is used in place of the regular\n pickling mechanisms; this results in much faster saves by\n saving arrays as separate .npy files on disk. If the file\n suffix is `.npy` than `numpy.save` is attempted on `obj`.\n Otherwise, (c)pickle is used.\n\n obj : object\n A Python object to be serialized.\n\n on_overwrite: A string specifying what to do if the file already\n exists.\n ignore: just overwrite it\n backup: make a copy of the file (.bak) and\n delete it when done saving the new copy.\n this allows recovery of the old version of\n the file if saving the new one fails\n \"\"\"\n\n\n filepath = preprocess(filepath)\n\n if os.path.exists(filepath):\n if on_overwrite == 'backup':\n backup = filepath + '.bak'\n shutil.move(filepath, backup)\n save(filepath, obj)\n try:\n os.remove(backup)\n except Exception, e:\n warnings.warn(\"Got an error while traing to remove \"+backup+\":\"+str(e))\n return\n else:\n assert on_overwrite == 'ignore'\n\n\n try:\n _save(filepath, obj)\n except RuntimeError, e:\n \"\"\" Sometimes for large theano graphs, pickle/cPickle exceed the\n maximum recursion depth. This seems to me like a fundamental\n design flaw in pickle/cPickle. The workaround I employ here\n is the one recommended to someone who had a similar problem\n on stackexchange:\n\n http://stackoverflow.com/questions/2134706/hitting-maximum-recursion-depth-using-pythons-pickle-cpickle\n\n Obviously this does not scale and could cause a crash\n but I don't see another solution short of writing our\n own implementation of pickle.\n \"\"\"\n if str(e).find('recursion') != -1:\n warnings.warn('pylearn2.utils.save encountered the following '\n 'error: ' + str(e) +\n '\\nAttempting to resolve this error by calling ' +\n 'sys.setrecusionlimit and retrying')\n old_limit = sys.getrecursionlimit()\n try:\n sys.setrecursionlimit(50000)\n _save(filepath, obj)\n finally:\n sys.setrecursionlimit(old_limit)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_pickle_protocol():\n try:\n protocol_str = os.environ['PYLEARN2_PICKLE_PROTOCOL']\n except KeyError:\n # If not defined, we default to 0 because this is the default\n # protocol used by cPickle.dump (and because it results in\n # maximum portability)\n protocol_str = '0'\n if protocol_str == 'pickle.HIGHEST_PROTOCOL':\n return pickle.HIGHEST_PROTOCOL\n return int(protocol_str)", "response": "Get the pickle protocol for this node."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_train_file(config_file_path):\n from pylearn2.config import yaml_parse\n\n suffix_to_strip = '.yaml'\n\n # publish environment variables related to file name\n if config_file_path.endswith(suffix_to_strip):\n config_file_full_stem = config_file_path[0:-len(suffix_to_strip)]\n else:\n config_file_full_stem = config_file_path\n\n for varname in [\"PYLEARN2_TRAIN_FILE_NAME\", #this one is deprecated\n \"PYLEARN2_TRAIN_FILE_FULL_STEM\"]: #this is the new, accepted name\n environ.putenv(varname, config_file_full_stem)\n\n directory = config_file_path.split('/')[:-1]\n directory = '/'.join(directory)\n if directory != '':\n directory += '/'\n environ.putenv(\"PYLEARN2_TRAIN_DIR\", directory)\n environ.putenv(\"PYLEARN2_TRAIN_BASE_NAME\", config_file_path.split('/')[-1] )\n environ.putenv(\"PYLEARN2_TRAIN_FILE_STEM\", config_file_full_stem.split('/')[-1] )\n\n return yaml_parse.load_path(config_file_path)", "response": "Loads and parses a yaml file for a Train object."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\npropagating forward through the layer.", "response": "def feed_forward(self, input_data, prediction=False):\n \"\"\"Propagate forward through the layer\n\n **Parameters:**\n\n input_data : ``GPUArray``\n Inpute data to perform dropout on.\n\n prediction : bool, optional\n Whether to use prediction model. If true, then the data is\n scaled by ``1 - dropout_probability`` uses dropout.\n\n **Returns:**\n \n dropout_data : ``GPUArray``\n The data after performing dropout.\n \"\"\"\n\n if input_data.shape[1] != self.n_in:\n raise ValueError('Number of outputs from previous layer (%d) '\n 'does not match number of inputs to this layer (%d)' %\n (input_data.shape[1], self.n_in))\n\n if not prediction:\n dropout_input = gpuarray.empty_like(input_data)\n dropout_mask = sample_dropout_mask(input_data,\n self.dropout_probability, target=dropout_input\n )\n return dropout_input, dropout_mask\n else:\n return (input_data * (1 - self.dropout_probability),)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef backprop(self, input_data, df_output, cache=None):\n\n if self.compute_input_gradients: \n apply_dropout_mask(df_output, dropout_mask)\n\n return tuple(), df_output", "response": "Backpropagate through the hidden layer."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates ctypes pointer to object.", "response": "def POINTER(obj):\n \"\"\"\n Create ctypes pointer to object.\n\n Notes\n -----\n This function converts None to a real NULL pointer because of bug\n in how ctypes handles None on 64-bit platforms.\n\n \"\"\"\n\n p = ctypes.POINTER(obj)\n if not isinstance(p.from_param, classmethod):\n def from_param(cls, x):\n if x is None:\n return cls()\n else:\n return x\n p.from_param = classmethod(from_param)\n\n return p"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef gpuarray_ptr(g):\n\n addr = int(g.gpudata)\n if g.dtype == np.int8:\n return ctypes.cast(addr, POINTER(ctypes.c_byte))\n if g.dtype == np.uint8:\n return ctypes.cast(addr, POINTER(ctypes.c_ubyte))\n if g.dtype == np.int16:\n return ctypes.cast(addr, POINTER(ctypes.c_short))\n if g.dtype == np.uint16:\n return ctypes.cast(addr, POINTER(ctypes.c_ushort))\n if g.dtype == np.int32:\n return ctypes.cast(addr, POINTER(ctypes.c_int))\n if g.dtype == np.uint32:\n return ctypes.cast(addr, POINTER(ctypes.c_uint))\n if g.dtype == np.int64:\n return ctypes.cast(addr, POINTER(ctypes.c_long))\n if g.dtype == np.uint64:\n return ctypes.cast(addr, POINTER(ctypes.c_ulong))\n if g.dtype == np.float32:\n return ctypes.cast(addr, POINTER(ctypes.c_float))\n elif g.dtype == np.float64:\n return ctypes.cast(addr, POINTER(ctypes.c_double))\n elif g.dtype == np.complex64:\n return ctypes.cast(addr, POINTER(cuFloatComplex))\n elif g.dtype == np.complex128:\n return ctypes.cast(addr, POINTER(cuDoubleComplex))\n else:\n raise ValueError('unrecognized type')", "response": "Return ctypes pointer to data in GPUAarray object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nallocate memory on the device associated with the current active .", "response": "def cudaMalloc(count, ctype=None):\n \"\"\"\n Allocate device memory.\n\n Allocate memory on the device associated with the current active\n context.\n\n Parameters\n ----------\n count : int\n Number of bytes of memory to allocate\n ctype : _ctypes.SimpleType, optional\n ctypes type to cast returned pointer.\n\n Returns\n -------\n ptr : ctypes pointer\n Pointer to allocated device memory.\n\n \"\"\"\n\n ptr = ctypes.c_void_p()\n status = _libcudart.cudaMalloc(ctypes.byref(ptr), count)\n cudaCheckStatus(status)\n if ctype != None:\n ptr = ctypes.cast(ptr, ctypes.POINTER(ctype))\n return ptr"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cudaMallocPitch(pitch, rows, cols, elesize):\n\n ptr = ctypes.c_void_p()\n status = _libcudart.cudaMallocPitch(ctypes.byref(ptr),\n ctypes.c_size_t(pitch), cols*elesize,\n rows)\n cudaCheckStatus(status)\n return ptr, pitch", "response": "Allocate pitched memory on the device associated with the current active\n ."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncopying memory from host to device.", "response": "def cudaMemcpy_htod(dst, src, count):\n \"\"\"\n Copy memory from host to device.\n\n Copy data from host memory to device memory.\n\n Parameters\n ----------\n dst : ctypes pointer\n Device memory pointer.\n src : ctypes pointer\n Host memory pointer.\n count : int\n Number of bytes to copy.\n\n \"\"\"\n\n status = _libcudart.cudaMemcpy(dst, src,\n ctypes.c_size_t(count),\n cudaMemcpyHostToDevice)\n cudaCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncopies memory from device to host memory.", "response": "def cudaMemcpy_dtoh(dst, src, count):\n \"\"\"\n Copy memory from device to host.\n\n Copy data from device memory to host memory.\n\n Parameters\n ----------\n dst : ctypes pointer\n Host memory pointer.\n src : ctypes pointer\n Device memory pointer.\n count : int\n Number of bytes to copy.\n\n \"\"\"\n\n status = _libcudart.cudaMemcpy(dst, src,\n ctypes.c_size_t(count),\n cudaMemcpyDeviceToHost)\n cudaCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cudaMemGetInfo():\n\n free = ctypes.c_size_t()\n total = ctypes.c_size_t()\n status = _libcudart.cudaMemGetInfo(ctypes.byref(free),\n ctypes.byref(total))\n cudaCheckStatus(status)\n return free.value, total.value", "response": "Return the amount of free and total device memory in bytes."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets current CUDA device number.", "response": "def cudaGetDevice():\n \"\"\"\n Get current CUDA device.\n\n Return the identifying number of the device currently used to\n process CUDA operations.\n\n Returns\n -------\n dev : int\n Device number.\n\n \"\"\"\n\n dev = ctypes.c_int()\n status = _libcudart.cudaGetDevice(ctypes.byref(dev))\n cudaCheckStatus(status)\n return dev.value"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the version of the CUDA driver as an integer.", "response": "def cudaDriverGetVersion():\n \"\"\"\n Get installed CUDA driver version.\n\n Return the version of the installed CUDA driver as an integer. If\n no driver is detected, 0 is returned.\n\n Returns\n -------\n version : int\n Driver version.\n\n \"\"\"\n\n version = ctypes.c_int()\n status = _libcudart.cudaDriverGetVersion(ctypes.byref(version))\n cudaCheckStatus(status)\n return version.value"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cudaPointerGetAttributes(ptr):\n\n attributes = cudaPointerAttributes()\n status = \\\n _libcudart.cudaPointerGetAttributes(ctypes.byref(attributes), ptr)\n cudaCheckStatus(status)\n return attributes.memoryType, attributes.device", "response": "Get memory pointer attributes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nevaluate a thunk in an environment.", "response": "def eval(thunk, env):\n \"\"\"Evaluate a thunk in an environment.\n\n Will defer the actual evaluation to the thunk itself, but adds two things:\n caching and recursion detection.\n\n Since we have to use a global evaluation stack (because there is a variety of functions that may\n be invoked, not just eval() but also __getitem__, and not all of them can pass along a context\n object), GCL evaluation is not thread safe.\n\n With regard to schemas:\n\n - A schema can be passed in from outside. The returned object will be validated to see that it\n conforms to the schema. The schema will be attached to the value if possible.\n - Some objects may contain their own schema, such as tuples. This would be out of scope of the\n eval() function, were it not for:\n - Schema validation can be disabled in an evaluation call stack. This is useful if we're\n evaluating a tuple only for its schema information. At that point, we're not interested if the\n object is value-complete.\n \"\"\"\n key = Activation.key(thunk, env)\n if Activation.activated(key):\n raise exceptions.RecursionError('Reference cycle')\n\n with Activation(key):\n return eval_cache.get(key, thunk.eval, env)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_node(self, key):\n if key in self.names:\n return self.values.get_member_node(key) if hasattr(self.values, 'get_member_node') else None\n return self.parent.get_node(key)", "response": "Delegate to our current value provider for the node belonging to this key."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_table(cls):\n schema_editor = getattr(connection, 'schema_editor', None)\n if schema_editor:\n with schema_editor() as schema_editor:\n schema_editor.create_model(cls)\n else:\n raw_sql, _ = connection.creation.sql_create_model(\n cls,\n no_style(),\n [])\n cls.delete_table()\n cursor = connection.cursor()\n try:\n cursor.execute(*raw_sql)\n finally:\n cursor.close()", "response": "Create a temporary table for the given model."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndelete a temporary table for model in test data base.", "response": "def delete_table(cls):\n \"\"\"\n delete_table\n\n Manually delete a temporary table for model in test data base.\n :return:\n \"\"\"\n schema_editor = getattr(connection, 'schema_editor', None)\n if schema_editor:\n with connection.schema_editor() as schema_editor:\n schema_editor.delete_model(cls)\n else:\n cursor = connection.cursor()\n try:\n with warnings.catch_warnings():\n warnings.filterwarnings('ignore', 'unknown table')\n cursor.execute('DROP TABLE IF EXISTS {0}'.format(cls._meta.db_table))\n finally:\n cursor.close()"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates temporary model only for given test method", "response": "def fake_me(cls, source):\n \"\"\"\n fake_me\n\n Class or method decorator\n\n Class decorator: create temporary table for all tests in SimpleTestCase.\n Method decorator: create temporary model only for given test method.\n :param source: SimpleTestCase or test function\n :return:\n \"\"\"\n if source and type(source) == type and issubclass(source, SimpleTestCase):\n return cls._class_extension(source)\n elif hasattr(source, '__call__'):\n return cls._decorator(source)\n else:\n raise AttributeError('source - must be a SimpleTestCase subclass of function')"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreset the current configuration to default settings.", "response": "def reset(cls):\n \"\"\"\n Reset to default settings\n \"\"\"\n cls.debug = False\n cls.disabled = False\n cls.overwrite = False\n cls.playback_only = False\n cls.recv_timeout = 5\n cls.recv_endmarkers = []\n cls.recv_size = None"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef walk(value, walker, path=None, seen=None):\n seen = seen or set()\n path = path or []\n\n # Recursion\n if id(value) in seen:\n walker.visitRecursion(path)\n return\n\n # Error\n if isinstance(value, Exception):\n walker.visitError(path, value)\n return\n\n # List\n if isinstance(value, list):\n # Not actually a tuple, but okay\n recurse = walker.enterList(value, path)\n if not recurse: return\n next_walker = walker if recurse is True else recurse\n\n with TempSetAdd(seen, id(value)):\n for i, x in enumerate(value):\n walk(x, next_walker, path=path + ['[%d]' % i], seen=seen)\n\n walker.leaveList(value, path)\n return\n\n # Scalar\n if not isinstance(value, framework.TupleLike):\n walker.visitScalar(path, value)\n return\n\n # Tuple\n recurse = walker.enterTuple(value, path)\n if not recurse: return\n next_walker = walker if recurse is True else recurse\n\n with TempSetAdd(seen, id(value)):\n keys = sorted(value.keys())\n for key in keys:\n key_path = path + [key]\n elm = get_or_error(value, key)\n walk(elm, next_walker, path=key_path, seen=seen)\n\n walker.leaveTuple(value, path)", "response": "Walks the _evaluated_ tree of the given GCL tuple."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef fingerprint(value):\n h = hashlib.sha256()\n _digest(value, h)\n return h.digest().encode('hex')", "response": "Return a hash value that uniquely identifies the GCL value."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the last 2 error messages from an error stack.", "response": "def compact_error(err):\n \"\"\"Return the the last 2 error messages from an error stack.\n\n These error messages turns out to be the most descriptive.\n \"\"\"\n def err2(e):\n if isinstance(e, exceptions.EvaluationError) and e.inner:\n message, i = err2(e.inner)\n if i == 1:\n return ', '.join([e.args[0], str(e.inner)]), i + 1\n else:\n return message, i + 1\n else:\n return str(e), 1\n return err2(err)[0]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef backprop(self, input_data, targets,\n cache=None):\n \"\"\" Backpropagate through the logistic layer.\n\n **Parameters:**\n\n input_data : ``GPUArray``\n Inpute data to compute activations for.\n\n targets : ``GPUArray``\n The target values of the units.\n\n cache : list of ``GPUArray``\n Cache obtained from forward pass. If the cache is\n provided, then the activations are not recalculated.\n\n **Returns:**\n\n gradients : tuple of ``GPUArray``\n Gradients with respect to the weights and biases in the\n form ``(df_weights, df_biases)``.\n\n df_input : ``GPUArray``\n Gradients with respect to the input.\n \"\"\"\n\n if cache is not None:\n activations = cache\n else:\n activations = self.feed_forward(input_data, prediction=False)\n\n if activations.shape != targets.shape:\n raise ValueError('Activations (shape = %s) and targets (shape = %s) are different sizes' %\n (activations.shape, targets.shape))\n\n delta = substract_matrix(activations, targets)\n nan_to_zeros(delta, delta)\n\n # Gradient wrt weights\n df_W = linalg.dot(input_data, delta, transa='T')\n # Gradient wrt bias\n df_b = matrix_sum_out_axis(delta, 0)\n\n # Gradient wrt input\n df_input = linalg.dot(delta, self.W, transb='T')\n\n # L1 penalty\n if self.l1_penalty_weight:\n df_W += self.l1_penalty_weight * sign(self.W)\n\n # L2 penalty\n if self.l2_penalty_weight:\n df_W += self.l2_penalty_weight * self.W\n\n return (df_W, df_b), df_input", "response": "Backpropagate through the logistic layer."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the cross entropy error of the input data.", "response": "def cross_entropy_error(self, input_data, targets, average=True,\n cache=None, prediction=False):\n \"\"\" Return the cross entropy error\n \"\"\"\n\n if cache is not None:\n activations = cache\n else:\n activations = \\\n self.feed_forward(input_data, prediction=prediction)\n\n loss = cross_entropy_logistic(activations, targets)\n\n if average: loss /= targets.shape[0]\n # assert np.isfinite(loss)\n return loss.get()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing comment lines and make subsequent indented lines into a code block athe.", "response": "def stylize_comment_block(lines):\n \"\"\"Parse comment lines and make subsequent indented lines into a code block\n block.\n \"\"\"\n normal, sep, in_code = range(3)\n state = normal\n for line in lines:\n indented = line.startswith(' ')\n empty_line = line.strip() == ''\n\n if state == normal and empty_line:\n state = sep\n elif state in [sep, normal] and indented:\n yield ''\n if indented:\n yield '.. code-block:: javascript'\n yield ''\n yield line\n state = in_code\n else:\n state = normal\n elif state == sep and not empty_line:\n yield ''\n yield line\n state = normal\n else:\n yield line\n if state == in_code and not (indented or empty_line):\n sep = normal"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sort_members(tup, names):\n scalars, tuples = partition(lambda x: not is_tuple_node(tup.member[x].value), names)\n unbound, bound = partition(lambda x: tup.member[x].value.is_unbound(), scalars)\n return usorted(unbound) + usorted(bound), usorted(tuples)", "response": "Return two pairs of members scalar and tuple members."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nresolves filename relatively against one of the given paths.", "response": "def resolve_file(fname, paths):\n \"\"\"Resolve filename relatively against one of the given paths, if possible.\"\"\"\n fpath = path.abspath(fname)\n for p in paths:\n spath = path.abspath(p)\n if fpath.startswith(spath):\n return fpath[len(spath) + 1:]\n return fname"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef generate(self):\n header = ' '.join('=' * self.width[i] for i in range(self.w))\n lines = [\n ' '.join(row[i].ljust(self.width[i]) for i in range(self.w))\n for row in self.rows]\n return [header] + lines + [header]", "response": "Generate a list of strings representing the table in RST format."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef partition(pred, iterable):\n 'Use a predicate to partition entries into false entries and true entries'\n # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9\n t1, t2 = itertools.tee(iterable)\n return list(filter(negate(pred), t1)), list(filter(pred, t2))", "response": "Use a predicate to partition entries into false entries and true entries"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef select(self, model):\n res = []\n\n def doSelect(value, pre, remaining):\n if not remaining:\n res.append((pre, value))\n else:\n # For the other selectors to work, value must be a Tuple or a list at this point.\n if not is_tuple(value) and not isinstance(value, list):\n return\n\n qhead, qtail = remaining[0], remaining[1:]\n if isinstance(qhead, tuple) and is_tuple(value):\n for alt in qhead:\n if alt in value:\n doSelect(value[alt], pre + [alt], qtail)\n elif qhead == '*':\n if isinstance(value, list):\n indices = range(len(value))\n reprs = [listKey(i) for i in indices]\n else:\n indices = value.keys()\n reprs = indices\n\n for key, rep in zip(indices, reprs):\n doSelect(value[key], pre + [rep], qtail)\n elif isinstance(qhead, int) and isinstance(value, list):\n doSelect(value[qhead], pre + [listKey(qhead)], qtail)\n elif is_tuple(value):\n if qhead in value:\n doSelect(value[qhead], pre + [qhead], qtail)\n\n for selector in self.selectors:\n doSelect(model, [], selector)\n\n return QueryResult(res)", "response": "Select nodes according to the input selector."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef deep(self):\n self.lists = {}\n ret = {}\n for path, value in self.paths_values():\n self.recursiveSet(ret, path, value)\n self.removeMissingValuesFromLists()\n return ret", "response": "Returns a deep dict of the values selected."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ldSet(self, what, key, value):\n if isListKey(key):\n # Make sure we keep the indexes consistent, insert missing_values\n # as necessary. We do remember the lists, so that we can remove\n # missing values after inserting all values from all selectors.\n self.lists[id(what)] = what\n ix = listKeyIndex(key)\n while len(what) <= ix:\n what.append(missing_value)\n what[ix] = value\n else:\n what[key] = value\n return value", "response": "Set the value of the list with the given key."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ldGet(self, what, key):\n if isListKey(key):\n return what[listKeyIndex(key)]\n else:\n return what[key]", "response": "List - aware get."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn True if the list contains the key.", "response": "def ldContains(self, what, key):\n \"\"\"List/dictinary/missing-aware contains.\n\n If the value is a \"missing_value\", we'll treat it as non-existent\n so it will be overwritten by an empty list/dict when necessary to\n assign child keys.\n \"\"\"\n if isListKey(key):\n i = listKeyIndex(key)\n return i < len(what) and what[i] != missing_value\n else:\n return key in what and what[key] != missing_value"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a list of nodes that have a recursive dependency.", "response": "def find_recursive_dependency(self):\n \"\"\"Return a list of nodes that have a recursive dependency.\"\"\"\n nodes_on_path = []\n\n def helper(nodes):\n for node in nodes:\n cycle = node in nodes_on_path\n nodes_on_path.append(node)\n if cycle or helper(self.deps.get(node, [])):\n return True\n nodes_on_path.pop()\n return False\n\n helper(self.unordered)\n return nodes_on_path"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef enterTuple(self, tuple, path):\n if skip_name(path):\n return False\n node = Node(path, tuple)\n if self.condition.matches(node):\n self.unordered.append(node)\n return False\n return True", "response": "Called for every tuple."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmake a sequence of applications from a list of tokens.", "response": "def mkApplications(location, *atoms):\n \"\"\"Make a sequence of applications from a list of tokens.\n\n atoms is a list of atoms, which will be handled left-associatively. E.g:\n\n ['foo', [], []] == foo()() ==> Application(Application('foo', []), [])\n \"\"\"\n atoms = list(atoms)\n while len(atoms) > 1:\n atoms[0:2] = [Application(location, atoms[0], atoms[1])]\n\n # Nothing left to apply\n return atoms[0]"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalls a function respecting all the various types of functions that exist.", "response": "def call_fn(fn, arglist, env):\n \"\"\"Call a function, respecting all the various types of functions that exist.\"\"\"\n if isinstance(fn, framework.LazyFunction):\n # The following looks complicated, but this is necessary because you can't\n # construct closures over the loop variable directly.\n thunks = [(lambda thunk: lambda: framework.eval(thunk, env))(th) for th in arglist.values]\n return fn(*thunks)\n\n evaled_args = framework.eval(arglist, env)\n if isinstance(fn, framework.EnvironmentFunction):\n return fn(*evaled_args, env=env)\n\n return fn(*evaled_args)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the schema spec from a run - time tuple.", "response": "def schema_spec_from_tuple(tup):\n \"\"\"Return the schema spec from a run-time tuple.\"\"\"\n if hasattr(tup, 'get_schema_spec'):\n # Tuples have a TupleSchema field that contains a model of the schema\n return schema.from_spec({\n 'fields': TupleSchemaAccess(tup),\n 'required': tup.get_required_fields()})\n return schema.AnySchema()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef make_schema_from(value, env):\n\n # So this thing may not need to evaluate anything[0]\n if isinstance(value, framework.Thunk):\n value = framework.eval(value, env)\n\n # We're a bit messy. In general, this has evaluated to a Schema object, but not necessarily:\n # for tuples and lists, we still need to treat the objects as specs.\n if isinstance(value, schema.Schema):\n return value\n\n if framework.is_tuple(value):\n # If it so happens that the thing is a tuple, we need to pass in the data in a bit of a\n # different way into the schema factory (in a dictionary with {fields, required} keys).\n return schema_spec_from_tuple(value)\n\n if framework.is_list(value):\n # [0] This list may contain tuples, which oughta be treated as specs, or already-resolved schema\n # objects (as returned by 'int' and 'string' literals). make_schema_from\n # deals with both.\n return schema.from_spec([make_schema_from(x, env) for x in value])\n\n raise exceptions.EvaluationError('Can\\'t make a schema from %r' % value)", "response": "Make a Schema object from the given spec."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef bracketedList(l, r, sep, expr, allow_missing_close=False):\n # We may need to backtrack for lists, because of list comprehension, but not for\n # any of the other lists\n strict = l != '['\n closer = sym(r) if not allow_missing_close else p.Optional(sym(r))\n if strict:\n return sym(l) - listMembers(sep, expr) - closer\n else:\n return sym(l) + listMembers(sep, expr) + closer", "response": "Parse bracketed list.\n\n Empty list is possible, as is a trailing separator."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef unquote(s):\n # Ignore the left- and rightmost chars (which should be quotes).\n # Use the Python engine to decode the escape sequence\n i, N = 1, len(s) - 1\n ret = []\n while i < N:\n if s[i] == '\\\\' and i < N - 1:\n ret.append(UNQUOTE_MAP.get(s[i+1], s[i+1]))\n i += 2\n else:\n ret.append(s[i])\n i += 1\n return ''.join(ret)", "response": "Unquote the indicated string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfunctioning to put a name on a pyparsing pattern.", "response": "def pattern(name, pattern):\n \"\"\"Function to put a name on a pyparsing pattern.\n\n Just for ease of debugging/tracing parse errors.\n \"\"\"\n pattern.setName(name)\n astracing.maybe_trace(pattern)\n return pattern"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmake a grammar that can be used to parse a single expression.", "response": "def make_grammar(allow_errors):\n \"\"\"Make the part of the grammar that depends on whether we swallow errors or not.\"\"\"\n if allow_errors in GRAMMAR_CACHE:\n return GRAMMAR_CACHE[allow_errors]\n\n tuple = p.Forward()\n catch_errors = p.Forward()\n catch_errors << (p.Regex('[^{};]*') - p.Optional(tuple) - p.Regex('[^;}]*'))\n\n def swallow_remainder():\n if allow_errors:\n return pattern('swallow_remainder', p.Suppress(catch_errors))\n return p.Empty()\n\n def swallow_errors(rule):\n \"\"\"Extend the production rule by potentially eating errors.\n\n This does not return a p.NoMatch() because that messes up the error messages.\n \"\"\"\n ret = rule\n if allow_errors:\n # Synchronize on the first semicolon or the first unbalanced closing curly\n ret = rule | pattern('catch_errors', parseWithLocation(p.Suppress(catch_errors), UnparseableNode))\n return ret\n\n class Grammar:\n keywords = ['and', 'or', 'not', 'if', 'then', 'else', 'include', 'inherit', 'null', 'true', 'false',\n 'for', 'in']\n\n # This is a hack: this condition helps uselessly recursing into the grammar for\n # juxtapositions.\n early_abort_scan = ~p.oneOf([';', ',', ']', '}', 'for' ])\n\n expression = pattern('expression', p.Forward())\n\n comment = p.Regex('#') + ~p.FollowedBy(sym('.')) + p.restOfLine\n doc_comment = pattern('doc_comment', (sym('#.') - p.restOfLine))\n\n quotedIdentifier = pattern('quotedIdentifier', p.QuotedString('`', multiline=False))\n\n # - Must start with an alphascore\n # - May contain alphanumericscores and special characters such as : and -\n # - Must not end in a special character\n identifier = pattern('identifier', parseWithLocation(quotedIdentifier | p.Regex(r'[a-zA-Z_]([a-zA-Z0-9_:-]*[a-zA-Z0-9_])?'), Identifier))\n\n # Variable identifier (can't be any of the keywords, which may have lower matching priority)\n variable = pattern('variable', ~p.MatchFirst(p.oneOf(keywords)) + pattern('identifier', parseWithLocation(identifier.copy(), Var)))\n\n # Contants\n integer = pattern('integer', parseWithLocation(p.Word(p.nums), convertAndMake(int, Literal)))\n floating = pattern('floating', parseWithLocation(p.Regex(r'\\d*\\.\\d+'), convertAndMake(float, Literal)))\n dq_string = pattern('dq_string', parseWithLocation(p.QuotedString('\"', escChar='\\\\', unquoteResults=False, multiline=True), convertAndMake(unquote, Literal)))\n sq_string = pattern('sq_string', parseWithLocation(p.QuotedString(\"'\", escChar='\\\\', unquoteResults=False, multiline=True), convertAndMake(unquote, Literal)))\n boolean = pattern('boolean', parseWithLocation(p.Keyword('true') | p.Keyword('false'), convertAndMake(mkBool, Literal)))\n null = pattern('null', parseWithLocation(p.Keyword('null'), Null))\n\n # List\n list_ = pattern('list', parseWithLocation(bracketedList('[', ']', ',', expression), List))\n\n # Tuple\n inherit = pattern('inherit', (kw('inherit') - p.ZeroOrMore(variable)).setParseAction(inheritNodes))\n schema_spec = pattern('schema_spec', parseWithLocation(p.Optional(p.Keyword('private').setParseAction(lambda: True), default=False)\n - p.Optional(p.Keyword('required').setParseAction(lambda: True), default=False)\n - p.Optional(expression, default=any_schema_expr), MemberSchemaNode))\n optional_schema = pattern('optional_schema', p.Optional(p.Suppress(':') - schema_spec, default=no_schema))\n\n expression_value = pattern('expression_value', sym('=') - swallow_errors(expression))\n void_value = pattern('void_value', parseWithLocation(p.FollowedBy(sym(';') | sym('}')), lambda loc: Void(loc, 'nonameyet')))\n member_value = pattern('member_value', swallow_errors(expression_value | void_value))\n named_member = pattern('named_member', parseWithLocation(identifier - optional_schema - member_value - swallow_remainder(), TupleMemberNode))\n documented_member = pattern('documented_member', parseWithLocation(parseWithLocation(p.ZeroOrMore(doc_comment), DocComment) + named_member, attach_doc_comment))\n tuple_member = early_abort_scan + pattern('tuple_member', swallow_errors(inherit | documented_member) - swallow_remainder())\n\n ErrorAwareTupleNode = functools.partial(TupleNode, allow_errors)\n tuple_members = pattern('tuple_members', parseWithLocation(listMembers(';', tuple_member), ErrorAwareTupleNode))\n tuple << pattern('tuple', parseWithLocation(bracketedList('{', '}', ';', tuple_member, allow_missing_close=allow_errors), ErrorAwareTupleNode))\n\n # Argument list will live by itself as a atom. Actually, it's a tuple, but we\n # don't call it that because we use that term for something else already :)\n arg_list = pattern('arg_list', bracketedList('(', ')', ',', expression).setParseAction(ArgList))\n\n parenthesized_expr = pattern('parenthesized_expr', (sym('(') - expression - ')').setParseAction(head))\n\n unary_op = pattern('unary_op', (p.oneOf(' '.join(functions.unary_operators.keys())) - expression).setParseAction(mkUnOp))\n\n if_then_else = pattern('if_then_else', parseWithLocation(kw('if') + expression +\n kw('then') + expression +\n kw('else') + expression, Condition))\n\n list_comprehension = pattern('list_comprehension', parseWithLocation(sym('[') + expression + kw('for') + variable + kw('in') +\n expression + p.Optional(kw('if') + expression) + sym(']'), ListComprehension))\n\n\n # We don't allow space-application here\n # Now our grammar is becoming very dirty and hackish\n deref = pattern('deref', p.Forward())\n include = pattern('include', parseWithLocation(kw('include') - deref, Include))\n\n atom = pattern('atom', (tuple\n | sq_string\n | dq_string\n | variable\n | floating\n | integer\n | boolean\n | list_\n | null\n | unary_op\n | parenthesized_expr\n | if_then_else\n | include\n | list_comprehension\n ))\n\n # We have two different forms of function application, so they can have 2\n # different precedences. This one: fn(args), which binds stronger than\n # dereferencing (fn(args).attr == (fn(args)).attr)\n applic1 = pattern('applic1', parseWithLocation(atom - p.ZeroOrMore(arg_list), mkApplications))\n\n # Dereferencing of an expression (obj.bar)\n deref << parseWithLocation(applic1 - p.ZeroOrMore(p.Suppress('.') - swallow_errors(identifier)), mkDerefs)\n\n # All binary operators at various precedence levels go here:\n # This piece of code does the moral equivalent of:\n #\n # T = F*F | F/F | F\n # E = T+T | T-T | T\n #\n # etc.\n term = deref\n for op_level in functions.binary_operators_before_juxtaposition:\n operator_syms = list(op_level.keys())\n term = (term - p.ZeroOrMore(p.oneOf(operator_syms) - term)).setParseAction(mkBinOps)\n\n # Juxtaposition function application (fn arg), must be 1-arg every time\n applic2 = pattern('applic2', parseWithLocation(term - p.ZeroOrMore(early_abort_scan + term), mkApplications))\n\n term = applic2\n for op_level in functions.binary_operators_after_juxtaposition:\n operator_syms = list(op_level.keys())\n term = (term - p.ZeroOrMore(p.oneOf(operator_syms) - term)).setParseAction(mkBinOps)\n\n expression << term\n\n # Two entry points: start at an arbitrary expression, or expect the top-level\n # scope to be a tuple.\n start = pattern('start', expression.copy().ignore(comment))\n start_tuple = tuple_members.ignore(comment)\n GRAMMAR_CACHE[allow_errors] = Grammar\n return Grammar"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef reads(s, filename, loader, implicit_tuple, allow_errors):\n try:\n the_context.filename = filename\n the_context.loader = loader\n\n grammar = make_grammar(allow_errors=allow_errors)\n root = grammar.start_tuple if implicit_tuple else grammar.start\n\n return root.parseWithTabs().parseString(s, parseAll=True)[0]\n except (p.ParseException, p.ParseSyntaxException) as e:\n loc = SourceLocation(s, find_offset(s, e.lineno, e.col))\n raise exceptions.ParseError(the_context.filename, loc, e.msg)", "response": "Load but don t evaluate a GCL expression from a string."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfind all AST nodes at the given filename line and column.", "response": "def find_tokens(self, q):\n \"\"\"Find all AST nodes at the given filename, line and column.\"\"\"\n found_me = []\n if hasattr(self, 'location'):\n if self.location.contains(q):\n found_me = [self]\n elif self._found_by(q):\n found_me = [self]\n\n cs = [n.find_tokens(q) for n in self._children()]\n return found_me + list(itertools.chain(*cs))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef applyTuple(self, tuple, right, env):\n if len(right) != 1:\n raise exceptions.EvaluationError('Tuple (%r) can only be applied to one argument, got %r' % (self.left, self.right))\n right = right[0]\n\n return tuple(right)", "response": "Apply a tuple to something else."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef applyIndex(self, lst, right):\n if len(right) != 1:\n raise exceptions.EvaluationError('%r can only be applied to one argument, got %r' % (self.left, self.right))\n right = right[0]\n\n if isinstance(right, int):\n return lst[right]\n\n raise exceptions.EvaluationError(\"Can't apply %r to argument (%r): integer expected, got %r\" % (self.left, self.right, right))", "response": "Apply a list to something else."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates the parameters of the current object based on the current velocity.", "response": "def pre_gradient_update(self):\n \"\"\" First step of Nesterov momentum method:\n take step in direction of accumulated gradient\n \"\"\"\n\n updates = zip(self.velocity, self.model.n_parameters * [1.])\n self.model.update_parameters(updates)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the classification error rate for the current state of the target.", "response": "def class_error(self, input_data, targets, average=True,\n cache=None, prediction=False):\n \"\"\" Return the classification error rate\n \"\"\"\n\n if cache is not None:\n activations = cache\n else:\n activations = \\\n self.feed_forward(input_data, prediction=prediction)\n\n targets = targets.get().argmax(1)\n class_error = np.sum(activations.get().argmax(1) != targets)\n\n if average: class_error = float(class_error) / targets.shape[0]\n return class_error"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef dot(x_gpu, y_gpu, transa='N', transb='N', handle=None, target=None):\n\n if handle is None:\n handle = _global_cublas_handle\n\n if len(x_gpu.shape) == 1 and len(y_gpu.shape) == 1:\n\n if x_gpu.size != y_gpu.size:\n raise ValueError('arrays must be of same length: '\n 'x_gpu.size = %d, y_gpu.size = %d' %\n (x_gpu.size, y_gpu.size))\n\n # Compute inner product for 1D arrays:\n if (x_gpu.dtype == np.complex64 and y_gpu.dtype == np.complex64):\n cublas_func = cublas.cublasCdotu\n elif (x_gpu.dtype == np.float32 and y_gpu.dtype == np.float32):\n cublas_func = cublas.cublasSdot\n elif (x_gpu.dtype == np.complex128 and y_gpu.dtype == np.complex128):\n cublas_func = cublas.cublasZdotu\n elif (x_gpu.dtype == np.float64 and y_gpu.dtype == np.float64):\n cublas_func = cublas.cublasDdot\n else:\n raise ValueError('unsupported combination of input types: '\n 'x_gpu.dtype = %s, y_gpu.dtype = %s' %\n (str(x_gpu.dtype), str(y_gpu.dtype)))\n\n return cublas_func(handle, x_gpu.size, x_gpu.gpudata, 1,\n y_gpu.gpudata, 1)\n else:\n\n # Get the shapes of the arguments (accounting for the\n # possibility that one of them may only have one dimension):\n x_shape = x_gpu.shape\n y_shape = y_gpu.shape\n if len(x_shape) == 1:\n x_shape = (1, x_shape[0])\n if len(y_shape) == 1:\n y_shape = (1, y_shape[0])\n\n # Perform matrix multiplication for 2D arrays:\n if (x_gpu.dtype == np.complex64 and y_gpu.dtype == np.complex64):\n cublas_func = cublas.cublasCgemm\n alpha = np.complex64(1.0)\n beta = np.complex64(0.0)\n elif (x_gpu.dtype == np.float32 and y_gpu.dtype == np.float32):\n cublas_func = cublas.cublasSgemm\n alpha = np.float32(1.0)\n beta = np.float32(0.0)\n elif (x_gpu.dtype == np.complex128 and y_gpu.dtype == np.complex128):\n cublas_func = cublas.cublasZgemm\n alpha = np.complex128(1.0)\n beta = np.complex128(0.0)\n elif (x_gpu.dtype == np.float64 and y_gpu.dtype == np.float64):\n cublas_func = cublas.cublasDgemm\n alpha = np.float64(1.0)\n beta = np.float64(0.0)\n else:\n raise ValueError('unsupported combination of input types: '\n 'x_gpu.dtype = %s, y_gpu.dtype = %s' %\n (str(x_gpu.dtype), str(y_gpu.dtype)))\n\n transa = lower(transa)\n transb = lower(transb)\n\n if transb in ['t', 'c']:\n m, k = y_shape\n elif transb in ['n']:\n k, m = y_shape\n else:\n raise ValueError('invalid value \"%s\" for transb' % transb)\n\n if transa in ['t', 'c']:\n l, n = x_shape\n elif transa in ['n']:\n n, l = x_shape\n else:\n raise ValueError('invalid value \"%s\" for transa' % transa)\n\n if l != k:\n raise ValueError('objects are not aligned: x_shape = %s, y_shape = %s' %\n (x_shape, y_shape))\n\n if transb == 'n':\n lda = max(1, m)\n else:\n lda = max(1, k)\n\n if transa == 'n':\n ldb = max(1, k)\n else:\n ldb = max(1, n)\n\n ldc = max(1, m)\n\n # Note that the desired shape of the output matrix is the transpose\n # of what CUBLAS assumes:\n\n if target is None:\n target = gpuarray.empty((n, ldc), x_gpu.dtype, allocator=memory_pool.allocate)\n \n cublas_func(handle, transb, transa, m, n, k, alpha, y_gpu.gpudata,\n lda, x_gpu.gpudata, ldb, beta, target.gpudata, ldc)\n\n return target", "response": "This function computes the dot product of two arrays."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef make_tempfile(data=None):\n \"Create a temp file, write our PID into it.\"\n with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp:\n temp.write(six.text_type(data if data is not None else os.getpid()))\n return temp.name", "response": "Create a temp file write our PID into it."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a list where each element contains the parameters for a task.", "response": "def parameters(self):\n \"\"\"Return a list where each element contains the parameters for a task.\n \"\"\"\n parameters = []\n for task in self.tasks:\n parameters.extend(task.parameters)\n return parameters"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates the parameters. ``value`` must be a list/tuple of length ``MultitaskTopLayer.n_tasks``, each element of which must have the correct number of parameters for the task.", "response": "def parameters(self, value):\n \"\"\"Update the parameters.\n\n ``value`` must be a list/tuple of length\n ``MultitaskTopLayer.n_tasks``, each element of which must have\n the correct number of parameters for the task.\n \"\"\"\n\n assert len(value) == self.n_parameters\n i = 0\n for task in self.tasks:\n task.parameters = value[i:i + task.n_parameters]\n i += task.n_parameters"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef feed_forward(self, input_data, prediction=False):\n\n activations = []\n\n for task in self.tasks:\n activations_task = task.feed_forward(input_data, prediction)\n activations.append(activations_task)\n\n return activations", "response": "Call feed_forward for each task and combine the activations for each output unit."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef backprop(self, input_data, targets, cache=None):\n\n df_input = gpuarray.zeros_like(input_data)\n\n if cache is None: cache = self.n_tasks * [None]\n\n gradients = []\n for targets_task, cache_task, task, task_weight in \\\n izip(targets, cache, self.tasks, self.task_weights):\n gradients_task, df_input_task = \\\n task.backprop(input_data, targets_task,\n cache_task)\n\n df_input = df_input.mul_add(1., df_input_task, task_weight)\n\n gradients.extend(gradients_task)\n\n return gradients, df_input", "response": "Compute gradients for each task and combine the results."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing the cross - entropy error for all tasks.", "response": "def cross_entropy_error(self, input_data, targets, average=True,\n cache=None, prediction=False,\n sum_errors=True):\n \"\"\" Computes the cross-entropy error for all tasks.\n \"\"\"\n\n loss = []\n if cache is None:\n cache = self.n_tasks * [None]\n\n for targets_task, cache_task, task in \\\n izip(targets, cache, self.tasks):\n loss.append(task.cross_entropy_error(\n input_data, targets_task, average=average,\n cache=cache_task,\n prediction=prediction))\n\n if sum_errors:\n return sum(loss)\n else:\n return loss"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parameters(self, value):\n self.W = value[0] if isinstance(value[0], GPUArray) else \\\n gpuarray.to_gpu(value[0])\n self.b = value[1] if isinstance(value[0], GPUArray) else \\\n gpuarray.to_gpu(value[1])", "response": "Update the parameters. ``value`` must have the shape\n ``(weights, biases)``"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef architecture(self):\n arch = {'class': self.__class__,\n 'n_in': self.n_in,\n 'n_units': self.n_units,\n 'activation_function': self.activation_function\n if hasattr(self, 'activation_function') else None}\n return arch", "response": "Returns a dictionary describing the architecture of the layer."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\npropagating forward through the layer.", "response": "def feed_forward(self, input_data, prediction=False):\n \"\"\"Propagate forward through the layer\n\n **Parameters:**\n\n input_data : ``GPUArray``\n Input data to compute activations for.\n\n prediction : bool, optional\n Whether to use prediction model. Only relevant when using\n dropout. If true, then weights are multiplied by\n 1 - dropout if the layer uses dropout.\n\n **Returns:**\n \n activations : ``GPUArray``\n The activations of the hidden units.\n \"\"\"\n\n if input_data.shape[1] != self.W.shape[0]:\n raise ValueError('Number of outputs from previous layer (%d) '\n 'does not match number of inputs to this layer (%d)' %\n (input_data.shape[1], self.W.shape[0]))\n\n activations = linalg.dot(input_data, self.W)\n activations = add_vec_to_mat(activations, self.b, inplace=True)\n\n self.f(activations)\n\n if self.dropout > 0:\n if prediction:\n activations *= 1 - self.dropout\n else:\n dropout_mask = sample_dropout_mask(activations, self.dropout)\n return activations, dropout_mask\n\n return (activations,)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cublasCreate():\n\n handle = ctypes.c_void_p()\n status = _libcublas.cublasCreate_v2(ctypes.byref(handle))\n cublasCheckStatus(status)\n return handle.value", "response": "Initialize CUBLAS.\n\n Initializes CUBLAS and creates a handle to a structure holding\n the CUBLAS library context.\n\n Returns\n -------\n handle : void_p\n CUBLAS context."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrelease CUBLAS resources. Releases hardware resources used by CUBLAS. Parameters ---------- handle : void_p CUBLAS context.", "response": "def cublasDestroy(handle):\n \"\"\"\n Release CUBLAS resources.\n\n Releases hardware resources used by CUBLAS.\n\n Parameters\n ----------\n handle : void_p\n CUBLAS context.\n \n \"\"\"\n\n status = _libcublas.cublasDestroy_v2(ctypes.c_void_p(handle))\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets CUBLAS version. Returns version number of installed CUBLAS libraries. Parameters ---------- handle : void_p CUBLAS context. Returns ------- version : int CUBLAS version.", "response": "def cublasGetVersion(handle):\n \"\"\"\n Get CUBLAS version.\n\n Returns version number of installed CUBLAS libraries.\n\n Parameters\n ----------\n handle : void_p\n CUBLAS context.\n\n Returns\n -------\n version : int\n CUBLAS version.\n\n \"\"\"\n \n version = ctypes.c_int()\n status = _libcublas.cublasGetVersion_v2(handle, ctypes.byref(version))\n cublasCheckStatus(status)\n return version.value"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting current CUBLAS library stream.", "response": "def cublasSetStream(handle, id):\n \"\"\"\n Set current CUBLAS library stream.\n \n Parameters\n ----------\n handle : id\n CUBLAS context.\n id : int\n Stream ID.\n\n \"\"\"\n\n status = _libcublas.cublasSetStream_v2(handle, id)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets current CUBLAS library stream.", "response": "def cublasGetStream(handle):\n \"\"\"\n Set current CUBLAS library stream.\n\n Parameters\n ----------\n handle : void_p\n CUBLAS context.\n \n Returns\n -------\n id : int\n Stream ID.\n \n \"\"\"\n \n id = ctypes.c_int()\n status = _libcublas.cublasGetStream_v2(handle, ctypes.byref(id))\n cublasCheckStatus(status)\n return id.value"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasCgbmv(handle, trans, m, n, kl, ku, alpha, A, lda,\n x, incx, beta, y, incy):\n \"\"\"\n Matrix-vector product for complex general banded matrix.\n\n \"\"\"\n\n status = _libcublas.cublasCgbmv_v2(handle,\n trans, m, n, kl, ku,\n ctypes.byref(cuda.cuFloatComplex(alpha.real,\n alpha.imag)),\n int(A), lda, int(x), incx,\n ctypes.byref(cuda.cuFloatComplex(beta.real,\n beta.imag)),\n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for complex general banded matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cublasDgemv(handle, trans, m, n, alpha, A, lda, x, incx, beta, y, incy):\n\n status = _libcublas.cublasDgemv_v2(handle,\n _CUBLAS_OP[trans], m, n,\n ctypes.byref(ctypes.c_double(alpha)),\n int(A), lda, int(x), incx,\n ctypes.byref(ctypes.c_double(beta)),\n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real general matrix."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cublasCgemv(handle, trans, m, n, alpha, A, lda, x, incx, beta, y, incy):\n\n status = _libcublas.cublasCgemv_v2(handle,\n _CUBLAS_OP[trans], m, n,\n ctypes.byref(cuda.cuFloatComplex(alpha.real,\n alpha.imag)),\n int(A), lda, int(x), incx,\n ctypes.byref(cuda.cuFloatComplex(beta.real,\n beta.imag)),\n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for complex general matrix."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrank operation on real general matrix.", "response": "def cublasSger(handle, m, n, alpha, x, incx, y, incy, A, lda):\n \"\"\"\n Rank-1 operation on real general matrix.\n\n \"\"\"\n \n status = _libcublas.cublasSger_v2(handle,\n m, n,\n ctypes.byref(ctypes.c_float(alpha)),\n int(x), incx,\n int(y), incy, int(A), lda)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nranking operation on real general matrix.", "response": "def cublasDger(handle, m, n, alpha, x, incx, y, incy, A, lda):\n \"\"\"\n Rank-1 operation on real general matrix.\n\n \"\"\"\n \n status = _libcublas.cublasDger_v2(handle,\n m, n,\n ctypes.byref(ctypes.c_double(alpha)),\n int(x), incx,\n int(y), incy, int(A), lda)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrank operation on complex general matrix.", "response": "def cublasCgeru(handle, m, n, alpha, x, incx, y, incy, A, lda):\n \"\"\"\n Rank-1 operation on complex general matrix.\n\n \"\"\"\n\n status = _libcublas.cublasCgeru_v2(handle,\n m, n, ctypes.byref(cuda.cuFloatComplex(alpha.real,\n alpha.imag)),\n int(x), incx, int(y), incy, int(A), lda)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cublasZgerc(handle, m, n, alpha, x, incx, y, incy, A, lda):\n\n status = _libcublas.cublasZgerc_v2(handle,\n m, n, ctypes.byref(cuda.cuDoubleComplex(alpha.real,\n alpha.imag)),\n int(x), incx, int(y), incy, int(A), lda)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for complex general matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasSsbmv(handle, uplo, n, k, alpha, A, lda, x, incx, beta, y, incy):\n\n status = _libcublas.cublasSsbmv_v2(handle,\n _CUBLAS_FILL_MODE[uplo], n, k,\n ctypes.byref(ctypes.c_float(alpha)),\n int(A), lda, int(x), incx,\n ctypes.byref(ctypes.c_float(beta)),\n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real symmetric - banded matrix."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cublasDspmv(handle, uplo, n, alpha, AP, x, incx, beta, y, incy):\n\n status = _libcublas.cublasDspmv_v2(handle,\n _CUBLAS_FILL_MODE[uplo], \n n,\n ctypes.byref(ctypes.c_double(alpha)),\n ctypes.byref(ctypes.c_double(AP)),\n int(x),\n incx,\n ctypes.byref(ctypes.c_double(beta)),\n int(y),\n incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real symmetric - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cublasSspr(handle, uplo, n, alpha, x, incx, AP):\n \n status = _libcublas.cublasSspr_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_float(alpha)), \n int(x), incx, int(AP)) \n cublasCheckStatus(status)", "response": "Solve symmetric - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nranks operation on real symmetric-packed matrix.", "response": "def cublasDspr(handle, uplo, n, alpha, x, incx, AP):\n \"\"\"\n Rank-1 operation on real symmetric-packed matrix.\n\n \"\"\"\n\n status = _libcublas.cublasDspr_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_double(alpha)), \n int(x), incx, int(AP)) \n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cublasSspr2(handle, uplo, n, alpha, x, incx, y, incy, AP):\n\n status = _libcublas.cublasSspr2_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_float(alpha)),\n int(x), incx, int(y), incy, int(AP)) \n \n cublasCheckStatus(status)", "response": "Matrix - matrix product for real symmetric - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasDspr2(handle, uplo, n, alpha, x, incx, y, incy, AP):\n\n status = _libcublas.cublasDspr2_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_double(alpha)), \n int(x), incx, int(y), incy, int(AP))\n cublasCheckStatus(status)", "response": "Matrix - matrix product for real symmetric - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cublasSsymv(handle, uplo, n, alpha, A, lda, x, incx, beta, y, incy):\n \n status = _libcublas.cublasSsymv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_float(alpha)),\n int(A), lda, int(x), incx,\n ctypes.byref(ctypes.c_float(beta)), \n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cublasDsymv(handle, uplo, n, alpha, A, lda, x, incx, beta, y, incy):\n\n status = _libcublas.cublasDsymv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_double(alpha)), \n int(A), lda, int(x), incx, \n ctypes.byref(ctypes.c_double(beta)), \n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cublasCsymv(handle, uplo, n, alpha, A, lda, x, incx, beta, y, incy):\n\n status = _libcublas.cublasCsymv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(cuda.cuFloatComplex(alpha.real,\n alpha.imag)), \n int(A), lda, int(x), incx, \n ctypes.byref(cuda.cuFloatComplex(beta.real,\n beta.imag)), \n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for complex symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasZsymv(handle, uplo, n, alpha, A, lda, x, incx, beta, y, incy):\n\n status = _libcublas.cublasZsymv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(cuda.cuDoubleComplex(alpha.real,\n alpha.imag)), \n int(A), lda, int(x), incx, \n ctypes.byref(cuda.cuDoubleComplex(beta.real,\n beta.imag)), \n int(y), incy)\n cublasCheckStatus(status)", "response": "Matrix - vector product for complex symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasSsyr(handle, uplo, n, alpha, x, incx, A, lda): \n \n status = _libcublas.cublasSsyr_v2(handle,\n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_float(alpha)),\n int(x), incx, int(A), lda)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nranking operation on real symmetric matrix.", "response": "def cublasDsyr(handle, uplo, n, alpha, x, incx, A, lda):\n \"\"\"\n Rank-1 operation on real symmetric matrix.\n\n \"\"\"\n\n status = _libcublas.cublasDsyr_v2(handle,\n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_double(alpha)), \n int(x), incx, int(A), lda)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cublasCsyr(handle, uplo, n, alpha, x, incx, A, lda):\n\n status = _libcublas.cublasCsyr_v2(handle,\n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(cuda.cuFloatComplex(alpha.real,\n alpha.imag)),\n int(x), incx, int(A), lda)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nranking operation on complex symmetric matrix.", "response": "def cublasZsyr(handle, uplo, n, alpha, x, incx, A, lda):\n \"\"\"\n Rank-1 operation on complex symmetric matrix.\n\n \"\"\"\n\n status = _libcublas.cublasZsyr_v2(handle,\n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(cuda.cuDoubleComplex(alpha.real,\n alpha.imag)),\n int(x), incx, int(A), lda)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cublasSsyr2(handle, uplo, n, alpha, x, incx, y, incy, A, lda):\n\n status = _libcublas.cublasSsyr2_v2(handle,\n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_float(alpha)),\n int(x), incx, int(y), incy,\n int(A), lda)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for real symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nranks operation on real symmetric matrix.", "response": "def cublasDsyr2(handle, uplo, n, alpha, x, incx, y, incy, A, lda):\n \"\"\"\n Rank-2 operation on real symmetric matrix.\n\n \"\"\"\n\n status = _libcublas.cublasDsyr2_v2(handle, \n _CUBLAS_FILL_MODE[uplo], n, \n ctypes.byref(ctypes.c_double(alpha)), \n int(x), incx, int(y), incy, \n int(A), lda) \n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasStpmv(handle, uplo, trans, diag, n, AP, x, incx):\n \n status = _libcublas.cublasStpmv_v2(handle,\n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n n, int(AP), int(x), incx)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real triangular - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasDtpmv(handle, uplo, trans, diag, n, AP, x, incx):\n\n status = _libcublas.cublasDtpmv_v2(handle,\n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n n, int(AP), int(x), incx)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real triangular - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsolve real triangular - packed system with one right - hand side.", "response": "def cublasStpsv(handle, uplo, trans, diag, n, AP, x, incx):\n \"\"\"\n Solve real triangular-packed system with one right-hand side.\n\n \"\"\"\n \n status = _libcublas.cublasStpsv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n n, int(AP), int(x), incx)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsolve real triangular-packed system with one right-hand side.", "response": "def cublasDtpsv(handle, uplo, trans, diag, n, AP, x, incx):\n \"\"\"\n Solve real triangular-packed system with one right-hand side.\n\n \"\"\"\n\n status = _libcublas.cublasDtpsv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n n, int(AP), int(x), incx)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cublasCtpsv(handle, uplo, trans, diag, n, AP, x, incx):\n \n status = _libcublas.cublasCtpsv_v2(handle,\n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n n, int(AP), int(x), incx)\n cublasCheckStatus(status)", "response": "Matrix - vector product for complex triangular - packed system."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cublasZtpsv(handle, uplo, trans, diag, n, AP, x, incx):\n \n status = _libcublas.cublasZtpsv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n n, int(AP), int(x), incx)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real triangular - packed system."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cublasDtrmv(handle, uplo, trans, diag, n, A, lda, x, inx):\n\n status = _libcublas.cublasDtrmv_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n n, int(A), lda, int(x), inx)\n cublasCheckStatus(status)", "response": "Matrix - vector product for real triangular matrix."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cublasCher(handle, uplo, n, alpha, x, incx, A, lda):\n\n status = _libcublas.cublasCher_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n n, alpha, int(x), incx, int(A), lda)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for real and imaginary part."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cublasZher(handle, uplo, n, alpha, x, incx, A, lda):\n \n status = _libcublas.cublasZher_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n n, alpha, int(x), incx, int(A), lda)\n cublasCheckStatus(status)", "response": "Matrix - vector product for Hermitian matrix."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cublasChpr(handle, uplo, n, alpha, x, incx, AP):\n \n status = _libcublas.cublasChpr_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n n, ctypes.byref(ctypes.c_float(alpha)),\n int(x), incx, int(AP))\n cublasCheckStatus(status)", "response": "Matrix - rank - 1 operation on Hermitian - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrank operation on Hermitian-packed matrix.", "response": "def cublasZhpr(handle, uplo, n, alpha, x, incx, AP):\n \"\"\"\n Rank-1 operation on Hermitian-packed matrix.\n\n \"\"\"\n \n status = _libcublas.cublasZhpr_v2(handle,\n _CUBLAS_FILL_MODE[uplo], \n n, ctypes.byref(ctypes.c_double(alpha)),\n int(x), incx, int(AP))\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrank operation on Hermitian-packed matrix.", "response": "def cublasChpr2(handle, uplo, n, alpha, x, inx, y, incy, AP):\n \"\"\"\n Rank-2 operation on Hermitian-packed matrix.\n \n \"\"\"\n\n status = _libcublas.cublasChpr2_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n n, ctypes.byref(cuda.cuFloatComplex(alpha.real,\n alpha.imag)),\n int(x), incx, int(y), incy, int(AP))\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cublasZhpr2(handle, uplo, n, alpha, x, inx, y, incy, AP):\n \n status = _libcublas.cublasZhpr2_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n n, ctypes.byref(cuda.cuDoubleComplex(alpha.real, \n alpha.imag)),\n int(x), incx, int(y), incy, int(AP))\n cublasCheckStatus(status)", "response": "Matrix - matrix product for Hermitian - packed matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cublasZgemm(handle, transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc):\n\n status = _libcublas.cublasZgemm_v2(handle,\n _CUBLAS_OP[transa],\n _CUBLAS_OP[transb], m, n, k, \n ctypes.byref(cuda.cuDoubleComplex(alpha.real,\n alpha.imag)),\n int(A), lda, int(B), ldb,\n ctypes.byref(cuda.cuDoubleComplex(beta.real,\n beta.imag)),\n int(C), ldc)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for complex general matrix."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cublasCsymm(handle, side, uplo, m, n, alpha, A, lda, B, ldb, beta, C, ldc):\n \n status = _libcublas.cublasCsymm_v2(handle, \n _CUBLAS_SIDE_MODE[side], \n _CUBLAS_FILL_MODE[uplo], \n m, n, ctypes.byref(cuda.cuFloatComplex(alpha.real, \n alpha.imag)),\n int(A), lda, int(B), ldb,\n ctypes.byref(cuda.cuFloatComplex(beta.real, \n beta.imag)),\n int(C), ldc)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for complex symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cublasDsyrk(handle, uplo, trans, n, k, alpha, A, lda, beta, C, ldc):\n \n status = _libcublas.cublasDsyrk_v2(handle,\n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n n, k, ctypes.byref(cuda.cuFloatComplex(alpha.real, \n alpha.imag)),\n int(A), lda, \n ctypes.byref(cuda.cuFloatComplex(beta.real,\n beta.imag)),\n int(C), ldc)\n cublasCheckStatus(status)", "response": "Matrix - k operation on real symmetric matrix."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cublasStrmm(handle, side, uplo, trans, diag, m, n, alpha, A, lda, B, ldb, C, ldc):\n \n status = _libcublas.cublasStrmm_v2(handle,\n _CUBLAS_SIDE_MODE[side], \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n m, n, ctypes.byref(ctypes.c_float(alpha)),\n int(A), lda, int(B), ldb, int(C), ldc)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for real triangular matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cublasZtrmm(handle, side, uplo, trans, diag, m, n, alpha, A, lda, B, ldb, C, ldc):\n \n status = _libcublas.cublasZtrmm_v2(handle, \n _CUBLAS_SIDE_MODE[side], \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n m, n, ctypes.byref(cuda.cuDoubleComplex(alpha.real, \n alpha.imag)),\n int(A), lda, int(B), ldb, int(C), ldc)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for complex triangular matrix."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsolves a real triangular system with multiple right - hand sides.", "response": "def cublasStrsm(handle, side, uplo, trans, diag, m, n, alpha, A, lda, B, ldb):\n \"\"\"\n Solve a real triangular system with multiple right-hand sides.\n\n \"\"\"\n \n status = _libcublas.cublasStrsm_v2(handle, \n _CUBLAS_SIDE_MODE[side], \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n m, n, ctypes.byref(ctypes.c_float(alpha)),\n int(A), lda, int(B), ldb)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsolving a real triangular system with multiple right-hand sides.", "response": "def cublasDtrsm(handle, side, uplo, trans, diag, m, n, alpha, A, lda, B, ldb):\n \"\"\"\n Solve a real triangular system with multiple right-hand sides.\n\n \"\"\"\n \n status = _libcublas.cublasDtrsm_v2(handle, \n _CUBLAS_SIDE_MODE[side], \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n m, n, ctypes.byref(ctypes.c_double(alpha)),\n int(A), lda, int(B), ldb)\n cublasCheckStatus(status)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cublasZtrsm(handle, side, uplo, transa, diag, m, n, alpha, A, lda, B, ldb):\n \n status = _libcublas.cublasZtrsm_v2(handle, \n _CUBLAS_SIDE_MODE[side], \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n _CUBLAS_DIAG[diag], \n m, n, ctypes.byref(cuda.cuDoubleComplex(alpha.real, \n alpha.imag)),\n int(A), lda, int(B), ldb)\n cublasCheckStatus(status)", "response": "Matrix - matrix product for complex triangular system."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cublasZherk(handle, uplo, trans, n, k, alpha, A, lda, beta, C, ldc):\n \n status = _libcublas.cublasZherk_v2(handle,\n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans],\n n, k, ctypes.byref(ctypes.c_double(alpha)),\n int(A), lda, \n ctypes.byref(ctypes.c_double(beta)),\n int(C), ldc)\n cublasCheckStatus(status)", "response": "Matrix - rank - k operation on Hermitian matrix."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cublasCher2k(handle, uplo, trans, n, k, alpha, A, lda, B, ldb, beta, C, ldc):\n \n status = _libcublas.cublasCher2k_v2(handle, \n _CUBLAS_FILL_MODE[uplo], \n _CUBLAS_OP[trans], \n n, k, ctypes.byref(cuda.cuFloatComplex(alpha.real, \n alpha.imag)),\n int(A), lda, int(B), ldb, \n ctypes.byref(cuda.cuFloatComplex(beta.real,\n beta.imag)),\n int(C), ldc)\n cublasCheckStatus(status)", "response": "Matrix - rank 2k operation on Hermitian matrix."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_EROS_lc(filename='lm0010n22323.time'):\n\n module_path = dirname(__file__)\n file_path = join(module_path, 'lightcurves', filename)\n\n data = np.loadtxt(file_path)\n date = data[:, 0]\n mag = data[:, 1]\n err = data[:, 2]\n\n return date, mag, err", "response": "Reads an EROS light curve and returns its data."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_rf_model():\n\n import gzip\n try:\n import cPickle as pickle\n except:\n import pickle\n\n module_path = dirname(__file__)\n file_path = join(module_path, 'models/rf.model.sub.github.gz')\n\n # For Python 3.\n if sys.version_info.major >= 3:\n clf = pickle.load(gzip.open(file_path, 'rb'), encoding='latin1')\n # For Python 2.\n else:\n clf = pickle.load(gzip.open(file_path, 'rb'))\n\n return clf", "response": "Loads the UPSILoN random forests classifier."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sample_dropout_mask(x, dropout_probability=.5, columns=None, stream=None, target=None,\n dropout_mask=None, dropout_prob_array=None):\n \"\"\" Samples a dropout mask and applies it in place\"\"\"\n\n assert x.flags.c_contiguous\n\n if columns is not None:\n assert len(columns) == 2\n x_tmp = x\n x = extract_columns(x, columns[0], columns[1])\n\n shape = x.shape\n\n if dropout_prob_array is None:\n dropout_prob_array = gpuarray.empty(shape, x.dtype, allocator=memory_pool.allocate)\n sampler.fill_uniform(dropout_prob_array, stream)\n\n if dropout_mask is None:\n dropout_mask = gpuarray.empty(shape, np.int8, allocator=memory_pool.allocate)\n\n if target is None: target = x\n \n all_kernels['sample_dropout_mask'](\n x, target, dropout_mask, dropout_prob_array,\n np.float32(dropout_probability))\n\n if columns is not None:\n insert_columns(x, x_tmp, columns[0])\n\n return dropout_mask", "response": "Samples a dropout mask and applies it in place"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nloading but don t evaluate a GCL expression from a string.", "response": "def reads(s, filename=None, loader=None, implicit_tuple=True, allow_errors=False):\n \"\"\"Load but don't evaluate a GCL expression from a string.\"\"\"\n return ast.reads(s,\n filename=filename or '',\n loader=loader or default_loader,\n implicit_tuple=implicit_tuple,\n allow_errors=allow_errors)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nload but don t evaluate a GCL expression from a file.", "response": "def read(filename, loader=None, implicit_tuple=True, allow_errors=False):\n \"\"\"Load but don't evaluate a GCL expression from a file.\"\"\"\n with open(filename, 'r') as f:\n return reads(f.read(),\n filename=filename,\n loader=loader,\n implicit_tuple=implicit_tuple,\n allow_errors=allow_errors)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef loads(s, filename=None, loader=None, implicit_tuple=True, env={}, schema=None):\n ast = reads(s, filename=filename, loader=loader, implicit_tuple=implicit_tuple)\n if not isinstance(env, framework.Environment):\n # For backwards compatibility we accept an Environment object. Otherwise assume it's a dict\n # whose bindings will add/overwrite the default bindings.\n env = framework.Environment(dict(_default_bindings, **env))\n obj = framework.eval(ast, env)\n return mod_schema.validate(obj, schema)", "response": "Load and evaluate a GCL expression from a string."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading and evaluate a GCL expression from a file.", "response": "def load(filename, loader=None, implicit_tuple=True, env={}, schema=None):\n \"\"\"Load and evaluate a GCL expression from a file.\"\"\"\n with open(filename, 'r') as f:\n return loads(f.read(),\n filename=filename,\n loader=loader,\n implicit_tuple=implicit_tuple,\n env=env,\n schema=schema)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nincreases linearly then stays flat", "response": "def linear_scheduler_up(init_value, target_value, duration):\n \"\"\" Increases linearly and then stays flat \"\"\"\n\n value = init_value\n t = 0\n while True:\n yield value\n t += 1\n if t < duration:\n value = init_value + t * (target_value - init_value) / duration\n else:\n value = target_value"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nincreases linearly to target_value, stays at target_value until t_decrease and then decreases linearly", "response": "def linear_scheduler_up_down(init_value, target_value, final_value,\n duration_up, t_decrease, duration_down):\n \"\"\" Increases linearly to target_value, stays at target_value until\n t_decrease and then decreases linearly\n \"\"\"\n\n value = init_value\n t = 0\n\n while True:\n yield value\n t += 1\n if t < duration_up:\n value = init_value + t * (target_value - init_value) / \\\n float(duration_up)\n elif t > t_decrease:\n value = target_value - (t - t_decrease) * \\\n (target_value - final_value) / \\\n float(duration_down)\n else:\n value = target_value"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nload a YAML configuration from a string or file - like object.", "response": "def load(stream, overrides=None, **kwargs):\n \"\"\"\n Loads a YAML configuration from a string or file-like object.\n\n Parameters\n ----------\n stream : str or object\n Either a string containing valid YAML or a file-like object\n supporting the .read() interface.\n overrides : dict, optional\n A dictionary containing overrides to apply. The location of\n the override is specified in the key as a dot-delimited path\n to the desired parameter, e.g. \"model.corruptor.corruption_level\".\n\n Returns\n -------\n graph : dict or object\n The dictionary or object (if the top-level element specified an\n Python object to instantiate).\n\n Notes\n -----\n Other keyword arguments are passed on to `yaml.load`.\n \"\"\"\n\n global is_initialized\n if not is_initialized:\n initialize()\n\n if isinstance(stream, basestring):\n string = stream\n else:\n string = '\\n'.join(stream.readlines())\n\n # processed_string = preprocess(string)\n\n proxy_graph = yaml.load(string, **kwargs)\n\n from . import init\n init_dict = proxy_graph.get('init', {})\n init(**init_dict)\n \n if overrides is not None:\n handle_overrides(proxy_graph, overrides)\n return instantiate_all(proxy_graph)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nload a YAML file into a dictionary.", "response": "def load_path(path, overrides=None, **kwargs):\n \"\"\"\n Convenience function for loading a YAML configuration from a file.\n\n Parameters\n ----------\n path : str\n The path to the file to load on disk.\n overrides : dict, optional\n A dictionary containing overrides to apply. The location of\n the override is specified in the key as a dot-delimited path\n to the desired parameter, e.g. \"model.corruptor.corruption_level\".\n\n Returns\n -------\n graph : dict or object\n The dictionary or object (if the top-level element specified an\n Python object to instantiate).\n\n Notes\n -----\n Other keyword arguments are passed on to `yaml.load`.\n \"\"\"\n f = open(path, 'r')\n content = ''.join(f.readlines())\n f.close()\n\n if not isinstance(content, str):\n raise AssertionError(\"Expected content to be of type str but it is \"+str(type(content)))\n\n return load(content, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nhandle any overrides for this model configuration.", "response": "def handle_overrides(graph, overrides):\n \"\"\"\n Handle any overrides for this model configuration.\n\n Parameters\n ----------\n graph : dict or object\n A dictionary (or an ObjectProxy) containing the object graph\n loaded from a YAML file.\n overrides : dict\n A dictionary containing overrides to apply. The location of\n the override is specified in the key as a dot-delimited path\n to the desired parameter, e.g. \"model.corruptor.corruption_level\".\n \"\"\"\n for key in overrides:\n levels = key.split('.')\n part = graph\n for lvl in levels[:-1]:\n try:\n part = part[lvl]\n except KeyError:\n raise KeyError(\"'%s' override failed at '%s'\", (key, lvl))\n try:\n part[levels[-1]] = overrides[key]\n except KeyError:\n raise KeyError(\"'%s' override failed at '%s'\", (key, levels[-1]))"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninitializes the configuration system by installing YAML handlers.", "response": "def initialize():\n \"\"\"\n Initialize the configuration system by installing YAML handlers.\n Automatically done on first call to load() specified in this file.\n \"\"\"\n global is_initialized\n # Add the custom multi-constructor\n yaml.add_multi_constructor('!obj:', multi_constructor)\n yaml.add_multi_constructor('!pkl:', multi_constructor_pkl)\n yaml.add_multi_constructor('!import:', multi_constructor_import)\n yaml.add_multi_constructor('!include:', multi_constructor_include)\n\n def import_constructor(loader, node):\n value = loader.construct_scalar(node)\n return try_to_import(value)\n\n yaml.add_constructor('!import', import_constructor)\n yaml.add_implicit_resolver(\n '!import',\n re.compile(r'(?:[a-zA-Z_][\\w_]+\\.)+[a-zA-Z_][\\w_]+')\n )\n is_initialized = True"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef feed_forward(self, input_data, prediction=False):\n\n if input_data.shape[1] != self.W.shape[0]:\n raise ValueError('Number of outputs from previous layer (%d) '\n 'does not match number of inputs to this layer (%d)' %\n (input_data.shape[1], self.W.shape[0]))\n\n activations = linalg.dot(input_data, self.W)\n activations = add_vec_to_mat(activations, self.b, inplace=True)\n\n return activations", "response": "Propagate forward through the layer."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngive abscissas x (which need not be equally spaced) and ordinates y, and given a desired oversampling factor ofac (a typical value being 4 or larger). this routine creates an array wk1 with a sequence of nout increasing frequencies (not angular frequencies) up to hifac times the \"average\" Nyquist frequency, and creates an array wk2 with the values of the Lomb normalized periodogram at those frequencies. The arrays x and y are not altered. This routine also returns jmax such that wk2(jmax) is the maximum element in wk2, and prob, an estimate of the significance of that maximum against the hypothesis of random noise. A small value of prob indicates that a significant periodic signal is present. Reference: Press, W. H. & Rybicki, G. B. 1989 ApJ vol. 338, p. 277-280. Fast algorithm for spectral analysis of unevenly sampled data (1989ApJ...338..277P) Arguments: X : Abscissas array, (e.g. an array of times). Y : Ordinates array, (e.g. corresponding counts). Ofac : Oversampling factor. Hifac : Hifac * \"average\" Nyquist frequency = highest frequency for which values of the Lomb normalized periodogram will be calculated. n_threads : number of threads to use. Returns: Wk1 : An array of Lomb periodogram frequencies. Wk2 : An array of corresponding values of the Lomb periodogram. Nout : Wk1 & Wk2 dimensions (number of calculated frequencies) Jmax : The array index corresponding to the MAX( Wk2 ). Prob : False Alarm Probability of the largest Periodogram value MACC : Number of interpolation points per 1/4 cycle of highest frequency History: 02/23/2009, v1.0, MF Translation of IDL code (orig. Numerical recipies)", "response": "def fasper(x, y, ofac, hifac, n_threads, MACC=4):\n \"\"\"\n Given abscissas x (which need not be equally spaced) and ordinates\n y, and given a desired oversampling factor ofac (a typical value\n being 4 or larger). this routine creates an array wk1 with a\n sequence of nout increasing frequencies (not angular frequencies)\n up to hifac times the \"average\" Nyquist frequency, and creates\n an array wk2 with the values of the Lomb normalized periodogram at\n those frequencies. The arrays x and y are not altered. This\n routine also returns jmax such that wk2(jmax) is the maximum\n element in wk2, and prob, an estimate of the significance of that\n maximum against the hypothesis of random noise. A small value of prob\n indicates that a significant periodic signal is present.\n \n Reference: \n Press, W. H. & Rybicki, G. B. 1989\n ApJ vol. 338, p. 277-280.\n Fast algorithm for spectral analysis of unevenly sampled data\n (1989ApJ...338..277P)\n\n Arguments:\n X : Abscissas array, (e.g. an array of times).\n Y : Ordinates array, (e.g. corresponding counts).\n Ofac : Oversampling factor.\n Hifac : Hifac * \"average\" Nyquist frequency = highest frequency\n for which values of the Lomb normalized periodogram will\n be calculated.\n n_threads : number of threads to use.\n \n Returns:\n Wk1 : An array of Lomb periodogram frequencies.\n Wk2 : An array of corresponding values of the Lomb periodogram.\n Nout : Wk1 & Wk2 dimensions (number of calculated frequencies)\n Jmax : The array index corresponding to the MAX( Wk2 ).\n Prob : False Alarm Probability of the largest Periodogram value\n MACC : Number of interpolation points per 1/4 cycle\n of highest frequency\n\n History:\n 02/23/2009, v1.0, MF\n Translation of IDL code (orig. Numerical recipies)\n \"\"\"\n #Check dimensions of input arrays\n n = long(len(x))\n if n != len(y):\n print('Incompatible arrays.')\n return\n \n #print x, y, hifac, ofac\n \n nout = int(0.5*ofac*hifac*n)\n nfreqt = long(ofac*hifac*n*MACC) #Size the FFT as next power\n nfreq = 64 # of 2 above nfreqt.\n\n while nfreq < nfreqt:\n nfreq = 2*nfreq\n\n ndim = long(2*nfreq)\n \n #Compute the mean, variance\n ave = y.mean()\n ##sample variance because the divisor is N-1\n var = ((y - y.mean())**2).sum()/(len(y) - 1) \n # and range of the data.\n xmin = x.min()\n xmax = x.max()\n xdif = xmax - xmin\n\n #extrapolate the data into the workspaces\n if is_pyfftw:\n wk1 = pyfftw.n_byte_align_empty(int(ndim), 16, 'complex') * 0.\n wk2 = pyfftw.n_byte_align_empty(int(ndim), 16, 'complex') * 0.\n else:\n wk1 = zeros(ndim, dtype='complex')\n wk2 = zeros(ndim, dtype='complex')\n\n fac = ndim/(xdif*ofac)\n fndim = ndim\n ck = ((x - xmin)*fac) % fndim\n ckk = (2.0*ck) % fndim\n \n for j in range(0, n):\n __spread__(y[j] - ave, wk1, ndim, ck[j], MACC)\n __spread__(1.0, wk2, ndim, ckk[j], MACC)\n\n #Take the Fast Fourier Transforms.\n if is_pyfftw:\n fft_wk1 = pyfftw.builders.ifft(wk1, planner_effort='FFTW_ESTIMATE',\n threads=n_threads)\n wk1 = fft_wk1() * len(wk1)\n fft_wk2 = pyfftw.builders.ifft(wk2, planner_effort='FFTW_ESTIMATE',\n threads=n_threads)\n wk2 = fft_wk2() * len(wk2)\n else:\n wk1 = ifft(wk1)*len(wk1)\n wk2 = ifft(wk2)*len(wk1)\n\n wk1 = wk1[1:nout + 1]\n wk2 = wk2[1:nout + 1]\n rwk1 = wk1.real\n iwk1 = wk1.imag\n rwk2 = wk2.real\n iwk2 = wk2.imag\n \n df = 1.0/(xdif*ofac)\n \n #Compute the Lomb value for each frequency\n hypo2 = 2.0*abs(wk2)\n hc2wt = rwk2/hypo2\n hs2wt = iwk2/hypo2\n\n cwt = sqrt(0.5 + hc2wt)\n swt = sign(hs2wt)*(sqrt(0.5 - hc2wt))\n den = 0.5*n + hc2wt*rwk2 + hs2wt*iwk2\n cterm = (cwt*rwk1 + swt*iwk1)**2./den\n sterm = (cwt*iwk1 - swt*rwk1)**2./(n - den)\n\n wk1 = df*(arange(nout, dtype='float') + 1.)\n wk2 = (cterm + sterm)/(2.0*var)\n pmax = wk2.max()\n jmax = wk2.argmax()\n\n #Significance estimation\n #expy = exp(-wk2) \n #effm = 2.0*(nout)/ofac \n #sig = effm*expy\n #ind = (sig > 0.01).nonzero()\n #sig[ind] = 1.0-(1.0-expy[ind])**effm\n\n #Estimate significance of largest peak value\n expy = exp(-pmax) \n effm = 2.0*(nout)/ofac\n prob = effm*expy\n\n if prob > 0.01: \n prob = 1.0 - (1.0 - expy)**effm\n\n return wk1, wk2, nout, jmax, prob"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_no_validate(self, key):\n x, env = self.get_thunk_env(key)\n\n # Check if this is a Thunk that needs to be lazily evaluated before we\n # return it.\n if isinstance(x, framework.Thunk):\n x = framework.eval(x, env)\n\n return x", "response": "Return an item without validating the schema."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef env(self, current_scope):\n return self.__env_cache.get(\n current_scope.ident,\n framework.Environment, current_scope,\n names=self.keys(),\n parent=framework.Environment({'self': current_scope}, parent=self.__parent_env))", "response": "Return an environment that will look up in current_scope for keys in\n this tuple and the parent env otherwise."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the thunk and environment for validating it in for the given key.", "response": "def get_thunk_env(self, k):\n \"\"\"Return the thunk AND environment for validating it in for the given key.\n\n There might be different envs in case the thunk comes from a different (composed) tuple. If the thunk needs its\n environment bound on retrieval, that will be done here.\n \"\"\"\n if k not in self.__items:\n raise exceptions.EvaluationError('Unknown key: %r in tuple %r' % (k, self))\n x = self.__items[k]\n\n env = self.env(self)\n\n # Bind this to the tuple's parent environment\n if isinstance(x, framework.BindableThunk):\n return x.bind(self.__parent_env), env\n return x, env"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef attach_schema(self, schem):\n self.tuple_schema = schema.AndSchema.make(self.tuple_schema, schem)", "response": "Attach a tuple schema to this object."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the evaluated schema expression from a subkey.", "response": "def get_schema_spec(self, key):\n \"\"\"Return the evaluated schema expression from a subkey.\"\"\"\n member_node = self._ast_node.member.get(key, None)\n if not member_node:\n return schema.AnySchema()\n\n s = framework.eval(member_node.member_schema, self.env(self))\n if not isinstance(s, schema.Schema):\n raise ValueError('Node %r with schema node %r should evaluate to Schema, got %r' % (member_node, member_node.member_schema, s))\n return s"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_required_fields(self):\n return [m.name for m in self._ast_node.members if m.member_schema.required]", "response": "Return the names of fields that are required according to the schema."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the AST node for the given member from the first tuple that serves it.", "response": "def get_member_node(self, key):\n \"\"\"Return the AST node for the given member, from the first tuple that serves it.\"\"\"\n for tup, _ in self.lookups:\n if key in tup:\n return tup.get_member_node(key)\n raise RuntimeError('Key not found in composite tuple: %r' % key)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef exportable_keys(self):\n keys = collections.defaultdict(list)\n for tup in self._tuples:\n for key, private in tup._keys_and_privacy().items():\n keys[key].append(private)\n return [k for k, ps in keys.items() if not any(ps)]", "response": "Return a list of keys that are exportable from this tuple."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef resolve(self, current_file, rel_path):\n search_path = [path.dirname(current_file)] + self.search_path\n\n target_path = None\n for search in search_path:\n if self.exists(path.join(search, rel_path)):\n target_path = path.normpath(path.join(search, rel_path))\n break\n\n if not target_path:\n raise exceptions.EvaluationError('No such file: %r, searched %s' %\n (rel_path, ':'.join(search_path)))\n\n return target_path, path.abspath(target_path)", "response": "Search the filesystem for a file with the given relative path."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsearch the filesystem for a fake file.", "response": "def resolve(self, current_file, rel_path):\n \"\"\"Search the filesystem.\"\"\"\n p = path.join(path.dirname(current_file), rel_path)\n if p not in self.file_dict:\n raise RuntimeError('No such fake file: %r' % p)\n return p, p"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef check_call_arguments(to_call, kwargs):\n if 'self' in kwargs.keys():\n raise TypeError(\"Your dictionary includes an entry for 'self', \"\n \"which is just asking for trouble\")\n\n orig_to_call = getattr(to_call, '__name__', str(to_call))\n if not isinstance(to_call, types.FunctionType):\n if hasattr(to_call, '__init__'):\n to_call = to_call.__init__\n elif hasattr(to_call, '__call__'):\n to_call = to_call.__call__\n\n args, varargs, keywords, defaults = inspect.getargspec(to_call)\n\n if any(not isinstance(arg, str) for arg in args):\n raise TypeError('%s uses argument unpacking, which is deprecated and '\n 'unsupported by this pylearn2' % orig_to_call)\n\n if varargs is not None:\n raise TypeError('%s has a variable length argument list, but '\n 'this is not supported by config resolution' %\n orig_to_call)\n\n if keywords is None:\n bad_keywords = [arg_name for arg_name in kwargs.keys()\n if arg_name not in args]\n\n if len(bad_keywords) > 0:\n bad = ', '.join(bad_keywords)\n args = [ arg for arg in args if arg != 'self' ]\n if len(args) == 0:\n matched_str = '(It does not support any keywords, actually)'\n else:\n matched = [ match(keyword, args) for keyword in bad_keywords ]\n matched_str = 'Did you mean %s?' % (', '.join(matched))\n raise TypeError('%s does not support the following '\n 'keywords: %s. %s' %\n (orig_to_call, bad, matched_str))\n\n if defaults is None:\n num_defaults = 0\n else:\n num_defaults = len(defaults)\n\n required = args[:len(args) - num_defaults]\n missing = [arg for arg in required if arg not in kwargs]\n\n if len(missing) > 0:\n #iff the im_self (or __self__) field is present, this is a\n # bound method, which has 'self' listed as an argument, but\n # which should not be supplied by kwargs\n is_bound = hasattr(to_call, 'im_self') or hasattr(to_call, '__self__')\n if len(missing) > 1 or missing[0] != 'self' or not is_bound:\n if 'self' in missing:\n missing.remove('self')\n missing = ', '.join([str(m) for m in missing])\n raise TypeError('%s did not get these expected '\n 'arguments: %s' % (orig_to_call, missing))", "response": "Checks the signature of a function or class or class to examine the arguments of a class or callable."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict(rf_model, features):\n\n import numpy as np\n\n from upsilon.extract_features.feature_set import get_feature_set\n feature_set = get_feature_set()\n\n # Grab only necessary features.\n cols = [feature for feature in features if feature in feature_set]\n cols = sorted(cols)\n filtered_features = []\n for i in range(len(cols)):\n filtered_features.append(features[cols[i]])\n filtered_features = np.array(filtered_features).reshape(1, -1)\n\n # Classify.\n classes = rf_model.classes_\n # Note that we're classifying a single source, so [0] need tobe added.\n probabilities = rf_model.predict_proba(filtered_features)[0]\n\n # Classification flag.\n flag = 0\n if features['period_SNR'] < 20. or is_period_alias(features['period']):\n flag = 1\n\n # Return class, probability, and flag.\n max_index = np.where(probabilities == np.max(probabilities))\n return classes[max_index][0], probabilities[max_index][0], flag", "response": "Predict the class of a single source."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fmt(str, args=None, env=None):\n # Normally, we'd just call str.format(**args), but we only want to evaluate\n # values from the tuple which are actually used in the string interpolation,\n # so we use proxy objects.\n\n # If no args are given, we're able to take the current environment.\n args = args or env\n proxies = {k: StringInterpolationProxy(args, k) for k in args.keys()}\n return str.format(**proxies)", "response": "Interpolate a string with the same\n name."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef compose_all(tups):\n from . import ast # I weep for humanity\n return functools.reduce(lambda x, y: x.compose(y), map(ast.make_tuple, tups), ast.make_tuple({}))", "response": "Compose all given tuples together."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if a given tuple has a key and the key is bound.", "response": "def has_key(tup, key):\n \"\"\"has(tuple, string) -> bool\n\n Return whether a given tuple has a key and the key is bound.\n \"\"\"\n if isinstance(tup, framework.TupleLike):\n return tup.is_bound(key)\n if isinstance(tup, dict):\n return key in tup\n if isinstance(tup, list):\n if not isinstance(key, int):\n raise ValueError('Key must be integer when checking list index')\n return key < len(tup)\n raise ValueError('Not a tuple-like object: %r' % tup)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nflatten a list of lists.", "response": "def flatten(list_of_lists):\n \"\"\"flatten([[A]]) -> [A]\n\n Flatten a list of lists.\n \"\"\"\n ret = []\n for lst in list_of_lists:\n if not isinstance(lst, list):\n raise ValueError('%r is not a list' % lst)\n ret.extend(lst)\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nextracting superdark data from DARKFILE or DRKCFILE.", "response": "def extract_dark(prihdr, scihdu):\n \"\"\"Extract superdark data from ``DARKFILE`` or ``DRKCFILE``.\n\n Parameters\n ----------\n prihdr : obj\n FITS primary header HDU.\n\n scihdu : obj\n Extension HDU of the science image.\n This is only used to extract subarray data.\n\n Returns\n -------\n dark : ndarray or `None`\n Superdark, if any. Subtract this to apply ``DARKCORR``.\n\n \"\"\"\n if prihdr.get('PCTECORR', 'OMIT') == 'COMPLETE':\n darkfile = prihdr.get('DRKCFILE', 'N/A')\n else:\n darkfile = prihdr.get('DARKFILE', 'N/A')\n\n if darkfile == 'N/A':\n return None\n\n darkfile = from_irafpath(darkfile)\n ampstring = prihdr['CCDAMP']\n\n # Calculate DARKTIME\n exptime = prihdr.get('EXPTIME', 0.0)\n flashdur = prihdr.get('FLASHDUR', 0.0)\n darktime = exptime + flashdur\n if exptime > 0: # Not BIAS\n darktime += 3.0\n\n with fits.open(darkfile) as hdudark:\n if ampstring == 'ABCD':\n dark = np.concatenate(\n (hdudark['sci', 1].data,\n hdudark['sci', 2].data[::-1, :]), axis=1)\n elif ampstring in ('A', 'B', 'AB'):\n dark = extract_ref(scihdu, hdudark['sci', 2])\n else:\n dark = extract_ref(scihdu, hdudark['sci', 1])\n\n dark = dark * darktime\n\n return dark"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nextract postflash data from FITS primary header HDU.", "response": "def extract_flash(prihdr, scihdu):\n \"\"\"Extract postflash data from ``FLSHFILE``.\n\n Parameters\n ----------\n prihdr : obj\n FITS primary header HDU.\n\n scihdu : obj\n Extension HDU of the science image.\n This is only used to extract subarray data.\n\n Returns\n -------\n flash : ndarray or `None`\n Postflash, if any. Subtract this to apply ``FLSHCORR``.\n\n \"\"\"\n flshfile = prihdr.get('FLSHFILE', 'N/A')\n flashsta = prihdr.get('FLASHSTA', 'N/A')\n flashdur = prihdr.get('FLASHDUR', 0.0)\n\n if flshfile == 'N/A' or flashdur <= 0:\n return None\n\n if flashsta != 'SUCCESSFUL':\n warnings.warn('Flash status is {0}'.format(flashsta),\n AstropyUserWarning)\n\n flshfile = from_irafpath(flshfile)\n ampstring = prihdr['CCDAMP']\n\n with fits.open(flshfile) as hduflash:\n if ampstring == 'ABCD':\n flash = np.concatenate(\n (hduflash['sci', 1].data,\n hduflash['sci', 2].data[::-1, :]), axis=1)\n elif ampstring in ('A', 'B', 'AB'):\n flash = extract_ref(scihdu, hduflash['sci', 2])\n else:\n flash = extract_ref(scihdu, hduflash['sci', 1])\n\n flash = flash * flashdur\n\n return flash"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef extract_flatfield(prihdr, scihdu):\n for ff in ['DFLTFILE', 'LFLTFILE']:\n vv = prihdr.get(ff, 'N/A')\n if vv != 'N/A':\n warnings.warn('{0}={1} is not accounted for'.format(ff, vv),\n AstropyUserWarning)\n\n flatfile = prihdr.get('PFLTFILE', 'N/A')\n\n if flatfile == 'N/A':\n return None\n\n flatfile = from_irafpath(flatfile)\n ampstring = prihdr['CCDAMP']\n\n with fits.open(flatfile) as hduflat:\n if ampstring == 'ABCD':\n invflat = np.concatenate(\n (1 / hduflat['sci', 1].data,\n 1 / hduflat['sci', 2].data[::-1, :]), axis=1)\n elif ampstring in ('A', 'B', 'AB'):\n invflat = 1 / extract_ref(scihdu, hduflat['sci', 2])\n else:\n invflat = 1 / extract_ref(scihdu, hduflat['sci', 1])\n\n return invflat", "response": "Extract flatfield data from the FITS primary header HDU and the science image."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef from_irafpath(irafpath):\n s = irafpath.split('$')\n\n if len(s) != 2:\n return irafpath\n if len(s[0]) == 0:\n return irafpath\n\n try:\n refdir = os.environ[s[0]]\n except KeyError:\n raise ValueError('{0} environment variable undefined'.format(s[0]))\n\n return os.path.join(refdir, s[1])", "response": "Resolve IRAF path like jref$ into actual file path."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nextracting the section of the relevant reference image that corresponds to the given science image.", "response": "def extract_ref(scihdu, refhdu):\n \"\"\"Extract section of the reference image that\n corresponds to the given science image.\n\n This only returns a view, not a copy of the\n reference image's array.\n\n Parameters\n ----------\n scihdu, refhdu : obj\n Extension HDU's of the science and reference image,\n respectively.\n\n Returns\n -------\n refdata : array-like\n Section of the relevant reference image.\n\n Raises\n ------\n NotImplementedError\n Either science or reference data are binned.\n\n ValueError\n Extracted section size mismatch.\n\n \"\"\"\n same_size, rx, ry, x0, y0 = find_line(scihdu, refhdu)\n\n # Use the whole reference image\n if same_size:\n return refhdu.data\n\n # Binned data\n if rx != 1 or ry != 1:\n raise NotImplementedError(\n 'Either science or reference data are binned')\n\n # Extract a view of the sub-section\n ny, nx = scihdu.data.shape\n refdata = refhdu.data[y0:y0+ny, x0:x0+nx]\n\n if refdata.shape != (ny, nx):\n raise ValueError('Extracted reference image is {0} but science image '\n 'is {1}'.format(refdata.shape, (ny, nx)))\n\n return refdata"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef find_line(scihdu, refhdu):\n sci_bin, sci_corner = get_corner(scihdu.header)\n ref_bin, ref_corner = get_corner(refhdu.header)\n\n # We can use the reference image directly, without binning\n # and without extracting a subset.\n if (sci_corner[0] == ref_corner[0] and sci_corner[1] == ref_corner[1] and\n sci_bin[0] == ref_bin[0] and sci_bin[1] == ref_bin[1] and\n scihdu.data.shape[1] == refhdu.data.shape[1]):\n same_size = True\n rx = 1\n ry = 1\n x0 = 0\n y0 = 0\n\n # Reference image is binned more than the science image.\n elif ref_bin[0] > sci_bin[0] or ref_bin[1] > sci_bin[1]:\n same_size = False\n rx = ref_bin[0] / sci_bin[0]\n ry = ref_bin[1] / sci_bin[1]\n x0 = (sci_corner[0] - ref_corner[0]) / ref_bin[0]\n y0 = (sci_corner[1] - ref_corner[1]) / ref_bin[1]\n\n # For subarray input images, whether they are binned or not.\n else:\n same_size = False\n\n # Ratio of bin sizes.\n ratiox = sci_bin[0] / ref_bin[0]\n ratioy = sci_bin[1] / ref_bin[1]\n\n if (ratiox * ref_bin[0] != sci_bin[0] or\n ratioy * ref_bin[1] != sci_bin[1]):\n raise ValueError('Science and reference data size mismatch')\n\n # cshift is the offset in units of unbinned pixels.\n # Divide by ref_bin to convert to units of pixels in the ref image.\n cshift = (sci_corner[0] - ref_corner[0], sci_corner[1] - ref_corner[1])\n xzero = cshift[0] / ref_bin[0]\n yzero = cshift[1] / ref_bin[1]\n\n if (xzero * ref_bin[0] != cshift[0] or\n yzero * ref_bin[1] != cshift[1]):\n warnings.warn('Subimage offset not divisible by bin size',\n AstropyUserWarning)\n\n rx = ratiox\n ry = ratioy\n x0 = xzero\n y0 = yzero\n\n # Ensure integer index\n x0 = int(x0)\n y0 = int(y0)\n\n return same_size, rx, ry, x0, y0", "response": "Return a copy of the base image that is part of the given science image and the corresponding subset of a reference image."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nobtain the bin and corner information for a subarray.", "response": "def get_corner(hdr, rsize=1):\n \"\"\"Obtain bin and corner information for a subarray.\n\n ``LTV1``, ``LTV2``, ``LTM1_1``, and ``LTM2_2`` keywords\n are extracted from the given extension header and converted\n to bin and corner values (0-indexed).\n\n ``LTV1`` for the CCD uses the beginning of the illuminated\n portion as the origin, not the beginning of the overscan region.\n Thus, the computed X-corner has the same origin as ``LTV1``,\n which is what we want, but it differs from the ``CENTERA1``\n header keyword, which has the beginning of the overscan region\n as origin.\n\n .. note:: Translated from ``calacs/lib/getcorner.c``.\n\n Parameters\n ----------\n hdr : obj\n Extension header.\n\n rsize : int, optional\n Size of reference pixel in units of high-res pixels.\n\n Returns\n -------\n bin : tuple of int\n Pixel size in X and Y.\n\n corner : tuple of int\n Corner of subarray in X and Y.\n\n \"\"\"\n ltm, ltv = get_lt(hdr)\n return from_lt(rsize, ltm, ltv)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nobtain the LTV and LTM keywords from the header.", "response": "def get_lt(hdr):\n \"\"\"Obtain the LTV and LTM keyword values.\n\n Note that this returns the values just as read from the header,\n which means in particular that the LTV values are for one-indexed\n pixel coordinates.\n\n LTM keywords are the diagonal elements of MWCS linear\n transformation matrix, while LTV's are MWCS linear transformation\n vector (1-indexed).\n\n .. note:: Translated from ``calacs/lib/getlt.c``.\n\n Parameters\n ----------\n hdr : obj\n Extension header.\n\n Returns\n -------\n ltm, ltv : tuple of float\n ``(LTM1_1, LTM2_2)`` and ``(LTV1, LTV2)``.\n Values are ``(1, 1)`` and ``(0, 0)`` if not found,\n to accomodate reference files with missing info.\n\n Raises\n ------\n ValueError\n Invalid LTM* values.\n\n \"\"\"\n ltm = (hdr.get('LTM1_1', 1.0), hdr.get('LTM2_2', 1.0))\n\n if ltm[0] <= 0 or ltm[1] <= 0:\n raise ValueError('(LTM1_1, LTM2_2) = {0} is invalid'.format(ltm))\n\n ltv = (hdr.get('LTV1', 0.0), hdr.get('LTV2', 0.0))\n return ltm, ltv"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef from_lt(rsize, ltm, ltv):\n dbinx = rsize / ltm[0]\n dbiny = rsize / ltm[1]\n\n dxcorner = (dbinx - rsize) - dbinx * ltv[0]\n dycorner = (dbiny - rsize) - dbiny * ltv[1]\n\n # Round off to the nearest integer.\n bin = (_nint(dbinx), _nint(dbiny))\n corner = (_nint(dxcorner), _nint(dycorner))\n\n return bin, corner", "response": "Compute the bin and corner location of unbinned pixels in units\n ."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef hdr_vals_for_overscan(root):\n with fits.open(root + '_spt.fits') as hdu:\n spthdr = hdu[0].header\n with fits.open(root + '_raw.fits') as hdu:\n prihdr = hdu[0].header\n xstart = spthdr['SS_A1CRN']\n ystart = spthdr['SS_A2CRN']\n xsize = spthdr['SS_A1SZE']\n ysize = spthdr['SS_A2SZE']\n ccdamp = prihdr['CCDAMP']\n\n return ccdamp, xstart, ystart, xsize, ysize", "response": "Retrieve header keyword values from RAW and SPT\n FITS files to pass on to check_oscntab."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef check_oscntab(oscntab, ccdamp, xsize, ysize, leading, trailing):\n tab = Table.read(oscntab)\n ccdamp = ccdamp.lower().rstrip()\n for row in tab:\n if (row['CCDAMP'].lower().rstrip() in ccdamp and\n row['NX'] == xsize and row['NY'] == ysize and\n row['TRIMX1'] == leading and row['TRIMX2'] == trailing):\n return True\n return False", "response": "Checks if the input parameters are in the OSCNTAB file."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check_overscan(xstart, xsize, total_prescan_pixels=24,\n total_science_pixels=4096):\n \"\"\"Check image for bias columns.\n\n Parameters\n ----------\n xstart : int\n Starting column of the readout in detector coordinates.\n\n xsize : int\n Number of columns in the readout.\n\n total_prescan_pixels : int\n Total prescan pixels for a single amplifier on a detector.\n Default is 24 for WFC.\n\n total_science_pixels : int\n Total science pixels across a detector.\n Default is 4096 for WFC (across two amplifiers).\n\n Returns\n -------\n hasoverscan : bool\n Indication if there are bias columns in the image.\n\n leading : int\n Number of bias columns on the A/C amplifiers\n side of the CCDs (\"TRIMX1\" in ``OSCNTAB``).\n\n trailing : int\n Number of bias columns on the B/D amplifiers\n side of the CCDs (\"TRIMX2\" in ``OSCNTAB``).\n\n \"\"\"\n hasoverscan = False\n leading = 0\n trailing = 0\n\n if xstart < total_prescan_pixels:\n hasoverscan = True\n leading = abs(xstart - total_prescan_pixels)\n\n if (xstart + xsize) > total_science_pixels:\n hasoverscan = True\n trailing = abs(total_science_pixels -\n (xstart + xsize - total_prescan_pixels))\n\n return hasoverscan, leading, trailing", "response": "Check image for bias columns."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef process_request(self, request):\n referer_url = request.META.get('HTTP_REFERER')\n return_to_index_url = request.session.get('return_to_index_url')\n\n try:\n if all((\n return_to_index_url,\n referer_url,\n request.method == 'GET',\n not request.is_ajax(),\n resolve(request.path).url_name in ('wagtailadmin_explore_root',\n 'wagtailadmin_explore'),\n )):\n perform_redirection = False\n referer_match = resolve(urlparse(referer_url).path)\n if all((\n referer_match.namespace == 'wagtailadmin_pages',\n referer_match.url_name in (\n 'add',\n 'edit',\n 'delete',\n 'unpublish',\n 'copy'\n ),\n )):\n perform_redirection = True\n elif all((\n not referer_match.namespace,\n referer_match.url_name in (\n 'wagtailadmin_pages_create',\n 'wagtailadmin_pages_edit',\n 'wagtailadmin_pages_delete',\n 'wagtailadmin_pages_unpublish'\n ),\n )):\n perform_redirection = True\n if perform_redirection:\n del request.session['return_to_index_url']\n return HttpResponseRedirect(return_to_index_url)\n\n except Resolver404:\n pass\n\n return None", "response": "Process the request and return the appropriate HTTP response."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run(configobj=None):\n acs2d(configobj['input'],\n exec_path=configobj['exec_path'],\n time_stamps=configobj['time_stamps'],\n verbose=configobj['verbose'],\n quiet=configobj['quiet']\n )", "response": "Run TEAL interface for the acs2d function."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns TEAL interface for the acscte function.", "response": "def run(configobj=None):\n \"\"\"\n TEAL interface for the `acscte` function.\n\n \"\"\"\n acscte(configobj['input'],\n exec_path=configobj['exec_path'],\n time_stamps=configobj['time_stamps'],\n verbose=configobj['verbose'],\n quiet=configobj['quiet'],\n single_core=configobj['single_core']\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _check_inputs(self):\n valid_detector = True\n valid_filter = True\n valid_date = True\n # Determine the submitted detector is valid\n if self.detector not in self._valid_detectors:\n msg = ('{} is not a valid detector option.\\n'\n 'Please choose one of the following:\\n{}\\n'\n '{}'.format(self.detector,\n '\\n'.join(self._valid_detectors),\n self._msg_div))\n LOG.error(msg)\n valid_detector = False\n\n # Determine if the submitted filter is valid\n if (self.filt is not None and valid_detector and\n self.filt not in self.valid_filters[self.detector]):\n msg = ('{} is not a valid filter for {}\\n'\n 'Please choose one of the following:\\n{}\\n'\n '{}'.format(self.filt, self.detector,\n '\\n'.join(self.valid_filters[self.detector]),\n self._msg_div))\n LOG.error(msg)\n valid_filter = False\n\n # Determine if the submitted date is valid\n date_check = self._check_date()\n if date_check is not None:\n LOG.error('{}\\n{}'.format(date_check, self._msg_div))\n valid_date = False\n\n if not valid_detector or not valid_filter or not valid_date:\n return False\n\n return True", "response": "Checks the inputs to ensure they are valid."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _check_date(self, fmt='%Y-%m-%d'):\n result = None\n try:\n dt_obj = dt.datetime.strptime(self.date, fmt)\n except ValueError:\n result = '{} does not match YYYY-MM-DD format'.format(self.date)\n else:\n if dt_obj < self._acs_installation_date:\n result = ('The observation date cannot occur '\n 'before ACS was installed ({})'\n .format(self._acs_installation_date.strftime(fmt)))\n elif dt_obj > self._extrapolation_date:\n result = ('The observation date cannot occur after the '\n 'maximum allowable date, {}. Extrapolations of the '\n 'instrument throughput after this date lead to '\n 'high uncertainties and are therefore invalid.'\n .format(self._extrapolation_date.strftime(fmt)))\n finally:\n return result", "response": "Convenience method for determining if the input date is valid."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsubmit a request to the ACS Zeropoint Calculator.", "response": "def _submit_request(self):\n \"\"\"Submit a request to the ACS Zeropoint Calculator.\n\n If an exception is raised during the request, an error message is\n given. Otherwise, the response is saved in the corresponding\n attribute.\n\n \"\"\"\n try:\n self._response = urlopen(self._url)\n except URLError as e:\n msg = ('{}\\n{}\\nThe query failed! Please check your inputs. '\n 'If the error persists, submit a ticket to the '\n 'ACS Help Desk at hsthelp.stsci.edu with the error message '\n 'displayed above.'.format(str(e), self._msg_div))\n LOG.error(msg)\n self._failed = True\n else:\n self._failed = False"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _parse_and_format(self):\n\n soup = BeautifulSoup(self._response.read(), 'html.parser')\n\n # Grab all elements in the table returned by the ZPT calc.\n td = soup.find_all('td')\n\n # Remove the units attached to PHOTFLAM and PHOTPLAM column names.\n td = [val.text.split(' ')[0] for val in td]\n\n # Turn the single list into a 2-D numpy array\n data = np.reshape(td,\n (int(len(td) / self._block_size), self._block_size))\n # Create the QTable, note that sometimes self._response will be empty\n # even though the return was successful; hence the try/except to catch\n # any potential index errors. Provide the user with a message and\n # set the zpt_table to None.\n try:\n tab = QTable(data[1:, :],\n names=data[0],\n dtype=[str, float, float, float, float, float])\n except IndexError as e:\n msg = ('{}\\n{}\\n There was an issue parsing the request. '\n 'Try resubmitting the query. If this issue persists, please '\n 'submit a ticket to the Help Desk at'\n 'https://stsci.service-now.com/hst'\n .format(e, self._msg_div))\n LOG.info(msg)\n self._zpt_table = None\n else:\n # If and only if no exception was raised, attach the units to each\n # column of the QTable. Note we skip the FILTER column because\n # Quantity objects in astropy must be numerical (i.e. not str)\n for col in tab.colnames:\n if col.lower() == 'filter':\n continue\n tab[col].unit = self._data_units[col]\n\n self._zpt_table = tab", "response": "Parse and format the results returned by the ACS Zeropoint Calculator."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fetch(self):\n LOG.info('Checking inputs...')\n valid_inputs = self._check_inputs()\n\n if valid_inputs:\n LOG.info('Submitting request to {}'.format(self._url))\n self._submit_request()\n if self._failed:\n return\n\n LOG.info('Parsing the response and formatting the results...')\n self._parse_and_format()\n return self.zpt_table\n\n LOG.error('Please fix the incorrect input(s)')", "response": "Submit the request to the ACS Zeropoints Calculator and parse the response into a table with the correct units."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a QuerySet of all the model instances that can be edited by the admin site.", "response": "def get_queryset(self, request):\n \"\"\"\n Returns a QuerySet of all model instances that can be edited by the\n admin site.\n \"\"\"\n qs = self.model._default_manager.get_queryset()\n ordering = self.get_ordering(request)\n if ordering:\n qs = qs.order_by(*ordering)\n return qs"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef unpublish_view(self, request, object_id):\n kwargs = {'model_admin': self, 'object_id': object_id}\n view_class = self.unpublish_view_class\n return view_class.as_view(**kwargs)(request)", "response": "Instantiates a class - based view that redirects to Wagtail s unpublish view for models that extend Page."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a list of permissions that can be assigned to the current page.", "response": "def get_permissions_for_registration(self):\n \"\"\"\n Utilised by Wagtail's 'register_permissions' hook to allow permissions\n for a model to be assigned to groups in settings. This is only required\n if the model isn't a Page model, and isn't registered as a Snippet\n \"\"\"\n from wagtail.wagtailsnippets.models import SNIPPET_MODELS\n if not self.is_pagemodel and self.model not in SNIPPET_MODELS:\n return self.permission_helper.get_all_model_permissions()\n return Permission.objects.none()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of urls that can be used to register the admin urls for the current object.", "response": "def get_admin_urls_for_registration(self):\n \"\"\"\n Utilised by Wagtail's 'register_admin_urls' hook to register urls for\n our the views that class offers.\n \"\"\"\n urls = (\n url(get_url_pattern(self.opts),\n self.index_view, name=get_url_name(self.opts)),\n url(get_url_pattern(self.opts, 'create'),\n self.create_view, name=get_url_name(self.opts, 'create')),\n url(get_object_specific_url_pattern(self.opts, 'edit'),\n self.edit_view, name=get_url_name(self.opts, 'edit')),\n url(get_object_specific_url_pattern(self.opts, 'confirm_delete'),\n self.confirm_delete_view,\n name=get_url_name(self.opts, 'confirm_delete')),\n )\n if self.inspect_view_enabled:\n urls = urls + (\n url(get_object_specific_url_pattern(self.opts, 'inspect'),\n self.inspect_view,\n name=get_url_name(self.opts, 'inspect')),\n )\n if self.is_pagemodel:\n urls = urls + (\n url(get_url_pattern(self.opts, 'choose_parent'),\n self.choose_parent_view,\n name=get_url_name(self.opts, 'choose_parent')),\n url(get_object_specific_url_pattern(self.opts, 'unpublish'),\n self.unpublish_view,\n name=get_url_name(self.opts, 'unpublish')),\n url(get_object_specific_url_pattern(self.opts, 'copy'),\n self.copy_view,\n name=get_url_name(self.opts, 'copy')),\n )\n return urls\n\n def construct_main_menu(self, request, menu_items):\n warnings.warn((\n \"The 'construct_main_menu' method is now deprecated. You \"\n \"should also remove the construct_main_menu hook from \"\n \"wagtail_hooks.py in your app folder.\"), DeprecationWarning)\n return menu_items"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a menu item for this group.", "response": "def get_menu_item(self):\n \"\"\"\n Utilised by Wagtail's 'register_menu_item' hook to create a menu\n for this group with a SubMenu linking to listing pages for any\n associated ModelAdmin instances\n \"\"\"\n if self.modeladmin_instances:\n submenu = SubMenu(self.get_submenu_items())\n return GroupMenuItem(self, self.get_menu_order(), submenu)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_permissions_for_registration(self):\n qs = Permission.objects.none()\n for instance in self.modeladmin_instances:\n qs = qs | instance.get_permissions_for_registration()\n return qs", "response": "Returns a queryset of permissions that can be assigned to this class"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a list of admin urls for this model admin.", "response": "def get_admin_urls_for_registration(self):\n \"\"\"\n Utilised by Wagtail's 'register_admin_urls' hook to register urls for\n used by any associated ModelAdmin instances\n \"\"\"\n urls = []\n for instance in self.modeladmin_instances:\n urls.extend(instance.get_admin_urls_for_registration())\n return urls"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns True if the menu item is shown.", "response": "def is_shown(self, request):\n \"\"\"\n If there aren't any visible items in the submenu, don't bother to show\n this menu item\n \"\"\"\n for menuitem in self.menu._registered_menu_items:\n if menuitem.is_shown(request):\n return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nruns TEAL interface for the acscteforwardmodel function.", "response": "def run(configobj=None):\n \"\"\"\n TEAL interface for the `acscteforwardmodel` function.\n\n \"\"\"\n acscteforwardmodel(configobj['input'],\n exec_path=configobj['exec_path'],\n time_stamps=configobj['time_stamps'],\n verbose=configobj['verbose'],\n quiet=configobj['quiet'],\n single_core=configobj['single_core']\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef run(configobj=None):\n acsccd(configobj['input'],\n exec_path=configobj['exec_path'],\n time_stamps=configobj['time_stamps'],\n verbose=configobj['verbose'],\n quiet=configobj['quiet'] #,\n #dqicorr=configobj['dqicorr'],\n #atodcorr=configobj['atodcorr'],\n #blevcorr=configobj['blevcorr'],\n #biascorr=configobj['biascorr']\n )", "response": "Run TEAL s acsccd function."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrun TEAL interface for the TEAL module", "response": "def run(configobj=None):\n \"\"\"TEAL interface for :func:`destripe_plus`.\"\"\"\n destripe_plus(\n configobj['input'],\n suffix=configobj['suffix'],\n stat=configobj['stat'],\n maxiter=configobj['maxiter'],\n sigrej=configobj['sigrej'],\n lower=configobj['lower'],\n upper=configobj['upper'],\n binwidth=configobj['binwidth'],\n scimask1=configobj['scimask1'],\n scimask2=configobj['scimask2'],\n dqbits=configobj['dqbits'],\n rpt_clean=configobj['rpt_clean'],\n atol=configobj['atol'],\n cte_correct=configobj['cte_correct'],\n clobber=configobj['clobber'],\n verbose=configobj['verbose'])"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a tuple containing a queryset to implement the search and a boolean indicating if the results may contain duplicates.", "response": "def get_search_results(self, request, queryset, search_term):\n \"\"\"\n Returns a tuple containing a queryset to implement the search,\n and a boolean indicating if the results may contain duplicates.\n \"\"\"\n # Apply keyword searches.\n def construct_search(field_name):\n if field_name.startswith('^'):\n return \"%s__istartswith\" % field_name[1:]\n elif field_name.startswith('='):\n return \"%s__iexact\" % field_name[1:]\n elif field_name.startswith('@'):\n return \"%s__search\" % field_name[1:]\n else:\n return \"%s__icontains\" % field_name\n\n use_distinct = False\n if self.search_fields and search_term:\n orm_lookups = [construct_search(str(search_field))\n for search_field in self.search_fields]\n for bit in search_term.split():\n or_queries = [models.Q(**{orm_lookup: bit})\n for orm_lookup in orm_lookups]\n queryset = queryset.filter(reduce(operator.or_, or_queries))\n if not use_distinct:\n for search_spec in orm_lookups:\n if lookup_needs_distinct(self.opts, search_spec):\n use_distinct = True\n break\n\n return queryset, use_distinct"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn all the parameters except IGNORED_PARAMS", "response": "def get_filters_params(self, params=None):\n \"\"\"\n Returns all params except IGNORED_PARAMS\n \"\"\"\n if not params:\n params = self.params\n lookup_params = params.copy() # a dictionary of the query string\n # Remove all the parameters that are globally and systematically\n # ignored.\n for ignored in IGNORED_PARAMS:\n if ignored in lookup_params:\n del lookup_params[ignored]\n return lookup_params"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_ordering_field(self, field_name):\n try:\n field = self.opts.get_field(field_name)\n return field.name\n except FieldDoesNotExist:\n # See whether field_name is a name of a non-field\n # that allows sorting.\n if callable(field_name):\n attr = field_name\n elif hasattr(self.model_admin, field_name):\n attr = getattr(self.model_admin, field_name)\n else:\n attr = getattr(self.model, field_name)\n return getattr(attr, 'admin_order_field', None)", "response": "Returns the name of the proper model field that allows sorting."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_ordering(self, request, queryset):\n params = self.params\n ordering = list(self.get_default_ordering(request))\n if ORDER_VAR in params:\n # Clear ordering and used params\n ordering = []\n order_params = params[ORDER_VAR].split('.')\n for p in order_params:\n try:\n none, pfx, idx = p.rpartition('-')\n field_name = self.list_display[int(idx)]\n order_field = self.get_ordering_field(field_name)\n if not order_field:\n continue # No 'admin_order_field', skip it\n # reverse order if order_field has already \"-\" as prefix\n if order_field.startswith('-') and pfx == \"-\":\n ordering.append(order_field[1:])\n else:\n ordering.append(pfx + order_field)\n except (IndexError, ValueError):\n continue # Invalid ordering specified, skip it.\n\n # Add the given query's ordering fields, if any.\n ordering.extend(queryset.query.order_by)\n\n # Ensure that the primary key is systematically present in the list of\n # ordering fields so we can guarantee a deterministic order across all\n # database backends.\n pk_name = self.opts.pk.name\n if not (set(ordering) & {'pk', '-pk', pk_name, '-' + pk_name}):\n # The two sets do not intersect, meaning the pk isn't present. So\n # we add it.\n ordering.append('-pk')\n\n return ordering", "response": "Returns the list of ordering fields for the change list."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_ordering_field_columns(self):\n\n # We must cope with more than one column having the same underlying\n # sort field, so we base things on column numbers.\n ordering = self._get_default_ordering()\n ordering_fields = OrderedDict()\n if ORDER_VAR not in self.params:\n # for ordering specified on model_admin or model Meta, we don't\n # know the right column numbers absolutely, because there might be\n # morr than one column associated with that ordering, so we guess.\n for field in ordering:\n if field.startswith('-'):\n field = field[1:]\n order_type = 'desc'\n else:\n order_type = 'asc'\n for index, attr in enumerate(self.list_display):\n if self.get_ordering_field(attr) == field:\n ordering_fields[index] = order_type\n break\n else:\n for p in self.params[ORDER_VAR].split('.'):\n none, pfx, idx = p.rpartition('-')\n try:\n idx = int(idx)\n except ValueError:\n continue # skip it\n ordering_fields[idx] = 'desc' if pfx == '-' else 'asc'\n return ordering_fields", "response": "Returns an OrderedDict of ordering field numbers and asc or desc."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_field_label(self, field_name, field=None):\n label = None\n if field is not None:\n label = getattr(field, 'verbose_name', None)\n if label is None:\n label = getattr(field, 'name', None)\n if label is None:\n label = field_name\n return label.capitalize()", "response": "Return a label to display for a field"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a display value for a field.", "response": "def get_field_display_value(self, field_name, field=None):\n \"\"\" Return a display value for a field \"\"\"\n\n \"\"\"\n Firstly, check for a 'get_fieldname_display' property/method on\n the model, and return the value of that, if present.\n \"\"\"\n val_funct = getattr(self.instance, 'get_%s_display' % field_name, None)\n if val_funct is not None:\n if callable(val_funct):\n return val_funct()\n return val_funct\n\n \"\"\"\n Secondly, if we have a real field, we can try to display something\n more useful for it.\n \"\"\"\n if field is not None:\n try:\n field_type = field.get_internal_type()\n if (\n field_type == 'ForeignKey' and\n field.related_model == get_image_model()\n ):\n # The field is an image\n return self.get_image_field_display(field_name, field)\n\n if (\n field_type == 'ForeignKey' and\n field.related_model == Document\n ):\n # The field is a document\n return self.get_document_field_display(field_name, field)\n\n except AttributeError:\n pass\n\n \"\"\"\n Resort to getting the value of 'field_name' from the instance.\n \"\"\"\n return getattr(self.instance, field_name,\n self.model_admin.get_empty_value_display())"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_image_field_display(self, field_name, field):\n image = getattr(self.instance, field_name)\n if image:\n fltr, _ = Filter.objects.get_or_create(spec='max-400x400')\n rendition = image.get_rendition(fltr)\n return rendition.img_tag\n return self.model_admin.get_empty_value_display()", "response": "Get the display of an image field"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_document_field_display(self, field_name, field):\n document = getattr(self.instance, field_name)\n if document:\n return mark_safe(\n '%s (%s, %s)' % (\n document.url,\n document.title,\n document.file_extension.upper(),\n filesizeformat(document.file.size),\n )\n )\n return self.model_admin.get_empty_value_display()", "response": "Render a link to a document field."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_dict_for_field(self, field_name):\n try:\n field = self.model._meta.get_field(field_name)\n except FieldDoesNotExist:\n field = None\n return {\n 'label': self.get_field_label(field_name, field),\n 'value': self.get_field_display_value(field_name, field),\n }", "response": "Returns a dictionary containing label and value values to display\n for a field."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_fields_dict(self):\n fields = []\n for field_name in self.model_admin.get_inspect_view_fields():\n fields.append(self.get_dict_for_field(field_name))\n return fields", "response": "Return a list of label value dictionaries to represent the\n fiels named by the model_admin class s get_inspect_view_fields method."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngenerates the actual list of data for the given result.", "response": "def items_for_result(view, result):\n \"\"\"\n Generates the actual list of data.\n \"\"\"\n model_admin = view.model_admin\n for field_name in view.list_display:\n empty_value_display = model_admin.get_empty_value_display()\n row_classes = ['field-%s' % field_name]\n try:\n f, attr, value = lookup_field(field_name, result, model_admin)\n except ObjectDoesNotExist:\n result_repr = empty_value_display\n else:\n empty_value_display = getattr(attr, 'empty_value_display', empty_value_display)\n if f is None or f.auto_created:\n allow_tags = getattr(attr, 'allow_tags', False)\n boolean = getattr(attr, 'boolean', False)\n if boolean or not value:\n allow_tags = True\n\n if django.VERSION >= (1, 9):\n result_repr = display_for_value(value, empty_value_display, boolean)\n else:\n result_repr = display_for_value(value, boolean)\n # Strip HTML tags in the resulting text, except if the\n # function has an \"allow_tags\" attribute set to True.\n if allow_tags:\n result_repr = mark_safe(result_repr)\n if isinstance(value, (datetime.date, datetime.time)):\n row_classes.append('nowrap')\n else:\n if isinstance(f, models.ManyToOneRel):\n field_val = getattr(result, f.name)\n if field_val is None:\n result_repr = empty_value_display\n else:\n result_repr = field_val\n else:\n if django.VERSION >= (1, 9):\n result_repr = display_for_field(value, f, empty_value_display)\n else:\n result_repr = display_for_field(value, f)\n\n if isinstance(f, (models.DateField, models.TimeField, models.ForeignKey)):\n row_classes.append('nowrap')\n if force_text(result_repr) == '':\n result_repr = mark_safe(' ')\n row_classes.extend(model_admin.get_extra_class_names_for_field_col(field_name, result))\n row_attributes_dict = model_admin.get_extra_attrs_for_field_col(field_name, result)\n row_attributes_dict['class'] = ' ' . join(row_classes)\n row_attributes = ''.join(' %s=\"%s\"' % (key, val) for key, val in row_attributes_dict.items())\n row_attributes_safe = mark_safe(row_attributes)\n yield format_html('{}', row_attributes_safe, result_repr)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef result_list(context):\n view = context['view']\n object_list = context['object_list']\n headers = list(result_headers(view))\n num_sorted_fields = 0\n for h in headers:\n if h['sortable'] and h['sorted']:\n num_sorted_fields += 1\n context.update({\n 'result_headers': headers,\n 'num_sorted_fields': num_sorted_fields,\n 'results': list(results(view, object_list))})\n return context", "response": "Displays the headers and data list together"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun TEAL acssum function.", "response": "def run(configobj=None):\n \"\"\"\n TEAL interface for the `acssum` function.\n\n \"\"\"\n acssum(configobj['input'],\n configobj['output'],\n exec_path=configobj['exec_path'],\n time_stamps=configobj['time_stamps'],\n verbose=configobj['verbose'],\n quiet=configobj['quiet']\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _detsat_one(filename, ext, sigma=2.0, low_thresh=0.1, h_thresh=0.5,\n small_edge=60, line_len=200, line_gap=75,\n percentile=(4.5, 93.0), buf=200, plot=False, verbose=False):\n \"\"\"Called by :func:`detsat`.\"\"\"\n if verbose:\n t_beg = time.time()\n\n fname = '{0}[{1}]'.format(filename, ext)\n\n # check extension\n if ext not in (1, 4, 'SCI', ('SCI', 1), ('SCI', 2)):\n warnings.warn('{0} is not a valid science extension for '\n 'ACS/WFC'.format(ext), AstropyUserWarning)\n\n # get the data\n image = fits.getdata(filename, ext)\n # image = im.astype('float64')\n\n # rescale the image\n p1, p2 = np.percentile(image, percentile)\n\n # there should always be some counts in the image, anything lower should\n # be set to one. Makes things nicer for finding edges.\n if p1 < 0:\n p1 = 0.0\n\n if verbose:\n print('Rescale intensity percentiles: {0}, {1}'.format(p1, p2))\n\n image = exposure.rescale_intensity(image, in_range=(p1, p2))\n\n # get the edges\n immax = np.max(image)\n edge = canny(image, sigma=sigma,\n low_threshold=immax * low_thresh,\n high_threshold=immax * h_thresh)\n\n # clean up the small objects, will make less noise\n morph.remove_small_objects(edge, min_size=small_edge, connectivity=8,\n in_place=True)\n\n # create an array of angles from 0 to 180, exactly 0 will get bad columns\n # but it is unlikely that a satellite will be exactly at 0 degrees, so\n # don't bother checking.\n # then, convert to radians.\n angle = np.radians(np.arange(2, 178, 0.5, dtype=float))\n\n # perform Hough Transform to detect straight lines.\n # only do if plotting to visualize the image in hough space.\n # otherwise just preform a Probabilistic Hough Transform.\n if plot and plt is not None:\n h, theta, d = transform.hough_line(edge, theta=angle)\n plt.ion()\n\n # perform Probabilistic Hough Transformation to get line segments.\n # NOTE: Results are slightly different from run to run!\n result = transform.probabilistic_hough_line(\n edge, threshold=210, line_length=line_len,\n line_gap=line_gap, theta=angle)\n result = np.asarray(result)\n n_result = len(result)\n\n # initially assume there is no satellite\n satellite = False\n\n # only continue if there was more than one point (at least a line)\n # returned from the PHT\n if n_result > 1:\n if verbose:\n print('Length of PHT result: {0}'.format(n_result))\n\n # create lists for X and Y positions of lines and build points\n x0 = result[:, 0, 0]\n y0 = result[:, 0, 1]\n x1 = result[:, 1, 0]\n y1 = result[:, 1, 1]\n\n # set some boundries\n ymax, xmax = image.shape\n topx = xmax - buf\n topy = ymax - buf\n\n if verbose:\n print('min(x0)={0:4d}, min(x1)={1:4d}, min(y0)={2:4d}, '\n 'min(y1)={3:4d}'.format(min(x0), min(x1), min(y0), min(y1)))\n print('max(x0)={0:4d}, max(x1)={1:4d}, max(y0)={2:4d}, '\n 'max(y1)={3:4d}'.format(max(x0), max(x1), max(y0), max(y1)))\n print('buf={0}'.format(buf))\n print('topx={0}, topy={1}'.format(topx, topy))\n\n # set up trail angle \"tracking\" arrays.\n # find the angle of each segment and filter things out.\n # TODO: this may be wrong. Try using arctan2.\n trail_angle = np.degrees(np.arctan((y1 - y0) / (x1 - x0)))\n # round to the nearest 5 degrees, trail should not be that curved\n round_angle = (5 * np.round(trail_angle * 0.2)).astype(int)\n\n # take out 90 degree things\n mask = round_angle % 90 != 0\n\n if not np.any(mask):\n if verbose:\n print('No round_angle found')\n return np.empty(0)\n\n round_angle = round_angle[mask]\n trail_angle = trail_angle[mask]\n result = result[mask]\n\n ang, num = stats.mode(round_angle)\n\n # do the filtering\n truth = round_angle == ang[0]\n\n if verbose:\n print('trail_angle: {0}'.format(trail_angle))\n print('round_angle: {0}'.format(round_angle))\n print('mode(round_angle): {0}'.format(ang[0]))\n\n # filter out the outliers\n trail_angle = trail_angle[truth]\n result = result[truth]\n n_result = len(result)\n\n if verbose:\n print('Filtered trail_angle: {0}'.format(trail_angle))\n\n if n_result < 1:\n return np.empty(0)\n\n # if there is an unreasonable amount of points, it picked up garbage\n elif n_result > 300:\n warnings.warn(\n 'Way too many segments results to be correct ({0}). '\n 'Rejecting detection on {1}.'.format(n_result, fname),\n AstropyUserWarning)\n return np.empty(0)\n\n # remake the point lists with things taken out\n x0 = result[:, 0, 0]\n y0 = result[:, 0, 1]\n x1 = result[:, 1, 0]\n y1 = result[:, 1, 1]\n\n min_x0 = min(x0)\n min_y0 = min(y0)\n min_x1 = min(x1)\n min_y1 = min(y1)\n\n max_x0 = max(x0)\n max_y0 = max(y0)\n max_x1 = max(x1)\n max_y1 = max(y1)\n\n mean_angle = np.mean(trail_angle)\n\n # make decisions on where the trail went and determine if a trail\n # traversed the image\n # top to bottom\n if (((min_y0 < buf) or (min_y1 < buf)) and\n ((max_y0 > topy) or (max_y1 > topy))):\n satellite = True\n if verbose:\n print('Trail Direction: Top to Bottom')\n\n # right to left\n elif (((min_x0 < buf) or (min_x1 < buf)) and\n ((max_x0 > topx) or (max_x1 > topx))):\n satellite = True\n if verbose:\n print('Trail Direction: Right to Left')\n\n # bottom to left\n elif (((min_x0 < buf) or (min_x1 < buf)) and\n ((min_y0 < buf) or (min_y1 < buf)) and\n (-1 > mean_angle > -89)):\n satellite = True\n if verbose:\n print('Trail Direction: Bottom to Left')\n\n # top to left\n elif (((min_x0 < buf) or (min_x1 < buf)) and\n ((max_y0 > topy) or (max_y1 > topy)) and\n (89 > mean_angle > 1)):\n satellite = True\n if verbose:\n print('Trail Direction: Top to Left')\n\n # top to right\n elif (((max_x0 > topx) or (max_x1 > topx)) and\n ((max_y0 > topy) or (max_y1 > topy)) and\n (-1 > mean_angle > -89)):\n satellite = True\n if verbose:\n print('Trail Direction: Top to Right')\n\n # bottom to right\n elif (((max_x0 > topx) or (max_x1 > topx)) and\n ((min_y0 < buf) or (min_y1 < buf)) and\n (89 > mean_angle > 1)):\n satellite = True\n if verbose:\n print('Trail Direction: Bottom to Right')\n\n if satellite:\n if verbose:\n print('{0} trail segment(s) detected'.format(n_result))\n print('Trail angle list (not returned): ')\n print(trail_angle)\n print('End point list:')\n for i, ((px0, py0), (px1, py1)) in enumerate(result, 1):\n print('{0:5d}. ({1:4d}, {2:4d}), ({3:4d}, {4:4d})'.format(\n i, px0, py0, px1, py1))\n\n if plot and plt is not None:\n mean = np.median(image)\n stddev = image.std()\n lower = mean - stddev\n upper = mean + stddev\n\n fig1, ax1 = plt.subplots()\n ax1.imshow(edge, cmap=plt.cm.gray)\n ax1.set_title('Edge image for {0}'.format(fname))\n\n for (px0, py0), (px1, py1) in result: # Draw trails\n ax1.plot((px0, px1), (py0, py1), scalex=False, scaley=False)\n\n fig2, ax2 = plt.subplots()\n ax2.imshow(\n np.log(1 + h),\n extent=(np.rad2deg(theta[-1]), np.rad2deg(theta[0]),\n d[-1], d[0]), aspect=0.02)\n ax2.set_title('Hough Transform')\n ax2.set_xlabel('Angles (degrees)')\n ax2.set_ylabel('Distance from Origin (pixels)')\n\n fig3, ax3 = plt.subplots()\n ax3.imshow(image, vmin=lower, vmax=upper, cmap=plt.cm.gray)\n ax3.set_title(fname)\n\n for (px0, py0), (px1, py1) in result: # Draw trails\n ax3.plot((px0, px1), (py0, py1), scalex=False, scaley=False)\n\n plt.draw()\n\n else: # length of result was too small\n result = np.empty(0)\n\n if verbose:\n print('No trail detected; found {0} segments'.format(n_result))\n\n if plot and plt is not None:\n fig1, ax1 = plt.subplots()\n ax1.imshow(edge, cmap=plt.cm.gray)\n ax1.set_title(fname)\n\n # Draw trails\n for (px0, py0), (px1, py1) in result:\n ax1.plot((px0, px1), (py0, py1), scalex=False, scaley=False)\n\n if verbose:\n t_end = time.time()\n print('Run time: {0} s'.format(t_end - t_beg))\n\n return result", "response": "This function is called by detsat. It will take a file and the extension and detsat it and return the image."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_valid_indices(shape, ix0, ix1, iy0, iy1):\n ymax, xmax = shape\n\n if ix0 < 0:\n ix0 = 0\n if ix1 > xmax:\n ix1 = xmax\n if iy0 < 0:\n iy0 = 0\n if iy1 > ymax:\n iy1 = ymax\n\n if iy1 <= iy0 or ix1 <= ix0:\n raise IndexError(\n 'array[{0}:{1},{2}:{3}] is invalid'.format(iy0, iy1, ix0, ix1))\n\n return list(map(int, [ix0, ix1, iy0, iy1]))", "response": "Give array shape and desired indices return indices that are\n correctly bounded by the shape."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntransforming a point from original image coordinates to rotated image coordinates and back. It assumes the rotation point is the center of an image. This works on a simple rotation transformation:: newx = (startx) * np.cos(angle) - (starty) * np.sin(angle) newy = (startx) * np.sin(angle) + (starty) * np.cos(angle) It takes into account the differences in image size. Parameters ---------- point : tuple Point to be rotated, in the format of ``(x, y)`` measured from origin. angle : float The angle in degrees to rotate the point by as measured counter-clockwise from the X axis. ishape : tuple The shape of the original image, taken from ``image.shape``. rshape : tuple The shape of the rotated image, in the form of ``rotate.shape``. reverse : bool, optional Transform from rotated coordinates back to non-rotated image. Returns ------- rotated_point : tuple Rotated point in the format of ``(x, y)`` as measured from origin.", "response": "def _rotate_point(point, angle, ishape, rshape, reverse=False):\n \"\"\"Transform a point from original image coordinates to rotated image\n coordinates and back. It assumes the rotation point is the center of an\n image.\n\n This works on a simple rotation transformation::\n\n newx = (startx) * np.cos(angle) - (starty) * np.sin(angle)\n newy = (startx) * np.sin(angle) + (starty) * np.cos(angle)\n\n It takes into account the differences in image size.\n\n Parameters\n ----------\n point : tuple\n Point to be rotated, in the format of ``(x, y)`` measured from\n origin.\n\n angle : float\n The angle in degrees to rotate the point by as measured\n counter-clockwise from the X axis.\n\n ishape : tuple\n The shape of the original image, taken from ``image.shape``.\n\n rshape : tuple\n The shape of the rotated image, in the form of ``rotate.shape``.\n\n reverse : bool, optional\n Transform from rotated coordinates back to non-rotated image.\n\n Returns\n -------\n rotated_point : tuple\n Rotated point in the format of ``(x, y)`` as measured from origin.\n\n \"\"\"\n # unpack the image and rotated images shapes\n if reverse:\n angle = (angle * -1)\n temp = ishape\n ishape = rshape\n rshape = temp\n\n # transform into center of image coordinates\n yhalf, xhalf = ishape\n yrhalf, xrhalf = rshape\n\n yhalf = yhalf / 2\n xhalf = xhalf / 2\n yrhalf = yrhalf / 2\n xrhalf = xrhalf / 2\n\n startx = point[0] - xhalf\n starty = point[1] - yhalf\n\n # do the rotation\n newx = startx * np.cos(angle) - starty * np.sin(angle)\n newy = startx * np.sin(angle) + starty * np.cos(angle)\n\n # add back the padding from changing the size of the image\n newx = newx + xrhalf\n newy = newy + yrhalf\n\n return (newx, newy)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef make_mask(filename, ext, trail_coords, sublen=75, subwidth=200, order=3,\n sigma=4, pad=10, plot=False, verbose=False):\n \"\"\"Create DQ mask for an image for a given satellite trail.\n This mask can be added to existing DQ data using :func:`update_dq`.\n\n .. note::\n\n Unlike :func:`detsat`, multiprocessing is not available for\n this function.\n\n Parameters\n ----------\n filename : str\n FITS image filename.\n\n ext : int, str, or tuple\n Extension for science data, as accepted by ``astropy.io.fits``.\n\n trail_coords : ndarray\n One of the trails returned by :func:`detsat`.\n This must be in the format of ``[[x0, y0], [x1, y1]]``.\n\n sublen : int, optional\n Length of strip to use as the fitting window for the trail.\n\n subwidth : int, optional\n Width of box to fit trail on.\n\n order : int, optional\n The order of the spline interpolation for image rotation.\n See :func:`skimage.transform.rotate`.\n\n sigma : float, optional\n Sigma of the satellite trail for detection. If points are\n a given sigma above the background in the subregion then it is\n marked as a satellite. This may need to be lowered for resolved\n trails.\n\n pad : int, optional\n Amount of extra padding in pixels to give the satellite mask.\n\n plot : bool, optional\n Plot the result.\n\n verbose : bool, optional\n Print extra information to the terminal, mostly for debugging.\n\n Returns\n -------\n mask : ndarray\n Boolean array marking the satellite trail with `True`.\n\n Raises\n ------\n ImportError\n Missing scipy or skimage>=0.11 packages.\n\n IndexError\n Invalid subarray indices.\n\n ValueError\n Image has no positive values, trail subarray too small, or\n trail profile not found.\n\n \"\"\"\n if not HAS_OPDEP:\n raise ImportError('Missing scipy or skimage>=0.11 packages')\n\n if verbose:\n t_beg = time.time()\n\n fname = '{0}[{1}]'.format(filename, ext)\n image = fits.getdata(filename, ext)\n\n dx = image.max()\n if dx <= 0:\n raise ValueError('Image has no positive values')\n\n # rescale the image\n image = image / dx\n # make sure everything is at least 0\n image[image < 0] = 0\n\n (x0, y0), (x1, y1) = trail_coords # p0, p1\n\n # Find out how much to rotate the image\n rad = np.arctan2(y1 - y0, x1 - x0)\n newrad = (np.pi * 2) - rad\n deg = np.degrees(rad)\n\n if verbose:\n print('Rotation: {0}'.format(deg))\n\n rotate = transform.rotate(image, deg, resize=True, order=order)\n\n if plot and plt is not None:\n plt.ion()\n mean = np.median(image)\n stddev = image.std()\n lower = mean - stddev\n upper = mean + stddev\n\n fig1, ax1 = plt.subplots()\n ax1.imshow(image, vmin=lower, vmax=upper, cmap=plt.cm.gray)\n ax1.set_title(fname)\n\n fig2, ax2 = plt.subplots()\n ax2.imshow(rotate, vmin=lower, vmax=upper, cmap=plt.cm.gray)\n ax2.set_title('{0} rotated by {1} deg'.format(fname, deg))\n\n plt.draw()\n\n # Will do all of this in the loop, but want to make sure there is a\n # good point first and that there is indeed a profile to fit.\n # get starting point\n sx, sy = _rotate_point((x0, y0), newrad, image.shape, rotate.shape)\n\n # start with one subarray around p0\n dx = int(subwidth / 2)\n ix0, ix1, iy0, iy1 = _get_valid_indices(\n rotate.shape, sx - dx, sx + dx, sy - sublen, sy + sublen)\n subr = rotate[iy0:iy1, ix0:ix1]\n if len(subr) <= sublen:\n raise ValueError('Trail subarray size is {0} but expected {1} or '\n 'larger'.format(len(subr), sublen))\n\n # Flatten the array so we are looking along rows\n # Take median of each row, should filter out most outliers\n # This list will get appended in the loop\n medarr = np.median(subr, axis=1)\n flat = [medarr]\n\n # get the outliers\n # mean = biweight_location(medarr)\n mean = sigma_clipped_stats(medarr)[0]\n stddev = biweight_midvariance(medarr)\n\n # only flag things that are sigma from the mean\n z = np.where(medarr > (mean + (sigma * stddev)))[0]\n\n if plot and plt is not None:\n fig1, ax1 = plt.subplots()\n ax1.plot(medarr, 'b.')\n ax1.plot(z, medarr[z], 'r.')\n ax1.set_xlabel('Index')\n ax1.set_ylabel('Value')\n ax1.set_title('Median array in flat[0]')\n plt.draw()\n\n # Make sure there is something in the first pass before trying to move on\n if len(z) < 1:\n raise ValueError(\n 'First look at finding a profile failed. '\n 'Nothing found at {0} from background! '\n 'Adjust parameters and try again.'.format(sigma))\n\n # get the bounds of the flagged points\n lower = z.min()\n upper = z.max()\n diff = upper - lower\n\n # add in a pading value to make sure all of the wings are accounted for\n lower = lower - pad\n upper = upper + pad\n\n # for plotting see how the profile was made (append to plot above)\n if plot and plt is not None:\n padind = np.arange(lower, upper)\n ax1.plot(padind, medarr[padind], 'yx')\n plt.draw()\n\n # start to create a mask\n mask = np.zeros(rotate.shape)\n lowerx, upperx, lowery, uppery = _get_valid_indices(\n mask.shape, np.floor(sx - subwidth), np.ceil(sx + subwidth),\n np.floor(sy - sublen + lower), np.ceil(sy - sublen + upper))\n mask[lowery:uppery, lowerx:upperx] = 1\n\n done = False\n first = True\n nextx = upperx # np.ceil(sx + subwidth)\n centery = np.ceil(lowery + diff) # np.ceil(sy - sublen + lower + diff)\n counter = 0\n\n while not done:\n # move to the right of the centerpoint first. do the same\n # as above but keep moving right until the edge is hit.\n ix0, ix1, iy0, iy1 = _get_valid_indices(\n rotate.shape, nextx - dx, nextx + dx,\n centery - sublen, centery + sublen)\n subr = rotate[iy0:iy1, ix0:ix1]\n\n # determines the edge, if the subr is not good, then the edge was\n # hit.\n if 0 in subr.shape:\n if verbose:\n print('Hit edge, subr shape={0}, first={1}'.format(\n subr.shape, first))\n if first:\n first = False\n centery = sy\n nextx = sx\n else:\n done = True\n continue\n\n medarr = np.median(subr, axis=1)\n flat.append(medarr)\n\n # mean = biweight_location(medarr)\n mean = sigma_clipped_stats(medarr, sigma=sigma)[0]\n # Might give RuntimeWarning\n stddev = biweight_midvariance(medarr)\n z = np.where(medarr > (mean + (sigma * stddev)))[0]\n\n if len(z) < 1:\n if first:\n if verbose:\n print('No good profile found for counter={0}. Start '\n 'moving left from starting point.'.format(counter))\n centery = sy\n nextx = sx\n first = False\n else:\n if verbose:\n print('z={0} is less than 1, subr shape={1}, '\n 'we are done'.format(z, subr.shape))\n done = True\n continue\n\n # get the bounds of the flagged points\n lower = z.min()\n upper = z.max()\n diff = upper - lower\n\n # add in a pading value to make sure all of the wings\n # are accounted for\n lower = np.floor(lower - pad)\n upper = np.ceil(upper + pad)\n lowerx, upperx, lowery, uppery = _get_valid_indices(\n mask.shape,\n np.floor(nextx - subwidth),\n np.ceil(nextx + subwidth),\n np.floor(centery - sublen + lower),\n np.ceil(centery - sublen + upper))\n mask[lowery:uppery, lowerx:upperx] = 1\n\n # lower_p = (lowerx, lowery)\n upper_p = (upperx, uppery)\n # lower_t = _rotate_point(\n # lower_p, newrad, image.shape, rotate.shape, reverse=True)\n upper_t = _rotate_point(\n upper_p, newrad, image.shape, rotate.shape, reverse=True)\n\n # lowy = np.floor(lower_t[1])\n highy = np.ceil(upper_t[1])\n # lowx = np.floor(lower_t[0])\n highx = np.ceil(upper_t[0])\n\n # Reset the next subr to be at the center of the profile\n if first:\n nextx = nextx + dx\n centery = lowery + diff # centery - sublen + lower + diff\n\n if (nextx + subwidth) > rotate.shape[1]:\n if verbose:\n print('Hit rotate edge at counter={0}'.format(counter))\n first = False\n elif (highy > image.shape[0]) or (highx > image.shape[1]):\n if verbose:\n print('Hit image edge at counter={0}'.format(counter))\n first = False\n\n if not first:\n centery = sy\n nextx = sx\n\n # Not first, this is the pass the other way.\n else:\n nextx = nextx - dx\n centery = lowery + diff # centery - sublen + lower + diff\n\n if (nextx - subwidth) < 0:\n if verbose:\n print('Hit rotate edge at counter={0}'.format(counter))\n done = True\n elif (highy > image.shape[0]) or (highx > image.shape[1]):\n if verbose:\n print('Hit image edge at counter={0}'.format(counter))\n done = True\n\n counter += 1\n\n # make sure it does not try to go infinetly\n if counter > 500:\n if verbose:\n print('Too many loops, exiting')\n done = True\n # End while\n\n rot = transform.rotate(mask, -deg, resize=True, order=1)\n ix0 = (rot.shape[1] - image.shape[1]) / 2\n iy0 = (rot.shape[0] - image.shape[0]) / 2\n lowerx, upperx, lowery, uppery = _get_valid_indices(\n rot.shape, ix0, image.shape[1] + ix0, iy0, image.shape[0] + iy0)\n mask = rot[lowery:uppery, lowerx:upperx]\n\n if mask.shape != image.shape:\n warnings.warn(\n 'Output mask shape is {0} but input image shape is '\n '{1}'.format(mask.shape, image.shape), AstropyUserWarning)\n\n # Change to boolean mask\n mask = mask.astype(np.bool)\n\n if plot and plt is not None:\n # debugging array\n test = image.copy()\n test[mask] = 0\n\n mean = np.median(test)\n stddev = test.std()\n lower = mean - stddev\n upper = mean + stddev\n\n fig1, ax1 = plt.subplots()\n ax1.imshow(test, vmin=lower, vmax=upper, cmap=plt.cm.gray)\n ax1.set_title('Masked image')\n\n fig2, ax2 = plt.subplots()\n ax2.imshow(mask, cmap=plt.cm.gray)\n ax2.set_title('DQ mask')\n\n plt.draw()\n\n if verbose:\n t_end = time.time()\n print('Run time: {0} s'.format(t_end - t_beg))\n\n return mask", "response": "Create a DQ mask for an image for a given satellite trail."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update_dq(filename, ext, mask, dqval=16384, verbose=True):\n with fits.open(filename, mode='update') as pf:\n dqarr = pf[ext].data\n old_mask = (dqval & dqarr) != 0 # Existing flagged trails\n new_mask = mask & ~old_mask # Only flag previously unflagged trails\n npix_updated = np.count_nonzero(new_mask)\n\n # Update DQ extension only if necessary\n if npix_updated > 0:\n pf[ext].data[new_mask] += dqval\n pf['PRIMARY'].header.add_history('{0} satdet v{1}({2})'.format(\n time.ctime(), __version__, __vdate__))\n pf['PRIMARY'].header.add_history(\n ' Updated {0} px in EXT {1} with DQ={2}'.format(\n npix_updated, ext, dqval))\n\n if verbose:\n fname = '{0}[{1}]'.format(filename, ext)\n\n print('DQ flag value is {0}'.format(dqval))\n print('Input... flagged NPIX={0}'.format(np.count_nonzero(mask)))\n print('Existing flagged NPIX={0}'.format(np.count_nonzero(old_mask)))\n print('Newly... flagged NPIX={0}'.format(npix_updated))\n\n if npix_updated > 0:\n print('{0} updated'.format(fname))\n else:\n print('No updates necessary for {0}'.format(fname))", "response": "Update the given image and DQ extension with the given mask and flag."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef detsat(searchpattern, chips=[1, 4], n_processes=4, sigma=2.0,\n low_thresh=0.1, h_thresh=0.5, small_edge=60, line_len=200,\n line_gap=75, percentile=(4.5, 93.0), buf=200, plot=False,\n verbose=True):\n \"\"\"Find satellite trails in the given images and extensions.\n The trails are calculated using Probabilistic Hough Transform.\n\n .. note::\n\n The trail endpoints found here are crude approximations.\n Use :func:`make_mask` to create the actual DQ mask for the trail(s)\n of interest.\n\n Parameters\n ----------\n searchpattern : str\n Search pattern for input FITS images, as accepted by\n :py:func:`glob.glob`.\n\n chips : list\n List of extensions for science data, as accepted by\n ``astropy.io.fits``.\n The default values of ``[1, 4]`` are tailored for ACS/WFC.\n\n n_processes : int\n Number of processes for multiprocessing, which is only useful\n if you are processing a lot of images or extensions.\n If 1 is given, no multiprocessing is done.\n\n sigma : float, optional\n The size of a Gaussian filter to use before edge detection.\n The default is 2, which is good for almost all images.\n\n low_thresh : float, optional\n The lower threshold for hysteresis linking of edge pieces.\n This should be between 0 and 1, and less than ``h_thresh``.\n\n h_thresh : float, optional\n The upper threshold for hysteresis linking of edge pieces.\n This should be between 0 and 1, and greater than ``low_thresh``.\n\n small_edge : int, optional\n Size of perimeter of small objects to remove in edge image.\n This significantly reduces noise before doing Hough Transform.\n If it is set too high, you will remove the edge of the\n satellite you are trying to find.\n\n line_len : int, optional\n Minimum line length for Probabilistic Hough Transform to fit.\n\n line_gap : int, optional\n The largest gap in points allowed for the Probabilistic\n Hough Transform.\n\n percentile : tuple of float, optional\n The percent boundaries to scale the image to before\n creating edge image.\n\n buf : int, optional\n How close to the edge of the image the satellite trail has to\n be to be considered a trail.\n\n plot : bool, optional\n Make plots of edge image, Hough space transformation, and\n rescaled image. This is only applicable if ``n_processes=1``.\n\n verbose : bool, optional\n Print extra information to the terminal, mostly for debugging.\n In multiprocessing mode, info from individual process is not printed.\n\n Returns\n -------\n results : dict\n Dictionary mapping ``(filename, ext)`` to an array of endpoints of\n line segments in the format of ``[[x0, y0], [x1, y1]]`` (if found) or\n an empty array (if not). These are the segments that have been\n identified as making up part of a satellite trail.\n\n errors : dict\n Dictionary mapping ``(filename, ext)`` to the error message explaining\n why processing failed.\n\n Raises\n ------\n ImportError\n Missing scipy or skimage>=0.11 packages.\n\n \"\"\"\n if not HAS_OPDEP:\n raise ImportError('Missing scipy or skimage>=0.11 packages')\n\n if verbose:\n t_beg = time.time()\n\n files = glob.glob(searchpattern)\n n_files = len(files)\n n_chips = len(chips)\n n_tot = n_files * n_chips\n n_cpu = multiprocessing.cpu_count()\n results = {}\n errors = {}\n\n if verbose:\n print('{0} file(s) found...'.format(n_files))\n\n # Nothing to do\n if n_files < 1 or n_chips < 1:\n return results, errors\n\n # Adjust number of processes\n if n_tot < n_processes:\n n_processes = n_tot\n if n_processes > n_cpu:\n n_processes = n_cpu\n\n # No multiprocessing\n if n_processes == 1:\n for fil in files:\n for chip in chips:\n if verbose:\n print('\\nProcessing {0}[{1}]...'.format(fil, chip))\n\n key = (fil, chip)\n try:\n result = _detsat_one(\n fil, chip, sigma=sigma,\n low_thresh=low_thresh, h_thresh=h_thresh,\n small_edge=small_edge, line_len=line_len,\n line_gap=line_gap, percentile=percentile, buf=buf,\n plot=plot, verbose=verbose)\n except Exception as e:\n errmsg = '{0}: {1}'.format(type(e), str(e))\n errors[key] = errmsg\n if verbose:\n print(errmsg)\n else:\n results[key] = result\n if verbose:\n print()\n\n # Multiprocessing.\n # The work queue is for things that need to be done and is shared by all\n # processes. When a worker finishes, its output is put into done queue.\n else:\n if verbose:\n print('Using {0} processes'.format(n_processes))\n\n work_queue = Queue()\n done_queue = Queue()\n processes = []\n\n for fil in files:\n for chip in chips:\n work_queue.put((fil, chip))\n\n for w in range(n_processes):\n p = Process(\n target=_satdet_worker, args=(work_queue, done_queue), kwargs={\n 'sigma': sigma, 'low_thresh': low_thresh,\n 'h_thresh': h_thresh, 'small_edge': small_edge,\n 'line_len': line_len, 'line_gap': line_gap,\n 'percentile': percentile, 'buf': buf})\n p.start()\n processes.append(p)\n work_queue.put('STOP')\n\n for p in processes:\n p.join()\n\n done_queue.put('STOP')\n\n # return a dictionary of lists\n for status in iter(done_queue.get, 'STOP'):\n key = (status[1], status[2])\n if status[0]: # Success\n results[key] = status[3]\n else: # Failed\n errors[key] = status[3]\n\n if verbose:\n if len(results) > 0:\n print('Number of trail segment(s) found:')\n for key in sorted(results):\n print(' {0}[{1}]: {2}'.format(\n key[0], key[1], len(results[key])))\n if len(errors) > 0:\n print('These have errors:')\n for key in sorted(errors):\n print(' {0}[{1}]'.format(key[0], key[1]))\n\n if verbose:\n t_end = time.time()\n print('Total run time: {0} s'.format(t_end - t_beg))\n\n return results, errors", "response": "This function calculates the DQ mask for the given searchpattern and chips."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef on(self, event: str) -> Callable:\n\n def outer(func):\n self.add_event(func, event)\n\n @wraps(func)\n def wrapper(*args, **kwargs):\n return func(*args, **kwargs)\n\n return wrapper\n\n return outer", "response": "Decorator for subscribing a function to a specific event."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a function to an event.", "response": "def add_event(self, func: Callable, event: str) -> None:\n \"\"\" Adds a function to a event.\n\n :param func: The function to call when event is emitted\n :type func: Callable\n\n :param event: Name of the event.\n :type event: str\n \"\"\"\n self._events[event].add(func)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef emit(self, event: str, *args, **kwargs) -> None:\n threads = kwargs.pop('threads', None)\n\n if threads:\n\n events = [\n Thread(target=f, args=args, kwargs=kwargs) for f in\n self._event_funcs(event)\n ]\n\n for event in events:\n event.start()\n\n else:\n for func in self._event_funcs(event):\n func(*args, **kwargs)", "response": "Emits an event and runs the subscribed functions."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef remove_event(self, func_name: str, event: str) -> None:\n event_funcs_copy = self._events[event].copy()\n\n for func in self._event_funcs(event):\n if func.__name__ == func_name:\n event_funcs_copy.remove(func)\n\n if self._events[event] == event_funcs_copy:\n err_msg = \"function doesn't exist inside event {} \".format(event)\n raise EventDoesntExist(err_msg)\n else:\n self._events[event] = event_funcs_copy", "response": "Removes a subscribed function from a specific event."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _event_funcs(self, event: str) -> Iterable[Callable]:\n for func in self._events[event]:\n yield func", "response": "Returns an iterable of the functions subscribed to an event."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _event_func_names(self, event: str) -> List[str]:\n return [func.__name__ for func in self._events[event]]", "response": "Returns the string name of each function subscribed to an event."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the total amount of subscribed events.", "response": "def _subscribed_event_count(self) -> int:\n \"\"\" Returns the total amount of subscribed events.\n\n :return: Integer amount events.\n :rtype: int\n \"\"\"\n event_counter = Counter() # type: Dict[Any, int]\n\n for key, values in self._events.items():\n event_counter[key] = len(values)\n\n return sum(event_counter.values())"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nperforming a stripe correction on the input image.", "response": "def perform_correction(image, output, stat=\"pmode1\", maxiter=15, sigrej=2.0,\n lower=None, upper=None, binwidth=0.3,\n mask=None, dqbits=None,\n rpt_clean=0, atol=0.01, clobber=False, verbose=True):\n \"\"\"\n Clean each input image.\n\n Parameters\n ----------\n image : str\n Input image name.\n\n output : str\n Output image name.\n\n mask : `numpy.ndarray`\n Mask array.\n\n maxiter, sigrej, clobber\n See :func:`clean`.\n\n dqbits : int, str, or None\n Data quality bits to be considered as \"good\" (or \"bad\").\n See :func:`clean` for more details.\n\n rpt_clean : int\n An integer indicating how many *additional* times stripe cleaning\n should be performed on the input image. Default = 0.\n\n atol : float, None\n The threshold for maximum absolute value of bias stripe correction\n below which repeated cleanings can stop. When `atol` is `None`\n cleaning will be repeated `rpt_clean` number of times.\n Default = 0.01 [e].\n\n verbose : bool\n Print informational messages. Default = True.\n\n \"\"\"\n # construct the frame to be cleaned, including the\n # associated data stuctures needed for cleaning\n frame = StripeArray(image)\n\n # combine user mask with image's DQ array:\n mask = _mergeUserMaskAndDQ(frame.dq, mask, dqbits)\n\n # Do the stripe cleaning\n Success, NUpdRows, NMaxIter, Bkgrnd, STDDEVCorr, MaxCorr, Nrpt = clean_streak(\n frame, stat=stat, maxiter=maxiter, sigrej=sigrej,\n lower=lower, upper=upper, binwidth=binwidth, mask=mask,\n rpt_clean=rpt_clean, atol=atol, verbose=verbose\n )\n\n if Success:\n if verbose:\n LOG.info('perform_correction - ===== Overall statistics for '\n 'de-stripe corrections: =====')\n\n if (STDDEVCorr > 1.5*0.9):\n LOG.warning('perform_correction - STDDEV of applied de-stripe '\n 'corrections ({:.3g}) exceeds\\nknown bias striping '\n 'STDDEV of 0.9e (see ISR ACS 2011-05) more than '\n '1.5 times.'.format(STDDEVCorr))\n\n elif verbose:\n LOG.info('perform_correction - STDDEV of applied de-stripe '\n 'corrections {:.3g}.'.format(STDDEVCorr))\n\n if verbose:\n LOG.info('perform_correction - Estimated background: '\n '{:.5g}.'.format(Bkgrnd))\n LOG.info('perform_correction - Maximum applied correction: '\n '{:.3g}.'.format(MaxCorr))\n LOG.info('perform_correction - Effective number of clipping '\n 'iterations: {}.'.format(NMaxIter))\n LOG.info('perform_correction - Effective number of additional '\n '(repeated) cleanings: {}.'.format(Nrpt))\n LOG.info('perform_correction - Total number of corrected rows: '\n '{}.'.format(NUpdRows))\n\n frame.write_corrected(output, clobber=clobber)\n frame.close()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclean the object store for a given image.", "response": "def clean_streak(image, stat=\"pmode1\", maxiter=15, sigrej=2.0,\n lower=None, upper=None, binwidth=0.3, mask=None,\n rpt_clean=0, atol=0.01, verbose=True):\n \"\"\"\n Apply destriping algorithm to input array.\n\n Parameters\n ----------\n image : `StripeArray` object\n Arrays are modifed in-place.\n\n stat : str\n Statistics for background computations\n (see :py:func:`clean` for more details)\n\n mask : `numpy.ndarray`\n Mask array. Pixels with zero values are masked out.\n\n maxiter, sigrej : see `clean`\n\n rpt_clean : int\n An integer indicating how many *additional* times stripe cleaning\n should be performed on the input image. Default = 0.\n\n atol : float, None\n The threshold for maximum absolute value of bias stripe correction\n below which repeated cleanings can stop. When `atol` is `None`\n cleaning will be repeated `rpt_clean` number of times.\n Default = 0.01 [e].\n\n verbose : bool\n Print informational messages. Default = True.\n\n Returns\n -------\n Success : bool\n Indicates successful execution.\n\n NUpdRows : int\n Number of updated rows in the image.\n\n NMaxIter : int\n Maximum number of clipping iterations performed on image rows.\n\n Bkgrnd, STDDEVCorr, MaxCorr : float\n Background, standard deviation of corrections and maximum correction\n applied to the non-flat-field-corrected (i.e., RAW) image rows.\n\n Nrpt : int\n Number of *additional* (performed *after* initial run) cleanings.\n\n \"\"\"\n # Optional package dependency\n try:\n from stsci.imagestats import ImageStats\n except ImportError:\n ImageStats = None\n\n if mask is not None and image.science.shape != mask.shape:\n raise ValueError('Mask shape does not match science data shape')\n\n Nrpt = 0\n warn_maxiter = False\n NUpdRows = 0\n NMaxIter = 0\n STDDEVCorr = 0.0\n MaxCorr = 0.0\n wmean = 0.0\n\n stat = stat.lower().strip()\n if stat not in ['pmode1', 'pmode2', 'mean', 'mode', 'median', 'midpt']:\n raise ValueError(\"Unsupported value for 'stat'.\")\n\n # array to hold the stripe amplitudes\n corr = np.empty(image.science.shape[0], dtype=np.float64)\n\n # array to hold cumulative stripe amplitudes and latest row npix:\n cumcorr = np.zeros(image.science.shape[0], dtype=np.float64)\n cnpix = np.zeros(image.science.shape[0], dtype=np.int)\n\n # other arrays\n corr_scale = np.empty(image.science.shape[0], dtype=np.float64)\n npix = np.empty(image.science.shape[0], dtype=np.int)\n sigcorr2 = np.zeros(image.science.shape[0], dtype=np.float64)\n updrows = np.zeros(image.science.shape[0], dtype=np.int)\n\n # for speed-up and to reduce rounding errors in ERR computations,\n # keep a copy of the squared error array:\n imerr2 = image.err**2\n\n # arrays for detecting oscillatory behaviour:\n nonconvi0 = np.arange(image.science.shape[0])\n corr0 = np.zeros(image.science.shape[0], dtype=np.float64)\n\n if stat == 'pmode1':\n # SExtractor-esque central value statistic; slightly sturdier against\n # skewness of pixel histogram due to faint source flux\n def getcorr(): return (2.5 * SMedian - 1.5 * SMean)\n\n elif stat == 'pmode2':\n # \"Original Pearson\"-ian estimate for mode:\n def getcorr(): return (3.0 * SMedian - 2.0 * SMean)\n\n elif stat == 'mean':\n def getcorr(): return (SMean)\n\n elif stat == 'median':\n def getcorr(): return (SMedian)\n\n elif stat == 'mode':\n if ImageStats is None:\n raise ImportError('stsci.imagestats is missing')\n\n def getcorr():\n imstat = ImageStats(image.science[i][BMask], 'mode',\n lower=lower, upper=upper, nclip=0)\n assert(imstat.npix == NPix)\n return (imstat.mode)\n\n elif stat == 'midpt':\n if ImageStats is None:\n raise ImportError('stsci.imagestats is missing')\n\n def getcorr():\n imstat = ImageStats(image.science[i][BMask], 'midpt',\n lower=lower, upper=upper, nclip=0)\n assert(imstat.npix == NPix)\n return (imstat.midpt)\n\n nmax_rpt = 1 if rpt_clean is None else max(1, rpt_clean+1)\n\n for rpt in range(nmax_rpt):\n Nrpt += 1\n\n if verbose:\n if Nrpt <= 1:\n if nmax_rpt > 1:\n LOG.info(\"clean_streak - Performing initial image bias \"\n \"de-stripe:\")\n else:\n LOG.info(\"clean_streak - Performing image bias de-stripe:\")\n else:\n LOG.info(\"clean_streak - Performing repeated image bias \"\n \"de-stripe #{}:\".format(Nrpt - 1))\n\n # reset accumulators and arrays:\n corr[:] = 0.0\n corr_scale[:] = 0.0\n npix[:] = 0\n\n tcorr = 0.0\n tnpix = 0\n tnpix2 = 0\n NMaxIter = 0\n\n # loop over rows to fit the stripe amplitude\n mask_arr = None\n for i in range(image.science.shape[0]):\n if mask is not None:\n mask_arr = mask[i]\n\n # row-by-row iterative sigma-clipped mean;\n # sigma, iters are adjustable\n SMean, SSig, SMedian, NPix, NIter, BMask = djs_iterstat(\n image.science[i], MaxIter=maxiter, SigRej=sigrej,\n Min=lower, Max=upper, Mask=mask_arr, lineno=i+1\n )\n\n if NPix > 0:\n corr[i] = getcorr()\n npix[i] = NPix\n corr_scale[i] = 1.0 / np.average(image.invflat[i][BMask])\n sigcorr2[i] = corr_scale[i]**2 * \\\n np.sum((image.err[i][BMask])**2)/NPix**2\n cnpix[i] = NPix\n tnpix += NPix\n tnpix2 += NPix*NPix\n tcorr += corr[i] * NPix\n\n if NIter > NMaxIter:\n NMaxIter = NIter\n\n if tnpix <= 0:\n LOG.warning('clean_streak - No good data points; cannot de-stripe.')\n return False, 0, 0, 0.0, 0.0, 0\n\n if NMaxIter >= maxiter:\n warn_maxiter = True\n\n # require that bias stripe corrections have zero mean:\n # 1. compute weighted background of the flat-fielded image:\n wmean = tcorr / tnpix\n Bkgrnd = wmean\n # 2. estimate corrections:\n corr[npix > 0] -= wmean\n\n # convert corrections to the \"raw\" space:\n corr *= corr_scale\n\n # weighted mean and max value for current corrections\n # to the *RAW* image:\n trim_npix = npix[npix > 0]\n trim_corr = corr[npix > 0]\n cwmean = np.sum(trim_npix * trim_corr) / tnpix\n current_max_corr = np.amax(np.abs(trim_corr - cwmean))\n wvar = np.sum(trim_npix * (trim_corr - cwmean) ** 2) / tnpix\n uwvar = wvar / (1.0 - float(tnpix2) / float(tnpix) ** 2)\n STDDEVCorr = np.sqrt(uwvar)\n\n # keep track of total corrections:\n cumcorr += corr\n\n # apply corrections row-by-row\n for i in range(image.science.shape[0]):\n if npix[i] < 1:\n continue\n updrows[i] = 1\n\n ffdark = (image.dark[i] + image.flash[i]) * image.invflat[i]\n t1 = np.maximum(image.science[i] + ffdark, 0.0)\n\n # stripe is constant along the row, before flatfielding;\n # afterwards it has the shape of the inverse flatfield\n truecorr = corr[i] * image.invflat[i]\n #truecorr_sig2 = sigcorr2[i] * image.invflat[i]**2 # DEBUG\n\n # correct the SCI extension\n image.science[i] -= truecorr\n\n t2 = np.maximum(image.science[i] + ffdark, 0.0)\n\n T = (t1 - t2) * image.invflat[i]\n\n # correct the ERR extension\n # NOTE: np.abs() in the err array recomputation is used for safety\n # only and, in principle, assuming no errors have been made\n # in the derivation of the formula, np.abs() should not be\n # necessary.\n imerr2[i] -= T\n image.err[i] = np.sqrt(np.abs(imerr2[i]))\n # NOTE: for debugging purposes, one may want to uncomment\n # next line:\n #assert( np.all(imerr2 >= 0.0))\n\n if atol is not None:\n if current_max_corr < atol:\n break\n\n # detect oscilatory non-convergence:\n nonconvi = np.nonzero(np.abs(corr) > atol)[0]\n nonconvi_int = np.intersect1d(nonconvi, nonconvi0)\n if (nonconvi.shape[0] == nonconvi0.shape[0] and\n nonconvi.shape[0] == nonconvi_int.shape[0] and\n np.all(corr0[nonconvi]*corr[nonconvi] < 0.0) and Nrpt > 1):\n LOG.warning(\"clean_streak - Repeat bias stripe cleaning\\n\"\n \"process appears to be oscillatory for {:d} image \"\n \"rows.\\nTry to adjust 'sigrej', 'maxiter', and/or \"\n \"'dqbits' parameters.\\n\"\n \"In addition, consider using masks or adjust \"\n \"existing masks.\".format(nonconvi.shape[0]))\n break\n\n nonconvi0 = nonconvi.copy()\n corr0 = corr.copy()\n\n if verbose:\n if Nrpt <= 1:\n LOG.info(\"clean_streak - Image bias de-stripe: Done.\")\n else:\n LOG.info(\"clean_streak - Repeated (#{}) image bias de-stripe: \"\n \"Done.\".format(Nrpt - 1))\n\n if verbose and Nrpt > 1:\n LOG.info('clean_streak - ===== Repeated de-stripe \"residual\" '\n 'estimates: =====')\n LOG.info('clean_streak - STDDEV of the last applied de-stripe '\n 'corrections {:.3g}'.format(STDDEVCorr))\n LOG.info('clean_streak - Maximum of the last applied correction: '\n '{:.3g}.'.format(current_max_corr))\n\n # add (in quadratures) an error term associated with the accuracy of\n # bias stripe correction:\n truecorr_sig2 = ((sigcorr2 * (image.invflat ** 2).T).T).astype(image.err.dtype) # noqa\n\n # update the ERR extension\n image.err[:, :] = np.sqrt(np.abs(imerr2 + truecorr_sig2))\n\n if warn_maxiter:\n LOG.warning(\n 'clean_streak - Maximum number of clipping iterations '\n 'specified by the user ({}) has been reached.'.format(maxiter))\n\n # weighted mean, sample variance, and max value for\n # total (cummulative) corrections to the *RAW* image:\n trim_cnpix = cnpix[cnpix > 0]\n trim_cumcorr = cumcorr[cnpix > 0]\n tcnpix = np.sum(trim_cnpix)\n tcnpix2 = np.sum(trim_cnpix ** 2)\n cwmean = np.sum(trim_cnpix * trim_cumcorr) / tcnpix\n trim_cumcorr -= cwmean\n wvar = np.sum(trim_cnpix * trim_cumcorr ** 2) / tcnpix\n uwvar = wvar / (1.0 - float(tcnpix2) / float(tcnpix) ** 2)\n STDDEVCorr = np.sqrt(uwvar)\n MaxCorr = np.amax(np.abs(trim_cumcorr))\n\n NUpdRows = np.sum(updrows)\n\n return True, NUpdRows, NMaxIter, Bkgrnd, STDDEVCorr, MaxCorr, Nrpt-1"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncomputes the iteration statistics for a single image array.", "response": "def djs_iterstat(InputArr, MaxIter=10, SigRej=3.0,\n Max=None, Min=None, Mask=None, lineno=None):\n \"\"\"\n Iterative sigma-clipping.\n\n Parameters\n ----------\n InputArr : `numpy.ndarray`\n Input image array.\n\n MaxIter, SigRej : see `clean`\n\n Max, Min : float\n Max and min values for clipping.\n\n Mask : `numpy.ndarray`\n Mask array to indicate pixels to reject, in addition to clipping.\n Pixels where mask is zero will be rejected.\n If not given, all pixels will be used.\n\n lineno : int or None\n Line number to be used in log and/or warning messages.\n\n Returns\n -------\n FMean, FSig, FMedian, NPix : float\n Mean, sigma, and median of final result.\n\n NIter : int\n Number of performed clipping iterations\n\n BMask : `numpy.ndarray`\n Logical image mask from the final iteration.\n\n \"\"\"\n NGood = InputArr.size\n ArrShape = InputArr.shape\n if NGood == 0:\n imrow = _write_row_number(lineno=lineno, offset=0, pad=1)\n LOG.warning('djs_iterstat - No data points given' + imrow)\n return 0, 0, 0, 0, 0, None\n if NGood == 1:\n imrow = _write_row_number(lineno=lineno, offset=0, pad=1)\n LOG.warning('djs_iterstat - Only one data point; '\n 'cannot compute stats{0}'.format(imrow))\n return 0, 0, 0, 0, 0, None\n if np.unique(InputArr).size == 1:\n imrow = _write_row_number(lineno=lineno, offset=0, pad=1)\n LOG.warning('djs_iterstat - Only one value in data; '\n 'cannot compute stats{0}'.format(imrow))\n return 0, 0, 0, 0, 0, None\n\n # Determine Max and Min\n if Max is None:\n Max = InputArr.max()\n if Min is None:\n Min = InputArr.min()\n\n # Use all pixels if no mask is provided\n if Mask is None:\n Mask = np.ones(ArrShape, dtype=np.byte)\n else:\n Mask = Mask.copy()\n\n # Reject those above Max and those below Min\n Mask[InputArr > Max] = 0\n Mask[InputArr < Min] = 0\n\n FMean = np.sum(1.0 * InputArr * Mask) / NGood\n FSig = np.sqrt(np.sum((1.0 * InputArr - FMean) ** 2 * Mask) / (NGood - 1))\n\n NLast = -1\n Iter = 0\n NGood = np.sum(Mask)\n if NGood < 2:\n imrow = _write_row_number(lineno=lineno, offset=0, pad=1)\n LOG.warning('djs_iterstat - No good data points; '\n 'cannot compute stats{0}'.format(imrow))\n return 0, 0, 0, 0, 0, None\n\n SaveMask = Mask.copy()\n if Iter >= MaxIter: # to support MaxIter=0\n NLast = NGood\n\n while (Iter < MaxIter) and (NLast != NGood) and (NGood >= 2):\n LoVal = FMean - SigRej * FSig\n HiVal = FMean + SigRej * FSig\n\n Mask[InputArr < LoVal] = 0\n Mask[InputArr > HiVal] = 0\n NLast = NGood\n npix = np.sum(Mask)\n\n if npix >= 2:\n FMean = np.sum(1.0 * InputArr * Mask) / npix\n FSig = np.sqrt(np.sum(\n (1.0 * InputArr - FMean) ** 2 * Mask) / (npix - 1))\n SaveMask = Mask.copy() # last mask used for computation of mean\n NGood = npix\n Iter += 1\n else:\n break\n\n logical_mask = SaveMask.astype(np.bool)\n\n if NLast > 1:\n FMedian = np.median(InputArr[logical_mask])\n NLast = NGood\n else:\n FMedian = FMean\n\n return FMean, FSig, FMedian, NLast, Iter, logical_mask"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run(configobj=None):\n clean(configobj['input'],\n suffix=configobj['suffix'],\n stat=configobj['stat'],\n maxiter=configobj['maxiter'],\n sigrej=configobj['sigrej'],\n lower=configobj['lower'],\n upper=configobj['upper'],\n binwidth=configobj['binwidth'],\n mask1=configobj['mask1'],\n mask2=configobj['mask2'],\n dqbits=configobj['dqbits'],\n rpt_clean=configobj['rpt_clean'],\n atol=configobj['atol'],\n cte_correct=configobj['cte_correct'],\n clobber=configobj['clobber'],\n verbose=configobj['verbose'])", "response": "run a TEAL clean function"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef configure_arrays(self):\n self.science = self.hdulist['sci', 1].data\n self.err = self.hdulist['err', 1].data\n self.dq = self.hdulist['dq', 1].data\n if (self.ampstring == 'ABCD'):\n self.science = np.concatenate(\n (self.science, self.hdulist['sci', 2].data[::-1, :]), axis=1)\n self.err = np.concatenate(\n (self.err, self.hdulist['err', 2].data[::-1, :]), axis=1)\n self.dq = np.concatenate(\n (self.dq, self.hdulist['dq', 2].data[::-1, :]), axis=1)\n self.ingest_dark()\n self.ingest_flash()\n self.ingest_flatfield()", "response": "Get the SCI and ERR data."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ingest_flatfield(self):\n\n self.invflat = extract_flatfield(\n self.hdulist[0].header, self.hdulist[1])\n\n # If BIAS or DARK, set flatfield to unity\n if self.invflat is None:\n self.invflat = np.ones_like(self.science)\n return\n\n # Apply the flatfield if necessary\n if self.flatcorr != 'COMPLETE':\n self.science = self.science * self.invflat\n self.err = self.err * self.invflat", "response": "Ingest the unity un"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write_corrected(self, output, clobber=False):\n\n # un-apply the flatfield if necessary\n if self.flatcorr != 'COMPLETE':\n self.science = self.science / self.invflat\n self.err = self.err / self.invflat\n\n # un-apply the post-flash if necessary\n if self.flshcorr != 'COMPLETE':\n self.science = self.science + self.flash\n\n # un-apply the dark if necessary\n if self.darkcorr != 'COMPLETE':\n self.science = self.science + self.dark\n\n # reverse the amp merge\n if (self.ampstring == 'ABCD'):\n tmp_1, tmp_2 = np.split(self.science, 2, axis=1)\n self.hdulist['sci', 1].data = tmp_1.copy()\n self.hdulist['sci', 2].data = tmp_2[::-1, :].copy()\n\n tmp_1, tmp_2 = np.split(self.err, 2, axis=1)\n self.hdulist['err', 1].data = tmp_1.copy()\n self.hdulist['err', 2].data = tmp_2[::-1, :].copy()\n else:\n self.hdulist['sci', 1].data = self.science.copy()\n self.hdulist['err', 1].data = self.err.copy()\n\n # Write the output\n self.hdulist.writeto(output, overwrite=clobber)", "response": "Write out the destriped data."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef calacs(input_file, exec_path=None, time_stamps=False, temp_files=False,\n verbose=False, debug=False, quiet=False, single_core=False,\n exe_args=None):\n \"\"\"\n Run the calacs.e executable as from the shell.\n\n By default this will run the calacs given by 'calacs.e'.\n\n Parameters\n ----------\n input_file : str\n Name of input file.\n\n exec_path : str, optional\n The complete path to a calacs executable.\n\n time_stamps : bool, optional\n Set to True to turn on the printing of time stamps.\n\n temp_files : bool, optional\n Set to True to have CALACS save temporary files.\n\n verbose : bool, optional\n Set to True for verbose output.\n\n debug : bool, optional\n Set to True to turn on debugging output.\n\n quiet : bool, optional\n Set to True for quiet output.\n\n single_core : bool, optional\n CTE correction in CALACS will by default try to use all available\n CPUs on your computer. Set this to True to force the use of just\n one CPU.\n\n exe_args : list, optional\n Arbitrary arguments passed to underlying executable call.\n Note: Implementation uses subprocess.call and whitespace is not\n permitted. E.g. use exe_args=['--nThreads', '1']\n\n \"\"\"\n if exec_path:\n if not os.path.exists(exec_path):\n raise OSError('Executable not found: ' + exec_path)\n\n call_list = [exec_path]\n else:\n call_list = ['calacs.e']\n\n if time_stamps:\n call_list.append('-t')\n\n if temp_files:\n call_list.append('-s')\n\n if verbose:\n call_list.append('-v')\n\n if debug:\n call_list.append('-d')\n\n if quiet:\n call_list.append('-q')\n\n if single_core:\n call_list.append('-1')\n\n if not os.path.exists(input_file):\n raise IOError('Input file not found: ' + input_file)\n\n call_list.append(input_file)\n\n if exe_args:\n call_list.extend(exe_args)\n\n subprocess.check_call(call_list)", "response": "This function runs the calacs. e executable as from the shell."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrunning TEAL s calacs function.", "response": "def run(configobj=None):\n \"\"\"\n TEAL interface for the `calacs` function.\n\n \"\"\"\n calacs(configobj['input_file'],\n exec_path=configobj['exec_path'],\n time_stamps=configobj['time_stamps'],\n temp_files=configobj['temp_files'],\n verbose=configobj['verbose'],\n debug=configobj['debug'],\n quiet=configobj['quiet'],\n single_core=configobj['single_core']\n )"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef has_any_permissions(self, user):\n for perm in self.get_all_model_permissions():\n if self.has_specific_permission(user, perm.codename):\n return True\n return False", "response": "Return a boolean to indicate whether the supplied user has any permissions at all on the associated model."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a queryset of valid parent pages for the current user.", "response": "def get_valid_parent_pages(self, user):\n \"\"\"\n Identifies possible parent pages for the current user by first looking\n at allowed_parent_page_models() on self.model to limit options to the\n correct type of page, then checking permissions on those individual\n pages to make sure we have permission to add a subpage to it.\n \"\"\"\n # Start with empty qs\n parents_qs = Page.objects.none()\n\n # Add pages of the correct type\n for pt in self.model.allowed_parent_page_models():\n pt_items = Page.objects.type(pt)\n parents_qs = parents_qs | pt_items\n\n # Exclude pages that we can't add subpages to\n for page in parents_qs.all():\n if not page.permissions_for_user(user).can_add_subpage():\n parents_qs = parents_qs.exclude(pk=page.pk)\n\n return parents_qs"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning TEAL s rejection.", "response": "def run(configobj=None):\n \"\"\"\n TEAL interface for the `acsrej` function.\n\n \"\"\"\n acsrej(configobj['input'],\n configobj['output'],\n exec_path=configobj['exec_path'],\n time_stamps=configobj['time_stamps'],\n verbose=configobj['verbose'],\n shadcorr=configobj['shadcorr'],\n crrejtab=configobj['crrejtab'],\n crmask=configobj['crmask'],\n scalense=configobj['scalense'],\n initgues=configobj['initgues'],\n skysub=configobj['skysub'],\n crsigmas=configobj['crsigmas'],\n crradius=configobj['crradius'],\n crthresh=configobj['crthresh'],\n badinpdq=configobj['badinpdq'],\n readnoise_only=configobj['readnoise_only'])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef monkeypatch_os_fork_functions():\n builtin_function = type(''.join)\n if hasattr(os, 'fork') and isinstance(os.fork, builtin_function):\n global _orig_os_fork\n _orig_os_fork = os.fork\n os.fork = os_fork_wrapper\n if hasattr(os, 'forkpty') and isinstance(os.forkpty, builtin_function):\n global _orig_os_forkpty\n _orig_os_forkpty = os.forkpty\n os.forkpty = os_forkpty_wrapper", "response": "Monkey patches os. fork and os. forkpty functions to use ForkSafeLock to acquire all locks before forking and release them afterwards."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef atfork(prepare=None, parent=None, child=None):\n assert not prepare or callable(prepare)\n assert not parent or callable(parent)\n assert not child or callable(child)\n _fork_lock.acquire()\n try:\n if prepare:\n _prepare_call_list.append(prepare)\n if parent:\n _parent_call_list.append(parent)\n if child:\n _child_call_list.append(child)\n finally:\n _fork_lock.release()", "response": "A Python work - a - like of pthread_atfork."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _call_atfork_list(call_list):\n exception_list = []\n for func in call_list:\n try:\n func()\n except:\n exception_list.append(sys.exc_info())\n return exception_list", "response": "Given a list of callables in call_list save them and return a list of sys. exc_info tuples for each exception raised."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalling all parent after fork callables and print the list of exceptions.", "response": "def parent_after_fork_release():\n \"\"\"\n Call all parent after fork callables, release the lock and print\n all prepare and parent callback exceptions.\n \"\"\"\n prepare_exceptions = list(_prepare_call_exceptions)\n del _prepare_call_exceptions[:]\n exceptions = _call_atfork_list(_parent_call_list)\n _fork_lock.release()\n _print_exception_list(prepare_exceptions, 'before fork')\n _print_exception_list(exceptions, 'after fork from parent')"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprinting a list of sys. exc_info tuples to the output_file.", "response": "def _print_exception_list(exceptions, message, output_file=None):\n \"\"\"\n Given a list of sys.exc_info tuples, print them all using the traceback\n module preceeded by a message and separated by a blank line.\n \"\"\"\n output_file = output_file or sys.stderr\n message = 'Exception %s:\\n' % message\n for exc_type, exc_value, exc_traceback in exceptions:\n output_file.write(message)\n traceback.print_exception(exc_type, exc_value, exc_traceback,\n file=output_file)\n output_file.write('\\n')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrap os. forkpty to run atfork handlers.", "response": "def os_forkpty_wrapper():\n \"\"\"Wraps os.forkpty() to run atfork handlers.\"\"\"\n pid = None\n prepare_to_fork_acquire()\n try:\n pid, fd = _orig_os_forkpty()\n finally:\n if pid == 0:\n child_after_fork_release()\n else:\n parent_after_fork_release()\n return pid, fd"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrap an object with a Maybe instance.", "response": "def maybe(value):\n \"\"\"Wraps an object with a Maybe instance.\n\n >>> maybe(\"I'm a value\")\n Something(\"I'm a value\")\n\n >>> maybe(None);\n Nothing\n\n Testing for value:\n\n >>> maybe(\"I'm a value\").is_some()\n True\n >>> maybe(\"I'm a value\").is_none()\n False\n >>> maybe(None).is_some()\n False\n >>> maybe(None).is_none()\n True\n\n Simplifying IF statements:\n\n >>> maybe(\"I'm a value\").get()\n \"I'm a value\"\n\n >>> maybe(\"I'm a value\").or_else(lambda: \"No value\")\n \"I'm a value\"\n\n >>> maybe(None).get()\n Traceback (most recent call last):\n ...\n NothingValueError: No such element\n\n >>> maybe(None).or_else(lambda: \"value\")\n 'value'\n\n >>> maybe(None).or_else(\"value\")\n 'value'\n\n Wrap around values from object's attributes:\n\n class Person(object):\n def __init__(name):\n self.eran = name\n\n eran = maybe(Person('eran'))\n\n >>> eran.name\n Something('eran')\n >>> eran.phone_number\n Nothing\n >>> eran.phone_number.or_else('no phone number')\n 'no phone number'\n\n >>> maybe(4) + 8\n Something(12)\n >>> maybe(4) - 2\n Something(2)\n >>> maybe(4) * 2\n Something(8)\n\n And methods:\n\n >>> maybe('VALUE').lower().get()\n 'value'\n >>> maybe(None).invalid().method().or_else('unknwon')\n 'unknwon'\n\n Enabled easily using NestedDictionaries without having to worry\n if a value is missing.\n For example lets assume we want to load some value from the\n following dictionary:\n nested_dict = maybe({\n 'store': {\n 'name': 'MyStore',\n 'departments': {\n 'sales': { 'head_count': '10' }\n }\n }\n })\n\n >>> nested_dict['store']['name'].get()\n 'MyStore'\n >>> nested_dict['store']['address']\n Nothing\n >>> nested_dict['store']['address']['street'].or_else('No Address Specified')\n 'No Address Specified'\n >>> nested_dict['store']['address']['street'].or_none() is None\n True\n >>> nested_dict['store']['address']['street'].or_empty_list()\n []\n >>> nested_dict['store']['departments']['sales']['head_count'].or_else('0')\n '10'\n >>> nested_dict['store']['departments']['marketing']['head_count'].or_else('0')\n '0'\n\n \"\"\"\n if isinstance(value, Maybe):\n return value\n\n if value is not None:\n return Something(value)\n\n return Nothing()"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef inputcooker_store_queue(self, char):\n if type(char) in [type(()), type([]), type(\"\")]:\n for v in char:\n self.cookedq.put(v)\n else:\n self.cookedq.put(char)", "response": "Put the cooked data in the input queue ( no locking needed"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef streamserver_handle(cls, socket, address):\n '''Translate this class for use in a StreamServer'''\n request = cls.dummy_request()\n request._sock = socket\n server = None\n cls(request, address, server)", "response": "Translate this class for use in a StreamServer"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef check_channel_shell_request(self, channel):\n '''Request to start a shell on the given channel'''\n try:\n self.channels[channel].start()\n except KeyError:\n log.error('Requested to start a channel (%r) that was not previously set up.', channel)\n return False\n else:\n return True", "response": "Check if the channel is ready to start a shell on the given channel."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrequesting to allocate a PTY terminal.", "response": "def check_channel_pty_request(self, channel, term, width, height, pixelwidth,\n pixelheight, modes):\n '''Request to allocate a PTY terminal.'''\n #self.sshterm = term\n #print \"term: %r, modes: %r\" % (term, modes)\n log.debug('PTY requested. Setting up %r.', self.telnet_handler)\n pty_thread = Thread( target=self.start_pty_request, args=(channel, term, modes) )\n self.channels[channel] = pty_thread\n \n return True"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef download(url, file_name):\r\n r = requests.get(url, stream=True)\r\n file_size = int(r.headers['Content-length'])\r\n '''\r\n if py3:\r\n file_size = int(u.getheader(\"Content-Length\")[0])\r\n else:\r\n file_size = int(u.info().getheaders(\"Content-Length\")[0])\r\n '''\r\n file_exists = False\r\n if os.path.isfile(file_name):\r\n local_file_size = os.path.getsize(file_name)\r\n if local_file_size == file_size:\r\n sha1_file = file_name + '.sha1'\r\n if os.path.isfile(sha1_file):\r\n print('sha1 found')\r\n with open(sha1_file) as f:\r\n expected_sha1 = f.read()\r\n BLOCKSIZE = 65536\r\n sha1 = hashlib.sha1()\r\n with open(file_name) as f:\r\n buff = f.read(BLOCKSIZE)\r\n while len(buff) > 0:\r\n sha1.update(buff)\r\n buff = f.read(BLOCKSIZE)\r\n if expected_sha1 == sha1:\r\n file_exists = True\r\n else:\r\n print(\"File corrupt. Downloading again.\")\r\n os.remove(file_name)\r\n else:\r\n file_exists = True\r\n else:\r\n print(\"File corrupt. Downloading again.\")\r\n os.remove(file_name)\r\n if not file_exists:\r\n factor = int(math.floor(math.log(file_size) / math.log(1024)))\r\n display_file_size = str(file_size / 1024 ** factor) + \\\r\n ['B', 'KB', 'MB', 'GB', 'TB', 'PB'][factor]\r\n print(\"Source: \" + url)\r\n print(\"Destination \" + file_name)\r\n print(\"Size: \" + display_file_size)\r\n file_size_dl = 0\r\n block_sz = 8192\r\n f = open(file_name, 'wb')\r\n pbar = ProgressBar(file_size)\r\n for chunk in r.iter_content(chunk_size=block_sz):\r\n if not chunk:\r\n continue\r\n chunk_size = len(chunk)\r\n file_size_dl += chunk_size\r\n f.write(chunk)\r\n pbar.update(chunk_size)\r\n # status = r\"%10d [%3.2f%%]\" % (file_size_dl, file_size_dl * 100. / file_size)\r\n # status = status + chr(8)*(len(status)+1)\r\n # print(status)\r\n f.close()\r\n else:\r\n print(\"File already exists - \" + file_name)\r\n return True", "response": "Download a single file from a URL and save it to a file."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrequesting an authentication of a specific order.", "response": "def authenticate(\n self, end_user_ip, personal_number=None, requirement=None, **kwargs\n ):\n \"\"\"Request an authentication order. The :py:meth:`collect` method\n is used to query the status of the order.\n\n Note that personal number is not needed when authentication is to\n be done on the same device, provided that the returned\n ``autoStartToken`` is used to open the BankID Client.\n\n Example data returned:\n\n .. code-block:: json\n\n {\n \"orderRef\":\"131daac9-16c6-4618-beb0-365768f37288\",\n \"autoStartToken\":\"7c40b5c9-fa74-49cf-b98c-bfe651f9a7c6\"\n }\n\n :param end_user_ip: IP address of the user requesting\n the authentication.\n :type end_user_ip: str\n :param personal_number: The Swedish personal number in\n format YYYYMMDDXXXX.\n :type personal_number: str\n :param requirement: An optional dictionary stating how the signature\n must be created and verified. See BankID Relying Party Guidelines,\n section 13.5 for more details.\n :type requirement: dict\n :return: The order response.\n :rtype: dict\n :raises BankIDError: raises a subclass of this error\n when error has been returned from server.\n\n \"\"\"\n data = {\"endUserIp\": end_user_ip}\n if personal_number:\n data[\"personalNumber\"] = personal_number\n if requirement and isinstance(requirement, dict):\n data[\"requirement\"] = requirement\n # Handling potentially changed optional in-parameters.\n data.update(kwargs)\n response = self.client.post(self._auth_endpoint, json=data)\n\n if response.status_code == 200:\n return response.json()\n else:\n raise get_json_error_class(response)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sign(\n self,\n end_user_ip,\n user_visible_data,\n personal_number=None,\n requirement=None,\n user_non_visible_data=None,\n **kwargs\n ):\n \"\"\"Request an signing order. The :py:meth:`collect` method\n is used to query the status of the order.\n\n Note that personal number is not needed when signing is to be done\n on the same device, provided that the returned ``autoStartToken``\n is used to open the BankID Client.\n\n Example data returned:\n\n .. code-block:: json\n\n {\n \"orderRef\":\"131daac9-16c6-4618-beb0-365768f37288\",\n \"autoStartToken\":\"7c40b5c9-fa74-49cf-b98c-bfe651f9a7c6\"\n }\n\n :param end_user_ip: IP address of the user requesting\n the authentication.\n :type end_user_ip: str\n :param user_visible_data: The information that the end user\n is requested to sign.\n :type user_visible_data: str\n :param personal_number: The Swedish personal number in\n format YYYYMMDDXXXX.\n :type personal_number: str\n :param requirement: An optional dictionary stating how the signature\n must be created and verified. See BankID Relying Party Guidelines,\n section 13.5 for more details.\n :type requirement: dict\n :param user_non_visible_data: Optional information sent with request\n that the user never sees.\n :type user_non_visible_data: str\n :return: The order response.\n :rtype: dict\n :raises BankIDError: raises a subclass of this error\n when error has been returned from server.\n\n \"\"\"\n data = {\"endUserIp\": end_user_ip}\n if personal_number:\n data[\"personalNumber\"] = personal_number\n data[\"userVisibleData\"] = self._encode_user_data(user_visible_data)\n if user_non_visible_data:\n data[\"userNonVisibleData\"] = self._encode_user_data(user_non_visible_data)\n if requirement and isinstance(requirement, dict):\n data[\"requirement\"] = requirement\n # Handling potentially changed optional in-parameters.\n data.update(kwargs)\n response = self.client.post(self._sign_endpoint, json=data)\n\n if response.status_code == 200:\n return response.json()\n else:\n raise get_json_error_class(response)", "response": "This method is used to request an order."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef collect(self, order_ref):\n response = self.client.post(\n self._collect_endpoint, json={\"orderRef\": order_ref}\n )\n\n if response.status_code == 200:\n return response.json()\n else:\n raise get_json_error_class(response)", "response": "This method returns the result of a sign or auth order."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncancels an ongoing sign or auth order.", "response": "def cancel(self, order_ref):\n \"\"\"Cancels an ongoing sign or auth order.\n\n This is typically used if the user cancels the order\n in your service or app.\n\n :param order_ref: The UUID string specifying which order to cancel.\n :type order_ref: str\n :return: Boolean regarding success of cancellation.\n :rtype: bool\n :raises BankIDError: raises a subclass of this error\n when error has been returned from server.\n\n \"\"\"\n response = self.client.post(self._cancel_endpoint, json={\"orderRef\": order_ref})\n\n if response.status_code == 200:\n return response.json() == {}\n else:\n raise get_json_error_class(response)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getc(self, block=True):\n if not block:\n if not len(self.cookedq):\n return ''\n while not len(self.cookedq):\n time.sleep(0.05)\n self.IQUEUELOCK.acquire()\n ret = self.cookedq[0]\n self.cookedq = self.cookedq[1:]\n self.IQUEUELOCK.release()\n return ret", "response": "Return one character from the input queue"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef inputcooker_store_queue(self, char):\n self.IQUEUELOCK.acquire()\n if type(char) in [type(()), type([]), type(\"\")]:\n for v in char:\n self.cookedq.append(v)\n else:\n self.cookedq.append(char)\n self.IQUEUELOCK.release()", "response": "Store the cooked data in the input queue."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef writemessage(self, text):\n # Need to grab the input queue lock to ensure the entered data doesn't change\n # before we're done rebuilding it.\n # Note that writemessage will eventually call writecooked\n self.IQUEUELOCK.acquire()\n TelnetHandlerBase.writemessage(self, text)\n self.IQUEUELOCK.release()", "response": "Put data in output queue rebuild the prompt and entered data"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nput data directly into the output queue", "response": "def writecooked(self, text):\n \"\"\"Put data directly into the output queue\"\"\"\n # Ensure this is the only thread writing\n self.OQUEUELOCK.acquire()\n TelnetHandlerBase.writecooked(self, text)\n self.OQUEUELOCK.release()"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef split_certificate(certificate_path, destination_folder, password=None):\n try:\n # Attempt Linux and Darwin call first.\n p = subprocess.Popen(\n [\"openssl\", \"version\"], stdout=subprocess.PIPE, stderr=subprocess.PIPE\n )\n sout, serr = p.communicate()\n openssl_executable_version = sout.decode().lower()\n if not (\n openssl_executable_version.startswith(\"openssl\")\n or openssl_executable_version.startswith(\"libressl\")\n ):\n raise BankIDError(\n \"OpenSSL executable could not be found. \"\n \"Splitting cannot be performed.\"\n )\n openssl_executable = \"openssl\"\n except Exception:\n # Attempt to call on standard Git for Windows path.\n p = subprocess.Popen(\n [\"C:\\\\Program Files\\\\Git\\\\mingw64\\\\bin\\\\openssl.exe\", \"version\"],\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n sout, serr = p.communicate()\n if not sout.decode().lower().startswith(\"openssl\"):\n raise BankIDError(\n \"OpenSSL executable could not be found. \"\n \"Splitting cannot be performed.\"\n )\n openssl_executable = \"C:\\\\Program Files\\\\Git\\\\mingw64\\\\bin\\\\openssl.exe\"\n\n if not os.path.exists(os.path.abspath(os.path.expanduser(destination_folder))):\n os.makedirs(os.path.abspath(os.path.expanduser(destination_folder)))\n\n # Paths to output files.\n out_cert_path = os.path.join(\n os.path.abspath(os.path.expanduser(destination_folder)), \"certificate.pem\"\n )\n out_key_path = os.path.join(\n os.path.abspath(os.path.expanduser(destination_folder)), \"key.pem\"\n )\n\n # Use openssl for converting to pem format.\n pipeline_1 = [\n openssl_executable,\n \"pkcs12\",\n \"-in\",\n \"{0}\".format(certificate_path),\n \"-passin\" if password is not None else \"\",\n \"pass:{0}\".format(password) if password is not None else \"\",\n \"-out\",\n \"{0}\".format(out_cert_path),\n \"-clcerts\",\n \"-nokeys\",\n ]\n p = subprocess.Popen(\n list(filter(None, pipeline_1)), stdout=subprocess.PIPE, stderr=subprocess.PIPE\n )\n p.communicate()\n pipeline_2 = [\n openssl_executable,\n \"pkcs12\",\n \"-in\",\n \"{0}\".format(certificate_path),\n \"-passin\" if password is not None else \"\",\n \"pass:{0}\".format(password) if password is not None else \"\",\n \"-out\",\n \"{0}\".format(out_key_path),\n \"-nocerts\",\n \"-nodes\",\n ]\n p = subprocess.Popen(\n list(filter(None, pipeline_2)), stdout=subprocess.PIPE, stderr=subprocess.PIPE\n )\n p.communicate()\n\n # Return path tuples.\n return out_cert_path, out_key_path", "response": "Splits a PKCS12 certificate into Base64 - encoded DER certificate and key."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprocess chars while not in a part.", "response": "def process_delimiter(self, char):\n '''Process chars while not in a part'''\n if char in self.whitespace:\n return\n if char in self.quote_chars:\n # Store the quote type (' or \") and switch to quote processing.\n self.inquote = char\n self.process_char = self.process_quote\n return\n if char == self.eol_char:\n self.complete = True\n return\n # Switch to processing a part.\n self.process_char = self.process_part\n self.process_char(char)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef process_part(self, char):\n '''Process chars while in a part'''\n if char in self.whitespace or char == self.eol_char:\n # End of the part.\n self.parts.append( ''.join(self.part) )\n self.part = []\n # Switch back to processing a delimiter.\n self.process_char = self.process_delimiter\n if char == self.eol_char:\n self.complete = True\n return\n if char in self.quote_chars:\n # Store the quote type (' or \") and switch to quote processing.\n self.inquote = char\n self.process_char = self.process_quote\n return\n self.part.append(char)", "response": "Process chars while in a part."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef process_quote(self, char):\n '''Process character while in a quote'''\n if char == self.inquote:\n # Quote is finished, switch to part processing.\n self.process_char = self.process_part\n return\n try:\n self.part.append(char)\n except:\n self.part = [ char ]", "response": "Process character while in a quote"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef process_escape(self, char):\n '''Handle the char after the escape char'''\n # Always only run once, switch back to the last processor.\n self.process_char = self.last_process_char\n if self.part == [] and char in self.whitespace:\n # Special case where \\ is by itself and not at the EOL.\n self.parts.append(self.escape_char)\n return\n if char == self.eol_char:\n # Ignore a cr.\n return\n unescaped = self.escape_results.get(char, self.escape_char+char)\n self.part.append(unescaped)", "response": "Handle the char after the escape char."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsteps through the line and process each character and process each character.", "response": "def process(self, line):\n '''Step through the line and process each character'''\n self.raw = self.raw + line\n try:\n if not line[-1] == self.eol_char:\n # Should always be here, but add it just in case.\n line = line + self.eol_char\n except IndexError:\n # Thrown if line == ''\n line = self.eol_char\n \n for char in line:\n if char == self.escape_char:\n # Always handle escaped characters.\n self.last_process_char = self.process_char\n self.process_char = self.process_escape\n continue\n self.process_char(char)\n if not self.complete:\n # Ask for more.\n self.process( self.handler.readline(prompt=self.handler.CONTINUE_PROMPT) )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef streamserver_handle(cls, sock, address):\n '''Translate this class for use in a StreamServer'''\n request = cls.false_request()\n request._sock = sock\n server = None\n log.debug(\"Accepted connection, starting telnet session.\")\n try:\n cls(request, address, server)\n except socket.error:\n pass", "response": "Translate this class for use in a StreamServer"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef setterm(self, term):\n \"Set the curses structures for this terminal\"\n log.debug(\"Setting termtype to %s\" % (term, ))\n curses.setupterm(term) # This will raise if the termtype is not supported\n self.TERM = term\n self.ESCSEQ = {}\n for k in self.KEYS.keys():\n str = curses.tigetstr(curses.has_key._capability_names[k])\n if str:\n self.ESCSEQ[str] = k\n # Create a copy to prevent altering the class\n self.CODES = self.CODES.copy()\n self.CODES['DEOL'] = curses.tigetstr('el')\n self.CODES['DEL'] = curses.tigetstr('dch1')\n self.CODES['INS'] = curses.tigetstr('ich1')\n self.CODES['CSRLEFT'] = curses.tigetstr('cub1')\n self.CODES['CSRRIGHT'] = curses.tigetstr('cuf1')", "response": "Set the curses structures for this terminal"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef setup(self):\n \"Connect incoming connection to a telnet session\"\n try:\n self.TERM = self.request.term\n except:\n pass\n self.setterm(self.TERM)\n self.sock = self.request._sock\n for k in self.DOACK.keys():\n self.sendcommand(self.DOACK[k], k)\n for k in self.WILLACK.keys():\n self.sendcommand(self.WILLACK[k], k)", "response": "Connect incoming connection to a telnet session"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending a telnet command ( IAC", "response": "def sendcommand(self, cmd, opt=None):\n \"Send a telnet command (IAC)\"\n if cmd in [DO, DONT]:\n if not self.DOOPTS.has_key(opt):\n self.DOOPTS[opt] = None\n if (((cmd == DO) and (self.DOOPTS[opt] != True))\n or ((cmd == DONT) and (self.DOOPTS[opt] != False))):\n self.DOOPTS[opt] = (cmd == DO)\n self.writecooked(IAC + cmd + opt)\n elif cmd in [WILL, WONT]:\n if not self.WILLOPTS.has_key(opt):\n self.WILLOPTS[opt] = ''\n if (((cmd == WILL) and (self.WILLOPTS[opt] != True))\n or ((cmd == WONT) and (self.WILLOPTS[opt] != False))):\n self.WILLOPTS[opt] = (cmd == WILL)\n self.writecooked(IAC + cmd + opt)\n else:\n self.writecooked(IAC + cmd)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _readline_echo(self, char, echo):\n if self._readline_do_echo(echo):\n self.write(char)", "response": "Echo a recieved character move cursor etc..."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndeal properly with inserted chars in a line.", "response": "def _readline_insert(self, char, echo, insptr, line):\n \"\"\"Deal properly with inserted chars in a line.\"\"\"\n if not self._readline_do_echo(echo):\n return\n # Write out the remainder of the line\n self.write(char + ''.join(line[insptr:]))\n # Cursor Left to the current insert point\n char_count = len(line) - insptr\n self.write(self.CODES['CSRLEFT'] * char_count)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ansi_to_curses(self, char):\n '''Handles reading ANSI escape sequences'''\n # ANSI sequences are:\n # ESC [ \n # If we see ESC, read a char\n if char != ESC:\n return char\n # If we see [, read another char\n if self.getc(block=True) != ANSI_START_SEQ:\n self._readline_echo(BELL, True)\n return theNULL\n key = self.getc(block=True)\n # Translate the key to curses\n try:\n return ANSI_KEY_TO_CURSES[key]\n except:\n self._readline_echo(BELL, True)\n return theNULL", "response": "Handles reading ANSI escape sequences"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef readline(self, echo=None, prompt='', use_history=True):\n \n line = []\n insptr = 0\n ansi = 0\n histptr = len(self.history)\n \n if self.DOECHO:\n self.write(prompt)\n self._current_prompt = prompt\n else:\n self._current_prompt = ''\n \n self._current_line = ''\n \n while True:\n c = self.getc(block=True)\n c = self.ansi_to_curses(c)\n if c == theNULL:\n continue\n \n elif c == curses.KEY_LEFT:\n if insptr > 0:\n insptr = insptr - 1\n self._readline_echo(self.CODES['CSRLEFT'], echo)\n else:\n self._readline_echo(BELL, echo)\n continue\n elif c == curses.KEY_RIGHT:\n if insptr < len(line):\n insptr = insptr + 1\n self._readline_echo(self.CODES['CSRRIGHT'], echo)\n else:\n self._readline_echo(BELL, echo)\n continue\n elif c == curses.KEY_UP or c == curses.KEY_DOWN:\n if not use_history:\n self._readline_echo(BELL, echo)\n continue\n if c == curses.KEY_UP:\n if histptr > 0:\n histptr = histptr - 1\n else:\n self._readline_echo(BELL, echo)\n continue\n elif c == curses.KEY_DOWN:\n if histptr < len(self.history):\n histptr = histptr + 1\n else:\n self._readline_echo(BELL, echo)\n continue\n line = []\n if histptr < len(self.history):\n line.extend(self.history[histptr])\n for char in range(insptr):\n self._readline_echo(self.CODES['CSRLEFT'], echo)\n self._readline_echo(self.CODES['DEOL'], echo)\n self._readline_echo(''.join(line), echo)\n insptr = len(line)\n continue\n elif c == chr(3):\n self._readline_echo('\\n' + curses.ascii.unctrl(c) + ' ABORT\\n', echo)\n return ''\n elif c == chr(4):\n if len(line) > 0:\n self._readline_echo('\\n' + curses.ascii.unctrl(c) + ' ABORT (QUIT)\\n', echo)\n return ''\n self._readline_echo('\\n' + curses.ascii.unctrl(c) + ' QUIT\\n', echo)\n return 'QUIT'\n elif c == chr(10):\n self._readline_echo(c, echo)\n result = ''.join(line)\n if use_history:\n self.history.append(result)\n if echo is False:\n if prompt:\n self.write( chr(10) )\n log.debug('readline: %s(hidden text)', prompt)\n else:\n log.debug('readline: %s%r', prompt, result)\n return result\n elif c == curses.KEY_BACKSPACE or c == chr(127) or c == chr(8):\n if insptr > 0:\n self._readline_echo(self.CODES['CSRLEFT'] + self.CODES['DEL'], echo)\n insptr = insptr - 1\n del line[insptr]\n else:\n self._readline_echo(BELL, echo)\n continue\n elif c == curses.KEY_DC:\n if insptr < len(line):\n self._readline_echo(self.CODES['DEL'], echo)\n del line[insptr]\n else:\n self._readline_echo(BELL, echo)\n continue\n else:\n if ord(c) < 32:\n c = curses.ascii.unctrl(c)\n if len(line) > insptr:\n self._readline_insert(c, echo, insptr, line)\n else:\n self._readline_echo(c, echo)\n line[insptr:insptr] = c\n insptr = insptr + len(c)\n if self._readline_do_echo(echo):\n self._current_line = line", "response": "Return a line of text including the terminating LF."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsending a packet with line ending.", "response": "def writeline(self, text):\n \"\"\"Send a packet with line ending.\"\"\"\n log.debug('writing line %r' % text)\n self.write(text+chr(10))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwrites out an asynchronous message then reconstruct the prompt and entered text.", "response": "def writemessage(self, text):\n \"\"\"Write out an asynchronous message, then reconstruct the prompt and entered text.\"\"\"\n log.debug('writing message %r', text)\n self.write(chr(10)+text+chr(10))\n self.write(self._current_prompt+''.join(self._current_line))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsending a packet to the socket. This function cooks output.", "response": "def write(self, text):\n \"\"\"Send a packet to the socket. This function cooks output.\"\"\"\n text = str(text) # eliminate any unicode or other snigglets\n text = text.replace(IAC, IAC+IAC)\n text = text.replace(chr(10), chr(13)+chr(10))\n self.writecooked(text)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _inputcooker_getc(self, block=True):\n if self.rawq:\n ret = self.rawq[0]\n self.rawq = self.rawq[1:]\n return ret\n if not block:\n if not self.inputcooker_socket_ready():\n return ''\n ret = self.sock.recv(20)\n self.eof = not(ret)\n self.rawq = self.rawq + ret\n if self.eof:\n raise EOFError\n return self._inputcooker_getc(block)", "response": "Get one character from the raw queue. Optionally blocking. Raise EOFError on end of stream."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nstores the cooked data in the correct queue", "response": "def _inputcooker_store(self, char):\n \"\"\"Put the cooked data in the correct queue\"\"\"\n if self.sb:\n self.sbdataq = self.sbdataq + char\n else:\n self.inputcooker_store_queue(char)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninputting Cooker - Transfer from raw queue to cooked queue.", "response": "def inputcooker(self):\n \"\"\"Input Cooker - Transfer from raw queue to cooked queue.\n\n Set self.eof when connection is closed. Don't block unless in\n the midst of an IAC sequence.\n \"\"\"\n try:\n while True:\n c = self._inputcooker_getc()\n if not self.iacseq:\n if c == IAC:\n self.iacseq += c\n continue\n elif c == chr(13) and not(self.sb):\n c2 = self._inputcooker_getc(block=False)\n if c2 == theNULL or c2 == '':\n c = chr(10)\n elif c2 == chr(10):\n c = c2\n else:\n self._inputcooker_ungetc(c2)\n c = chr(10)\n elif c in [x[0] for x in self.ESCSEQ.keys()]:\n 'Looks like the begining of a key sequence'\n codes = c\n for keyseq in self.ESCSEQ.keys():\n if len(keyseq) == 0:\n continue\n while codes == keyseq[:len(codes)] and len(codes) <= keyseq:\n if codes == keyseq:\n c = self.ESCSEQ[keyseq]\n break\n codes = codes + self._inputcooker_getc()\n if codes == keyseq:\n break\n self._inputcooker_ungetc(codes[1:])\n codes = codes[0]\n self._inputcooker_store(c)\n elif len(self.iacseq) == 1:\n 'IAC: IAC CMD [OPTION only for WILL/WONT/DO/DONT]'\n if c in (DO, DONT, WILL, WONT):\n self.iacseq += c\n continue\n self.iacseq = ''\n if c == IAC:\n self._inputcooker_store(c)\n else:\n if c == SB: # SB ... SE start.\n self.sb = 1\n self.sbdataq = ''\n elif c == SE: # SB ... SE end.\n self.sb = 0\n # Callback is supposed to look into\n # the sbdataq\n self.options_handler(self.sock, c, NOOPT)\n elif len(self.iacseq) == 2:\n cmd = self.iacseq[1]\n self.iacseq = ''\n if cmd in (DO, DONT, WILL, WONT):\n self.options_handler(self.sock, cmd, c)\n except (EOFError, socket.error):\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cmdHELP(self, params):\n if params:\n cmd = params[0].upper()\n if self.COMMANDS.has_key(cmd):\n method = self.COMMANDS[cmd]\n doc = method.__doc__.split(\"\\n\")\n docp = doc[0].strip()\n docl = '\\n'.join( [l.strip() for l in doc[2:]] )\n if not docl.strip(): # If there isn't anything here, use line 1\n docl = doc[1].strip()\n self.writeline(\n \"%s %s\\n\\n%s\" % (\n cmd,\n docp,\n docl,\n )\n )\n return\n else:\n self.writeline(\"Command '%s' not known\" % cmd)\n else:\n self.writeline(\"Help on built in commands\\n\")\n keys = self.COMMANDS.keys()\n keys.sort()\n for cmd in keys:\n method = self.COMMANDS[cmd]\n if getattr(method, 'hidden', False):\n continue\n if method.__doc__ == None:\n self.writeline(\"no help for command %s\" % method)\n return\n doc = method.__doc__.split(\"\\n\")\n docp = doc[0].strip()\n docs = doc[1].strip()\n if len(docp) > 0:\n docps = \"%s - %s\" % (docp, docs, )\n else:\n docps = \"- %s\" % (docs, )\n self.writeline(\n \"%s %s\" % (\n cmd,\n docps,\n )\n )", "response": "Display help on built in commands"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndisplay the command history", "response": "def cmdHISTORY(self, params):\n \"\"\"\n Display the command history\n \"\"\"\n cnt = 0\n self.writeline('Command history\\n')\n for line in self.history:\n cnt = cnt + 1\n self.writeline(\"%-5d : %s\" % (cnt, ''.join(line)))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks the authentication and sets the username of the currently connected terminal. Returns True or False", "response": "def authentication_ok(self):\n '''Checks the authentication and sets the username of the currently connected terminal. Returns True or False'''\n username = None\n password = None\n if self.authCallback:\n if self.authNeedUser:\n username = self.readline(prompt=self.PROMPT_USER, use_history=False)\n if self.authNeedPass:\n password = self.readline(echo=False, prompt=self.PROMPT_PASS, use_history=False)\n if self.DOECHO:\n self.write(\"\\n\")\n try:\n self.authCallback(username, password)\n except:\n self.username = None\n return False\n else:\n # Successful authentication\n self.username = username\n return True\n else:\n # No authentication desired\n self.username = None\n return True"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef handle(self):\n \"The actual service to which the user has connected.\"\n if self.TELNET_ISSUE:\n self.writeline(self.TELNET_ISSUE)\n if not self.authentication_ok():\n return\n if self.DOECHO:\n self.writeline(self.WELCOME)\n\n self.session_start()\n while self.RUNSHELL:\n raw_input = self.readline(prompt=self.PROMPT).strip()\n self.input = self.input_reader(self, raw_input)\n self.raw_input = self.input.raw\n if self.input.cmd:\n cmd = self.input.cmd.upper()\n params = self.input.params\n if self.COMMANDS.has_key(cmd):\n try:\n self.COMMANDS[cmd](params)\n except:\n log.exception('Error calling %s.' % cmd)\n (t, p, tb) = sys.exc_info()\n if self.handleException(t, p, tb):\n break\n else:\n self.writeerror(\"Unknown command '%s'\" % cmd)\n log.debug(\"Exiting handler\")", "response": "The actual service to which the user has connected."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getc(self, block=True):\n try:\n return self.cookedq.get(block)\n except gevent.queue.Empty:\n return ''", "response": "Return one character from the input queue"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrequest an authentication order.", "response": "def authenticate(self, personal_number, **kwargs):\n \"\"\"Request an authentication order. The :py:meth:`collect` method\n is used to query the status of the order.\n\n :param personal_number: The Swedish personal number\n in format YYYYMMDDXXXX.\n :type personal_number: str\n :return: The OrderResponse parsed to a dictionary.\n :rtype: dict\n :raises BankIDError: raises a subclass of this error\n when error has been returned from server.\n\n \"\"\"\n if \"requirementAlternatives\" in kwargs:\n warnings.warn(\n \"Requirement Alternatives \" \"option is not tested.\", BankIDWarning\n )\n\n try:\n out = self.client.service.Authenticate(\n personalNumber=personal_number, **kwargs\n )\n except Error as e:\n raise get_error_class(e, \"Could not complete Authenticate order.\")\n\n return self._dictify(out)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrequest an signing order. The :py:meth:`collect` method is used to query the status of the order. :param user_visible_data: The information that the end user is requested to sign. :type user_visible_data: str :param personal_number: The Swedish personal number in format YYYYMMDDXXXX. :type personal_number: str :return: The OrderResponse parsed to a dictionary. :rtype: dict :raises BankIDError: raises a subclass of this error when error has been returned from server.", "response": "def sign(self, user_visible_data, personal_number=None, **kwargs):\n \"\"\"Request an signing order. The :py:meth:`collect` method\n is used to query the status of the order.\n\n :param user_visible_data: The information that the end user is\n requested to sign.\n :type user_visible_data: str\n :param personal_number: The Swedish personal number in\n format YYYYMMDDXXXX.\n :type personal_number: str\n :return: The OrderResponse parsed to a dictionary.\n :rtype: dict\n :raises BankIDError: raises a subclass of this error\n when error has been returned from server.\n\n \"\"\"\n if \"requirementAlternatives\" in kwargs:\n warnings.warn(\n \"Requirement Alternatives option is not tested.\", BankIDWarning\n )\n\n if isinstance(user_visible_data, six.text_type):\n data = base64.b64encode(user_visible_data.encode(\"utf-8\")).decode(\"ascii\")\n else:\n data = base64.b64encode(user_visible_data).decode(\"ascii\")\n\n try:\n out = self.client.service.Sign(\n userVisibleData=data, personalNumber=personal_number, **kwargs\n )\n except Error as e:\n raise get_error_class(e, \"Could not complete Sign order.\")\n\n return self._dictify(out)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef collect(self, order_ref):\n try:\n out = self.client.service.Collect(order_ref)\n except Error as e:\n raise get_error_class(e, \"Could not complete Collect call.\")\n\n return self._dictify(out)", "response": "Collect the progress status of the order with the specified order reference."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntransform the replies to a regular Python dict with strings and datetimes.", "response": "def _dictify(self, doc):\n \"\"\"Transforms the replies to a regular Python dict with\n strings and datetimes.\n\n Tested with BankID version 2.5 return data.\n\n :param doc: The response as interpreted by :py:mod:`zeep`.\n :returns: The response parsed to a dict.\n :rtype: dict\n\n \"\"\"\n return {\n k: (self._dictify(doc[k]) if hasattr(doc[k], \"_xsd_type\") else doc[k])\n for k in doc\n }"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload all image formats if needed.", "response": "def intercept_image_formats(self, options):\n \"\"\"\n Load all image formats if needed.\n \"\"\"\n if 'entityTypes' in options:\n for entity in options['entityTypes']:\n if entity['type'] == ENTITY_TYPES.IMAGE and 'imageFormats' in entity:\n if entity['imageFormats'] == '__all__':\n entity['imageFormats'] = get_all_image_formats()\n\n return options"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if self._timestamp is not None:\n self._normalized_timestamp = decimal.Decimal(self._timestamp)\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CopyFromDateTimeString(self, time_string):\n date_time_values = self._CopyDateTimeFromString(time_string)\n\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n\n self._timestamp = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n\n self.is_local_time = False", "response": "Copies a POSIX timestamp from a date and time string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncopying the POSIX timestamp to a date and time string.", "response": "def CopyToDateTimeString(self):\n \"\"\"Copies the POSIX timestamp to a date and time string.\n\n Returns:\n str: date and time value formatted as: \"YYYY-MM-DD hh:mm:ss\" or None\n if the timestamp is missing.\n \"\"\"\n if self._timestamp is None:\n return None\n\n number_of_days, hours, minutes, seconds = self._GetTimeValues(\n self._timestamp)\n\n year, month, day_of_month = self._GetDateValuesWithEpoch(\n number_of_days, self._EPOCH)\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}'.format(\n year, month, day_of_month, hours, minutes, seconds)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nretrieving the normalized timestamp.", "response": "def _GetNormalizedTimestamp(self):\n \"\"\"Retrieves the normalized timestamp.\n\n Returns:\n decimal.Decimal: normalized timestamp, which contains the number of\n seconds since January 1, 1970 00:00:00 and a fraction of second used\n for increased precision, or None if the normalized timestamp cannot be\n determined.\n \"\"\"\n if self._normalized_timestamp is None:\n if self._timestamp is not None:\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) /\n definitions.MILLISECONDS_PER_SECOND)\n\n return self._normalized_timestamp"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CopyFromDateTimeString(self, time_string):\n date_time_values = self._CopyDateTimeFromString(time_string)\n\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n microseconds = date_time_values.get('microseconds', 0)\n\n timestamp = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n timestamp *= definitions.MILLISECONDS_PER_SECOND\n\n if microseconds:\n milliseconds, _ = divmod(\n microseconds, definitions.MILLISECONDS_PER_SECOND)\n timestamp += milliseconds\n\n self._timestamp = timestamp\n self.is_local_time = False", "response": "Copies a POSIX timestamp from a date and time string."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if self._timestamp is not None:\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) /\n definitions.MICROSECONDS_PER_SECOND)\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if self._timestamp is not None:\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) /\n definitions.NANOSECONDS_PER_SECOND)\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _CopyFromDateTimeString(self, time_string):\n date_time_values = self._CopyDateTimeFromString(time_string)\n\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n microseconds = date_time_values.get('microseconds', None)\n\n timestamp = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n timestamp *= definitions.NANOSECONDS_PER_SECOND\n\n if microseconds:\n nanoseconds = microseconds * definitions.MILLISECONDS_PER_SECOND\n timestamp += nanoseconds\n\n self._normalized_timestamp = None\n self._timestamp = timestamp", "response": "Copies a POSIX timestamp from a date and time string."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncopies the POSIX timestamp to a date and time string.", "response": "def _CopyToDateTimeString(self):\n \"\"\"Copies the POSIX timestamp to a date and time string.\n\n Returns:\n str: date and time value formatted as: \"YYYY-MM-DD hh:mm:ss.#########\" or\n None if the timestamp is missing or invalid.\n \"\"\"\n if self._timestamp is None:\n return None\n\n timestamp, nanoseconds = divmod(\n self._timestamp, definitions.NANOSECONDS_PER_SECOND)\n number_of_days, hours, minutes, seconds = self._GetTimeValues(timestamp)\n\n year, month, day_of_month = self._GetDateValuesWithEpoch(\n number_of_days, self._EPOCH)\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}.{6:09d}'.format(\n year, month, day_of_month, hours, minutes, seconds, nanoseconds)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if self._timestamp is not None:\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) - self._DELPHI_TO_POSIX_BASE)\n self._normalized_timestamp *= definitions.SECONDS_PER_DAY\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncopy a Delphi TDateTime timestamp from a string.", "response": "def CopyFromDateTimeString(self, time_string):\n \"\"\"Copies a Delphi TDateTime timestamp from a string.\n\n Args:\n time_string (str): date and time value formatted as:\n YYYY-MM-DD hh:mm:ss.######[+-]##:##\n\n Where # are numeric digits ranging from 0 to 9 and the seconds\n fraction can be either 3 or 6 digits. The time of day, seconds\n fraction and time zone offset are optional. The default time zone\n is UTC.\n\n Raises:\n ValueError: if the time string is invalid or not supported.\n \"\"\"\n date_time_values = self._CopyDateTimeFromString(time_string)\n\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n microseconds = date_time_values.get('microseconds', None)\n\n if year > 9999:\n raise ValueError('Unsupported year value: {0:d}.'.format(year))\n\n timestamp = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n\n timestamp = float(timestamp) / definitions.SECONDS_PER_DAY\n timestamp += self._DELPHI_TO_POSIX_BASE\n if microseconds is not None:\n timestamp += float(microseconds) / definitions.MICROSECONDS_PER_DAY\n\n self._normalized_timestamp = None\n self._timestamp = timestamp\n self.is_local_time = False"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve the normalized timestamp.", "response": "def _GetNormalizedTimestamp(self):\n \"\"\"Retrieves the normalized timestamp.\n\n Returns:\n float: normalized timestamp, which contains the number of seconds since\n January 1, 1970 00:00:00 and a fraction of second used for increased\n precision, or None if the normalized timestamp cannot be determined.\n \"\"\"\n if self._normalized_timestamp is None:\n if (self._timestamp is not None and self._timestamp >= self._INT64_MIN and\n self._timestamp <= self._INT64_MAX):\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) /\n definitions.MICROSECONDS_PER_SECOND)\n self._normalized_timestamp -= self._WEBKIT_TO_POSIX_BASE\n\n return self._normalized_timestamp"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadjust the date and time values for a time zone offset.", "response": "def _AdjustForTimeZoneOffset(\n self, year, month, day_of_month, hours, minutes, time_zone_offset):\n \"\"\"Adjusts the date and time values for a time zone offset.\n\n Args:\n year (int): year e.g. 1970.\n month (int): month, where 1 represents January.\n day_of_month (int): day of the month, where 1 represents the first day.\n hours (int): hours.\n minutes (int): minutes.\n time_zone_offset (int): time zone offset in number of minutes from UTC.\n\n Returns:\n tuple[int, int, int, int, int, int]: time zone correct year, month,\n day_of_month, hours and minutes values.\n \"\"\"\n hours_from_utc, minutes_from_utc = divmod(time_zone_offset, 60)\n\n minutes += minutes_from_utc\n\n # Since divmod makes sure the sign of minutes_from_utc is positive\n # we only need to check the upper bound here, because hours_from_utc\n # remains signed it is corrected accordingly.\n if minutes >= 60:\n minutes -= 60\n hours += 1\n\n hours += hours_from_utc\n if hours < 0:\n hours += 24\n day_of_month -= 1\n\n elif hours >= 24:\n hours -= 24\n day_of_month += 1\n\n days_per_month = self._GetDaysPerMonth(year, month)\n if day_of_month < 1:\n month -= 1\n if month < 1:\n month = 12\n year -= 1\n\n day_of_month += self._GetDaysPerMonth(year, month)\n\n elif day_of_month > days_per_month:\n month += 1\n if month > 12:\n month = 1\n year += 1\n\n day_of_month -= days_per_month\n\n return year, month, day_of_month, hours, minutes"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncopying a date from a string.", "response": "def _CopyDateFromString(self, date_string):\n \"\"\"Copies a date from a string.\n\n Args:\n date_string (str): date value formatted as: YYYY-MM-DD\n\n Returns:\n tuple[int, int, int]: year, month, day of month.\n\n Raises:\n ValueError: if the date string is invalid or not supported.\n \"\"\"\n date_string_length = len(date_string)\n\n # The date string should at least contain 'YYYY-MM-DD'.\n if date_string_length < 10:\n raise ValueError('Date string too short.')\n\n if date_string[4] != '-' or date_string[7] != '-':\n raise ValueError('Invalid date string.')\n\n try:\n year = int(date_string[0:4], 10)\n except ValueError:\n raise ValueError('Unable to parse year.')\n\n try:\n month = int(date_string[5:7], 10)\n except ValueError:\n raise ValueError('Unable to parse month.')\n\n try:\n day_of_month = int(date_string[8:10], 10)\n except ValueError:\n raise ValueError('Unable to parse day of month.')\n\n days_per_month = self._GetDaysPerMonth(year, month)\n if day_of_month < 1 or day_of_month > days_per_month:\n raise ValueError('Day of month value out of bounds.')\n\n return year, month, day_of_month"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncopies a time from a string.", "response": "def _CopyTimeFromString(self, time_string):\n \"\"\"Copies a time from a string.\n\n Args:\n time_string (str): time value formatted as:\n hh:mm:ss.######[+-]##:##\n\n Where # are numeric digits ranging from 0 to 9 and the seconds\n fraction can be either 3 or 6 digits. The seconds fraction and\n time zone offset are optional.\n\n Returns:\n tuple[int, int, int, int, int]: hours, minutes, seconds, microseconds,\n time zone offset in minutes.\n\n Raises:\n ValueError: if the time string is invalid or not supported.\n \"\"\"\n time_string_length = len(time_string)\n\n # The time string should at least contain 'hh:mm:ss'.\n if time_string_length < 8:\n raise ValueError('Time string too short.')\n\n if time_string[2] != ':' or time_string[5] != ':':\n raise ValueError('Invalid time string.')\n\n try:\n hours = int(time_string[0:2], 10)\n except ValueError:\n raise ValueError('Unable to parse hours.')\n\n if hours not in range(0, 24):\n raise ValueError('Hours value: {0:d} out of bounds.'.format(hours))\n\n try:\n minutes = int(time_string[3:5], 10)\n except ValueError:\n raise ValueError('Unable to parse minutes.')\n\n if minutes not in range(0, 60):\n raise ValueError('Minutes value: {0:d} out of bounds.'.format(minutes))\n\n try:\n seconds = int(time_string[6:8], 10)\n except ValueError:\n raise ValueError('Unable to parse day of seconds.')\n\n # TODO: support a leap second?\n if seconds not in range(0, 60):\n raise ValueError('Seconds value: {0:d} out of bounds.'.format(seconds))\n\n microseconds = None\n time_zone_offset = None\n\n time_zone_string_index = 8\n while time_zone_string_index < time_string_length:\n if time_string[time_zone_string_index] in ('+', '-'):\n break\n\n time_zone_string_index += 1\n\n # The calculations that follow rely on the time zone string index\n # to point beyond the string in case no time zone offset was defined.\n if time_zone_string_index == time_string_length - 1:\n time_zone_string_index += 1\n\n if time_string_length > 8 and time_string[8] == '.':\n time_fraction_length = time_zone_string_index - 9\n if time_fraction_length not in (3, 6):\n raise ValueError('Invalid time string.')\n\n try:\n time_fraction = time_string[9:time_zone_string_index]\n time_fraction = int(time_fraction, 10)\n except ValueError:\n raise ValueError('Unable to parse time fraction.')\n\n if time_fraction_length == 3:\n time_fraction *= 1000\n\n microseconds = time_fraction\n\n if time_zone_string_index < time_string_length:\n if (time_string_length - time_zone_string_index != 6 or\n time_string[time_zone_string_index + 3] != ':'):\n raise ValueError('Invalid time string.')\n\n try:\n hours_from_utc = int(time_string[\n time_zone_string_index + 1:time_zone_string_index + 3])\n except ValueError:\n raise ValueError('Unable to parse time zone hours offset.')\n\n if hours_from_utc not in range(0, 15):\n raise ValueError('Time zone hours offset value out of bounds.')\n\n try:\n minutes_from_utc = int(time_string[\n time_zone_string_index + 4:time_zone_string_index + 6])\n except ValueError:\n raise ValueError('Unable to parse time zone minutes offset.')\n\n if minutes_from_utc not in range(0, 60):\n raise ValueError('Time zone minutes offset value out of bounds.')\n\n # pylint: disable=invalid-unary-operand-type\n time_zone_offset = (hours_from_utc * 60) + minutes_from_utc\n\n # Note that when the sign of the time zone offset is negative\n # the difference needs to be added. We do so by flipping the sign.\n if time_string[time_zone_string_index] != '-':\n time_zone_offset = -time_zone_offset\n\n return hours, minutes, seconds, microseconds, time_zone_offset"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _GetDateValues(\n self, number_of_days, epoch_year, epoch_month, epoch_day_of_month):\n \"\"\"Determines date values.\n\n Args:\n number_of_days (int): number of days since epoch.\n epoch_year (int): year that is the start of the epoch e.g. 1970.\n epoch_month (int): month that is the start of the epoch, where\n 1 represents January.\n epoch_day_of_month (int): day of month that is the start of the epoch,\n where 1 represents the first day.\n\n Returns:\n tuple[int, int, int]: year, month, day of month.\n\n Raises:\n ValueError: if the epoch year, month or day of month values are out\n of bounds.\n \"\"\"\n if epoch_year < 0:\n raise ValueError('Epoch year value: {0:d} out of bounds.'.format(\n epoch_year))\n\n if epoch_month not in range(1, 13):\n raise ValueError('Epoch month value: {0:d} out of bounds.'.format(\n epoch_month))\n\n epoch_days_per_month = self._GetDaysPerMonth(epoch_year, epoch_month)\n if epoch_day_of_month < 1 or epoch_day_of_month > epoch_days_per_month:\n raise ValueError('Epoch day of month value: {0:d} out of bounds.'.format(\n epoch_day_of_month))\n\n before_epoch = number_of_days < 0\n\n year = epoch_year\n month = epoch_month\n if before_epoch:\n month -= 1\n if month <= 0:\n month = 12\n year -= 1\n\n number_of_days += epoch_day_of_month\n if before_epoch:\n number_of_days *= -1\n\n # Align with the start of the year.\n while month > 1:\n days_per_month = self._GetDaysPerMonth(year, month)\n if number_of_days < days_per_month:\n break\n\n if before_epoch:\n month -= 1\n else:\n month += 1\n\n if month > 12:\n month = 1\n year += 1\n\n number_of_days -= days_per_month\n\n # Align with the start of the next century.\n _, remainder = divmod(year, 100)\n for _ in range(remainder, 100):\n days_in_year = self._GetNumberOfDaysInYear(year)\n if number_of_days < days_in_year:\n break\n\n if before_epoch:\n year -= 1\n else:\n year += 1\n\n number_of_days -= days_in_year\n\n days_in_century = self._GetNumberOfDaysInCentury(year)\n while number_of_days > days_in_century:\n if before_epoch:\n year -= 100\n else:\n year += 100\n\n number_of_days -= days_in_century\n days_in_century = self._GetNumberOfDaysInCentury(year)\n\n days_in_year = self._GetNumberOfDaysInYear(year)\n while number_of_days > days_in_year:\n if before_epoch:\n year -= 1\n else:\n year += 1\n\n number_of_days -= days_in_year\n days_in_year = self._GetNumberOfDaysInYear(year)\n\n days_per_month = self._GetDaysPerMonth(year, month)\n while number_of_days > days_per_month:\n if before_epoch:\n month -= 1\n else:\n month += 1\n\n if month <= 0:\n month = 12\n year -= 1\n elif month > 12:\n month = 1\n year += 1\n\n number_of_days -= days_per_month\n days_per_month = self._GetDaysPerMonth(year, month)\n\n if before_epoch:\n days_per_month = self._GetDaysPerMonth(year, month)\n number_of_days = days_per_month - number_of_days\n\n elif number_of_days == 0:\n number_of_days = 31\n month = 12\n year -= 1\n\n return year, month, number_of_days", "response": "Determines the date values for the given number of days since the epoch."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndetermines date values for the specified number of days since the epoch.", "response": "def _GetDateValuesWithEpoch(self, number_of_days, date_time_epoch):\n \"\"\"Determines date values.\n\n Args:\n number_of_days (int): number of days since epoch.\n date_time_epoch (DateTimeEpoch): date and time of the epoch.\n\n Returns:\n tuple[int, int, int]: year, month, day of month.\n \"\"\"\n return self._GetDateValues(\n number_of_days, date_time_epoch.year, date_time_epoch.month,\n date_time_epoch.day_of_month)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nretrieving the day of the year for a specific day of a month in a given year.", "response": "def _GetDayOfYear(self, year, month, day_of_month):\n \"\"\"Retrieves the day of the year for a specific day of a month in a year.\n\n Args:\n year (int): year e.g. 1970.\n month (int): month, where 1 represents January.\n day_of_month (int): day of the month, where 1 represents the first day.\n\n Returns:\n int: day of year.\n\n Raises:\n ValueError: if the month or day of month value is out of bounds.\n \"\"\"\n if month not in range(1, 13):\n raise ValueError('Month value out of bounds.')\n\n days_per_month = self._GetDaysPerMonth(year, month)\n if day_of_month < 1 or day_of_month > days_per_month:\n raise ValueError('Day of month value out of bounds.')\n\n day_of_year = day_of_month\n for past_month in range(1, month):\n day_of_year += self._GetDaysPerMonth(year, past_month)\n\n return day_of_year"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _GetDaysPerMonth(self, year, month):\n if month not in range(1, 13):\n raise ValueError('Month value out of bounds.')\n\n days_per_month = self._DAYS_PER_MONTH[month - 1]\n if month == 2 and self._IsLeapYear(year):\n days_per_month += 1\n\n return days_per_month", "response": "Retrieves the number of days in a specific month."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _GetNumberOfDaysInCentury(self, year):\n if year < 0:\n raise ValueError('Year value out of bounds.')\n\n year, _ = divmod(year, 100)\n\n if self._IsLeapYear(year):\n return 36525\n return 36524", "response": "Retrieves the number of days in a century."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _GetNumberOfSecondsFromElements(\n self, year, month, day_of_month, hours, minutes, seconds):\n \"\"\"Retrieves the number of seconds from the date and time elements.\n\n Args:\n year (int): year e.g. 1970.\n month (int): month, where 1 represents January.\n day_of_month (int): day of the month, where 1 represents the first day.\n hours (int): hours.\n minutes (int): minutes.\n seconds (int): seconds.\n\n Returns:\n int: number of seconds since January 1, 1970 00:00:00 or None if year,\n month or day of month are not set.\n\n Raises:\n ValueError: if the time elements are invalid.\n \"\"\"\n if not year or not month or not day_of_month:\n return None\n\n # calendar.timegm does not sanity check the time elements.\n if hours is None:\n hours = 0\n elif hours not in range(0, 24):\n raise ValueError('Hours value: {0!s} out of bounds.'.format(hours))\n\n if minutes is None:\n minutes = 0\n elif minutes not in range(0, 60):\n raise ValueError('Minutes value: {0!s} out of bounds.'.format(minutes))\n\n # TODO: support a leap second?\n if seconds is None:\n seconds = 0\n elif seconds not in range(0, 60):\n raise ValueError('Seconds value: {0!s} out of bounds.'.format(seconds))\n\n # Note that calendar.timegm() does not raise when date is: 2013-02-29.\n days_per_month = self._GetDaysPerMonth(year, month)\n if day_of_month < 1 or day_of_month > days_per_month:\n raise ValueError('Day of month value out of bounds.')\n\n # calendar.timegm requires the time tuple to contain at least\n # 6 integer values.\n time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds)\n\n number_of_seconds = calendar.timegm(time_elements_tuple)\n\n return int(number_of_seconds)", "response": "Retrieves the number of seconds from the date and time elements."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _GetTimeValues(self, number_of_seconds):\n number_of_seconds = int(number_of_seconds)\n number_of_minutes, seconds = divmod(number_of_seconds, 60)\n number_of_hours, minutes = divmod(number_of_minutes, 60)\n number_of_days, hours = divmod(number_of_hours, 24)\n return number_of_days, hours, minutes, seconds", "response": "Determines time values.\n\n Args:\n number_of_seconds (int|decimal.Decimal): number of seconds.\n\n Returns:\n tuple[int, int, int, int]: days, hours, minutes, seconds."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncopying the date time value to a stat timestamp tuple.", "response": "def CopyToStatTimeTuple(self):\n \"\"\"Copies the date time value to a stat timestamp tuple.\n\n Returns:\n tuple[int, int]: a POSIX timestamp in seconds and the remainder in\n 100 nano seconds or (None, None) on error.\n \"\"\"\n normalized_timestamp = self._GetNormalizedTimestamp()\n if normalized_timestamp is None:\n return None, None\n\n if self._precision in (\n definitions.PRECISION_1_NANOSECOND,\n definitions.PRECISION_100_NANOSECONDS,\n definitions.PRECISION_1_MICROSECOND,\n definitions.PRECISION_1_MILLISECOND,\n definitions.PRECISION_100_MILLISECONDS):\n remainder = int((normalized_timestamp % 1) * self._100NS_PER_SECOND)\n\n return int(normalized_timestamp), remainder\n\n return int(normalized_timestamp), None"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CopyToDateTimeStringISO8601(self):\n date_time_string = self.CopyToDateTimeString()\n if date_time_string:\n date_time_string = date_time_string.replace(' ', 'T')\n date_time_string = '{0:s}Z'.format(date_time_string)\n return date_time_string", "response": "Copies the date time value to an ISO 8601 date and time string."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve the date represented by the date and time values.", "response": "def GetDate(self):\n \"\"\"Retrieves the date represented by the date and time values.\n\n Returns:\n tuple[int, int, int]: year, month, day of month or (None, None, None)\n if the date and time values do not represent a date.\n \"\"\"\n normalized_timestamp = self._GetNormalizedTimestamp()\n if normalized_timestamp is None:\n return None, None, None\n\n number_of_days, _, _, _ = self._GetTimeValues(normalized_timestamp)\n\n try:\n return self._GetDateValuesWithEpoch(\n number_of_days, self._EPOCH_NORMALIZED_TIME)\n\n except ValueError:\n return None, None, None"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetPlasoTimestamp(self):\n normalized_timestamp = self._GetNormalizedTimestamp()\n if normalized_timestamp is None:\n return None\n\n normalized_timestamp *= definitions.MICROSECONDS_PER_SECOND\n normalized_timestamp = normalized_timestamp.quantize(\n 1, rounding=decimal.ROUND_HALF_UP)\n return int(normalized_timestamp)", "response": "Retrieves a timestamp that is compatible with plaso."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetTimeOfDay(self):\n normalized_timestamp = self._GetNormalizedTimestamp()\n if normalized_timestamp is None:\n return None, None, None\n\n _, hours, minutes, seconds = self._GetTimeValues(normalized_timestamp)\n return hours, minutes, seconds", "response": "Retrieves the time of day represented by the date and time values."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if (self._timestamp is not None and self._timestamp >= 0 and\n self._timestamp <= self._UINT32_MAX):\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) - self._HFS_TO_POSIX_BASE)\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nretrieve the normalized timestamp.", "response": "def _GetNormalizedTimestamp(self):\n \"\"\"Retrieves the normalized timestamp.\n\n Returns:\n decimal.Decimal: normalized timestamp, which contains the number of\n seconds since January 1, 1970 00:00:00 and a fraction of second used\n for increased precision, or None if the normalized timestamp cannot be\n determined.\n \"\"\"\n if self._normalized_timestamp is None:\n if self._number_of_seconds is not None:\n self._normalized_timestamp = (\n decimal.Decimal(self._microseconds) /\n definitions.MICROSECONDS_PER_SECOND)\n self._normalized_timestamp += decimal.Decimal(self._number_of_seconds)\n\n return self._normalized_timestamp"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncopies a fake timestamp from a date and time string.", "response": "def CopyFromDateTimeString(self, time_string):\n \"\"\"Copies a fake timestamp from a date and time string.\n\n Args:\n time_string (str): date and time value formatted as:\n YYYY-MM-DD hh:mm:ss.######[+-]##:##\n\n Where # are numeric digits ranging from 0 to 9 and the seconds\n fraction can be either 3 or 6 digits. The time of day, seconds\n fraction and time zone offset are optional. The default time zone\n is UTC.\n \"\"\"\n date_time_values = self._CopyDateTimeFromString(time_string)\n\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n\n self._normalized_timestamp = None\n self._number_of_seconds = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n self._microseconds = date_time_values.get('microseconds', None)\n\n self.is_local_time = False"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if self._number_of_seconds is not None:\n self._normalized_timestamp = (\n decimal.Decimal(self.deciseconds) /\n definitions.DECISECONDS_PER_SECOND)\n self._normalized_timestamp += decimal.Decimal(self._number_of_seconds)\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncopying the RFC2579 date - time to a date and time string.", "response": "def CopyToDateTimeString(self):\n \"\"\"Copies the RFC2579 date-time to a date and time string.\n\n Returns:\n str: date and time value formatted as: \"YYYY-MM-DD hh:mm:ss.#\" or\n None if the number of seconds is missing.\n \"\"\"\n if self._number_of_seconds is None:\n return None\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}.{6:01d}'.format(\n self.year, self.month, self.day_of_month, self.hours, self.minutes,\n self.seconds, self.deciseconds)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nopens the SMS gateway. Returns whether or not a new connection was required.", "response": "def open(self):\n\t\t\"\"\"\n\t\tEnsures we have a connection to the SMS gateway. Returns whether or not a new connection was required (True or False).\n\t\t\"\"\"\n\t\tif self.connection:\n\t\t\t# Nothing to do if the connection is already open.\n\t\t\treturn False\n\n\t\tself.connection = self._get_twilio_client()\n\t\treturn True"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _send(self, email_message):\n\t\tif not email_message.recipients():\n\t\t\treturn False\n\t\tfrom_email = email_message.from_email\n\t\trecipients = email_message.recipients()\n\t\ttry:\n\t\t\tself.connection.messages.create(\n\t\t\t\tto=recipients,\n\t\t\t\tfrom_=from_email,\n\t\t\t\tbody=email_message.body\n\t\t\t)\n\t\texcept Exception:\n\t\t\tif not self.fail_silently:\n\t\t\t\traise\n\t\t\treturn False\n\t\treturn True", "response": "A helper method that does the actual sending."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve the normalized timestamp.", "response": "def _GetNormalizedTimestamp(self):\n \"\"\"Retrieves the normalized timestamp.\n\n Returns:\n decimal.Decimal: normalized timestamp, which contains the number of\n seconds since January 1, 1970 00:00:00 and a fraction of second used\n for increased precision, or None if the normalized timestamp cannot be\n determined.\n \"\"\"\n if self._normalized_timestamp is None:\n if self._number_of_seconds is not None and self._number_of_seconds >= 0:\n self._normalized_timestamp = (\n decimal.Decimal(self._number_of_seconds) +\n self._FAT_DATE_TO_POSIX_BASE)\n\n return self._normalized_timestamp"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nretrieving the number of seconds from a FAT date time.", "response": "def _GetNumberOfSeconds(self, fat_date_time):\n \"\"\"Retrieves the number of seconds from a FAT date time.\n\n Args:\n fat_date_time (int): FAT date time.\n\n Returns:\n int: number of seconds since January 1, 1980 00:00:00.\n\n Raises:\n ValueError: if the month, day of month, hours, minutes or seconds\n value is out of bounds.\n \"\"\"\n day_of_month = (fat_date_time & 0x1f)\n month = ((fat_date_time >> 5) & 0x0f)\n year = (fat_date_time >> 9) & 0x7f\n\n days_per_month = self._GetDaysPerMonth(year, month)\n if day_of_month < 1 or day_of_month > days_per_month:\n raise ValueError('Day of month value out of bounds.')\n\n number_of_days = self._GetDayOfYear(1980 + year, month, day_of_month)\n number_of_days -= 1\n for past_year in range(0, year):\n number_of_days += self._GetNumberOfDaysInYear(past_year)\n\n fat_date_time >>= 16\n\n seconds = (fat_date_time & 0x1f) * 2\n minutes = (fat_date_time >> 5) & 0x3f\n hours = (fat_date_time >> 11) & 0x1f\n\n if hours not in range(0, 24):\n raise ValueError('Hours value out of bounds.')\n\n if minutes not in range(0, 60):\n raise ValueError('Minutes value out of bounds.')\n\n if seconds not in range(0, 60):\n raise ValueError('Seconds value out of bounds.')\n\n number_of_seconds = (((hours * 60) + minutes) * 60) + seconds\n number_of_seconds += number_of_days * definitions.SECONDS_PER_DAY\n return number_of_seconds"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_sms_connection(backend=None, fail_silently=False, **kwds):\n\tklass = import_string(backend or settings.SMS_BACKEND)\n\treturn klass(fail_silently=fail_silently, **kwds)", "response": "Load an SMS backend and return an instance of it."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef send_sms(message, from_number, recipient_list, fail_silently=False, auth_user=None, auth_password=None, connection=None):\n\tconnection = connection or get_sms_connection(username=auth_user, password=auth_password, fail_silently=fail_silently)\n\tmail = SMSMessage(message, from_number, recipient_list, connection=connection)\n\n\treturn mail.send()", "response": "Wrapper for sending a single message to a recipient list."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsending a mass SMS message to each recipient in datatuple.", "response": "def send_mass_sms(datatuple, fail_silently=False, auth_user=None, auth_password=None, connection=None):\n\t\"\"\"\n\tGiven a datatuple of (subject, message, from_email, recipient_list), sends\n\teach message to each recipient list. Returns the number of emails sent.\n\n\tIf from_email is None, the DEFAULT_FROM_EMAIL setting is used.\n\tIf auth_user and auth_password are set, they're used to log in.\n\tIf auth_user is None, the EMAIL_HOST_USER setting is used.\n\tIf auth_password is None, the EMAIL_HOST_PASSWORD setting is used.\n\n\tNote: The API for this method is frozen. New code wanting to extend the\n\tfunctionality should use the EmailMessage class directly.\n\n\thttps://github.com/django/django/blob/master/django/core/mail/__init__.py#L64\n\t\"\"\"\n\timport smsish.sms.backends.rq\n\tif isinstance(connection, smsish.sms.backends.rq.SMSBackend):\n\t\traise NotImplementedError\n\tconnection = connection or get_sms_connection(username=auth_user, password=auth_password, fail_silently=fail_silently)\n\tmessages = [SMSMessage(message, from_number, recipient, connection=connection) for message, from_number, recipient in datatuple]\n\treturn connection.send_messages(messages)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nretrieves the normalized timestamp.", "response": "def _GetNormalizedTimestamp(self):\n \"\"\"Retrieves the normalized timestamp.\n\n Returns:\n decimal.Decimal: normalized timestamp, which contains the number of\n seconds since January 1, 1970 00:00:00 and a fraction of second used\n for increased precision, or None if the normalized timestamp cannot be\n determined.\n \"\"\"\n if self._normalized_timestamp is None:\n if (self._timestamp is not None and self._timestamp >= 0 and\n self._timestamp <= self._UINT64_MAX):\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) / self._100NS_PER_SECOND)\n self._normalized_timestamp -= self._FILETIME_TO_POSIX_BASE\n\n return self._normalized_timestamp"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncopying the FILETIME timestamp to a date and time string.", "response": "def CopyToDateTimeString(self):\n \"\"\"Copies the FILETIME timestamp to a date and time string.\n\n Returns:\n str: date and time value formatted as: \"YYYY-MM-DD hh:mm:ss.#######\" or\n None if the timestamp is missing or invalid.\n \"\"\"\n if (self._timestamp is None or self._timestamp < 0 or\n self._timestamp > self._UINT64_MAX):\n return None\n\n timestamp, remainder = divmod(self._timestamp, self._100NS_PER_SECOND)\n number_of_days, hours, minutes, seconds = self._GetTimeValues(timestamp)\n\n year, month, day_of_month = self._GetDateValuesWithEpoch(\n number_of_days, self._EPOCH)\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}.{6:07d}'.format(\n year, month, day_of_month, hours, minutes, seconds, remainder)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nretrieving the normalized timestamp.", "response": "def _GetNormalizedTimestamp(self):\n \"\"\"Retrieves the normalized timestamp.\n\n Returns:\n decimal.Decimal: normalized timestamp, which contains the number of\n seconds since January 1, 1970 00:00:00 and a fraction of second\n used for increased precision, or None if the normalized timestamp\n cannot be determined.\n \"\"\"\n if self._normalized_timestamp is None:\n if (self._timestamp is not None and self._timestamp >= 0 and\n self._timestamp <= self._UINT60_MAX):\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) / self._100NS_PER_SECOND)\n self._normalized_timestamp -= self._UUID_TO_POSIX_BASE\n\n return self._normalized_timestamp"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncopying the time elements and fraction of second to a string.", "response": "def CopyToDateTimeString(cls, time_elements_tuple, fraction_of_second):\n \"\"\"Copies the time elements and fraction of second to a string.\n\n Args:\n time_elements_tuple (tuple[int, int, int, int, int, int]):\n time elements, contains year, month, day of month, hours, minutes and\n seconds.\n fraction_of_second (decimal.Decimal): fraction of second, which must be a\n value between 0.0 and 1.0.\n\n Returns:\n str: date and time value formatted as:\n YYYY-MM-DD hh:mm:ss\n\n Raises:\n ValueError: if the fraction of second is out of bounds.\n \"\"\"\n if fraction_of_second < 0.0 or fraction_of_second >= 1.0:\n raise ValueError('Fraction of second value: {0:f} out of bounds.'.format(\n fraction_of_second))\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}'.format(\n time_elements_tuple[0], time_elements_tuple[1], time_elements_tuple[2],\n time_elements_tuple[3], time_elements_tuple[4], time_elements_tuple[5])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CopyMicrosecondsToFractionOfSecond(cls, microseconds):\n if microseconds < 0 or microseconds >= definitions.MICROSECONDS_PER_SECOND:\n raise ValueError(\n 'Number of microseconds value: {0:d} out of bounds.'.format(\n microseconds))\n\n milliseconds, _ = divmod(\n microseconds, definitions.MICROSECONDS_PER_MILLISECOND)\n return decimal.Decimal(milliseconds) / definitions.MILLISECONDS_PER_SECOND", "response": "Copies the number of microseconds to a fraction of second value."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CopyToDateTimeString(cls, time_elements_tuple, fraction_of_second):\n if fraction_of_second < 0.0 or fraction_of_second >= 1.0:\n raise ValueError('Fraction of second value: {0:f} out of bounds.'.format(\n fraction_of_second))\n\n milliseconds = int(fraction_of_second * definitions.MILLISECONDS_PER_SECOND)\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}.{6:03d}'.format(\n time_elements_tuple[0], time_elements_tuple[1], time_elements_tuple[2],\n time_elements_tuple[3], time_elements_tuple[4], time_elements_tuple[5],\n milliseconds)", "response": "Copies the time elements and fraction of second to a date and time string."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CopyMicrosecondsToFractionOfSecond(cls, microseconds):\n if microseconds < 0 or microseconds >= definitions.MICROSECONDS_PER_SECOND:\n raise ValueError(\n 'Number of microseconds value: {0:d} out of bounds.'.format(\n microseconds))\n\n return decimal.Decimal(microseconds) / definitions.MICROSECONDS_PER_SECOND", "response": "Copies the number of microseconds to a fraction of second value."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CopyToDateTimeString(cls, time_elements_tuple, fraction_of_second):\n if fraction_of_second < 0.0 or fraction_of_second >= 1.0:\n raise ValueError('Fraction of second value: {0:f} out of bounds.'.format(\n fraction_of_second))\n\n microseconds = int(fraction_of_second * definitions.MICROSECONDS_PER_SECOND)\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}.{6:06d}'.format(\n time_elements_tuple[0], time_elements_tuple[1], time_elements_tuple[2],\n time_elements_tuple[3], time_elements_tuple[4], time_elements_tuple[5],\n microseconds)", "response": "Copies the time elements and fraction of second to a date and time string."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CreatePrecisionHelper(cls, precision):\n precision_helper_class = cls._PRECISION_CLASSES.get(precision, None)\n if not precision_helper_class:\n raise ValueError('Unsupported precision: {0!s}'.format(precision))\n\n return precision_helper_class", "response": "Creates a date time precision helper."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef CopyFromDateTimeString(self, time_string):\n super(APFSTime, self)._CopyFromDateTimeString(time_string)\n\n if (self._timestamp is None or self._timestamp < self._INT64_MIN or\n self._timestamp > self._INT64_MAX):\n raise ValueError('Date time value not supported.')", "response": "Copies an APFS timestamp from a date and time string."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncopy the APFS timestamp to a date and time string.", "response": "def CopyToDateTimeString(self):\n \"\"\"Copies the APFS timestamp to a date and time string.\n\n Returns:\n str: date and time value formatted as: \"YYYY-MM-DD hh:mm:ss.#########\" or\n None if the timestamp is missing or invalid.\n \"\"\"\n if (self._timestamp is None or self._timestamp < self._INT64_MIN or\n self._timestamp > self._INT64_MAX):\n return None\n\n return super(APFSTime, self)._CopyToDateTimeString()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncopying the Cocoa timestamp to a date and time string.", "response": "def CopyToDateTimeString(self):\n \"\"\"Copies the Cocoa timestamp to a date and time string.\n\n Returns:\n str: date and time value formatted as: YYYY-MM-DD hh:mm:ss.###### or\n None if the timestamp cannot be copied to a date and time string.\n \"\"\"\n if self._timestamp is None:\n return None\n\n number_of_days, hours, minutes, seconds = self._GetTimeValues(\n int(self._timestamp))\n\n year, month, day_of_month = self._GetDateValuesWithEpoch(\n number_of_days, self._EPOCH)\n\n microseconds = int(\n (self._timestamp % 1) * definitions.MICROSECONDS_PER_SECOND)\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}.{6:06d}'.format(\n year, month, day_of_month, hours, minutes, seconds, microseconds)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsends an SMSMessage instance to the specified backend.", "response": "def send_sms_message(sms_message, backend=None, fail_silently=False):\n\t\"\"\"\n\tSend an SMSMessage instance using a connection given by the specified `backend`.\n\t\"\"\"\n\twith get_sms_connection(backend=backend, fail_silently=fail_silently) as connection:\n\t\tresult = connection.send_messages([sms_message])\n\treturn result"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending SMS messages to the backend and returns a list of RQ Jobs.", "response": "def send_messages(self, sms_messages):\n\t\t\"\"\"\n\t\tReceives a list of SMSMessage instances and returns a list of RQ `Job` instances.\n\t\t\"\"\"\n\t\tresults = []\n\t\tfor message in sms_messages:\n\t\t\ttry:\n\t\t\t\tassert message.connection is None\n\t\t\texcept AssertionError:\n\t\t\t\tif not self.fail_silently:\n\t\t\t\t\traise\n\t\t\tbackend = self.backend\n\t\t\tfail_silently = self.fail_silently\n\t\t\tresult = django_rq.enqueue(self._send, message, backend=backend, fail_silently=fail_silently)\n\t\t\tresults.append(result)\n\t\treturn results"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if (self._timestamp is not None and self._timestamp >= self._INT64_MIN and\n self._timestamp <= self._INT64_MAX):\n self._normalized_timestamp = (\n decimal.Decimal(self._timestamp) /\n definitions.MILLISECONDS_PER_SECOND)\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncopy the POSIX timestamp to a date and time string.", "response": "def CopyToDateTimeString(self):\n \"\"\"Copies the POSIX timestamp to a date and time string.\n\n Returns:\n str: date and time value formatted as: \"YYYY-MM-DD hh:mm:ss.######\" or\n None if the timestamp is missing.\n \"\"\"\n if (self._timestamp is None or self._timestamp < self._INT64_MIN or\n self._timestamp > self._INT64_MAX):\n return None\n\n return super(JavaTime, self).CopyToDateTimeString()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Image(props):\n image_model = get_image_model()\n alignment = props.get('alignment', 'left')\n alt_text = props.get('altText', '')\n\n try:\n image = image_model.objects.get(id=props['id'])\n except image_model.DoesNotExist:\n return DOM.create_element('img', {'alt': alt_text})\n\n image_format = get_image_format(alignment)\n rendition = get_rendition_or_not_found(image, image_format.filter_spec)\n\n return DOM.create_element('img', dict(rendition.attrs_dict, **{\n 'class': image_format.classnames,\n 'src': rendition.url,\n 'alt': alt_text,\n }))", "response": "Return an image element for the given properties."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncopies a date and time from an ISO 8601 date and time string.", "response": "def _CopyDateTimeFromStringISO8601(self, time_string):\n \"\"\"Copies a date and time from an ISO 8601 date and time string.\n\n Args:\n time_string (str): time value formatted as:\n hh:mm:ss.######[+-]##:##\n\n Where # are numeric digits ranging from 0 to 9 and the seconds\n fraction can be either 3 or 6 digits. The fraction of second and\n time zone offset are optional.\n\n Returns:\n tuple[int, int, int, int, int]: hours, minutes, seconds, microseconds,\n time zone offset in minutes.\n\n Raises:\n ValueError: if the time string is invalid or not supported.\n \"\"\"\n if not time_string:\n raise ValueError('Invalid time string.')\n\n time_string_length = len(time_string)\n\n year, month, day_of_month = self._CopyDateFromString(time_string)\n\n if time_string_length <= 10:\n return {\n 'year': year,\n 'month': month,\n 'day_of_month': day_of_month}\n\n # If a time of day is specified the time string it should at least\n # contain 'YYYY-MM-DDThh'.\n if time_string[10] != 'T':\n raise ValueError(\n 'Invalid time string - missing as date and time separator.')\n\n hours, minutes, seconds, microseconds, time_zone_offset = (\n self._CopyTimeFromStringISO8601(time_string[11:]))\n\n if time_zone_offset:\n year, month, day_of_month, hours, minutes = self._AdjustForTimeZoneOffset(\n year, month, day_of_month, hours, minutes, time_zone_offset)\n\n date_time_values = {\n 'year': year,\n 'month': month,\n 'day_of_month': day_of_month,\n 'hours': hours,\n 'minutes': minutes,\n 'seconds': seconds}\n\n if microseconds is not None:\n date_time_values['microseconds'] = microseconds\n return date_time_values"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _CopyFromDateTimeValues(self, date_time_values):\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n\n self._normalized_timestamp = None\n self._number_of_seconds = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n self._time_elements_tuple = (\n year, month, day_of_month, hours, minutes, seconds)\n\n self.is_local_time = False", "response": "Copies time elements from date and time values."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncopies a time from an ISO 8601 date and time string.", "response": "def _CopyTimeFromStringISO8601(self, time_string):\n \"\"\"Copies a time from an ISO 8601 date and time string.\n\n Args:\n time_string (str): time value formatted as:\n hh:mm:ss.######[+-]##:##\n\n Where # are numeric digits ranging from 0 to 9 and the seconds\n fraction can be either 3 or 6 digits. The faction of second and\n time zone offset are optional.\n\n Returns:\n tuple[int, int, int, int, int]: hours, minutes, seconds, microseconds,\n time zone offset in minutes.\n\n Raises:\n ValueError: if the time string is invalid or not supported.\n \"\"\"\n if time_string.endswith('Z'):\n time_string = time_string[:-1]\n\n time_string_length = len(time_string)\n\n # The time string should at least contain 'hh'.\n if time_string_length < 2:\n raise ValueError('Time string too short.')\n\n try:\n hours = int(time_string[0:2], 10)\n except ValueError:\n raise ValueError('Unable to parse hours.')\n\n if hours not in range(0, 24):\n raise ValueError('Hours value: {0:d} out of bounds.'.format(hours))\n\n minutes = None\n seconds = None\n microseconds = None\n time_zone_offset = None\n\n time_string_index = 2\n\n # Minutes are either specified as 'hhmm', 'hh:mm' or as a fractional part\n # 'hh[.,]###'.\n if (time_string_index + 1 < time_string_length and\n time_string[time_string_index] not in ('.', ',')):\n if time_string[time_string_index] == ':':\n time_string_index += 1\n\n if time_string_index + 2 > time_string_length:\n raise ValueError('Time string too short.')\n\n try:\n minutes = time_string[time_string_index:time_string_index + 2]\n minutes = int(minutes, 10)\n except ValueError:\n raise ValueError('Unable to parse minutes.')\n\n time_string_index += 2\n\n # Seconds are either specified as 'hhmmss', 'hh:mm:ss' or as a fractional\n # part 'hh:mm[.,]###' or 'hhmm[.,]###'.\n if (time_string_index + 1 < time_string_length and\n time_string[time_string_index] not in ('.', ',')):\n if time_string[time_string_index] == ':':\n time_string_index += 1\n\n if time_string_index + 2 > time_string_length:\n raise ValueError('Time string too short.')\n\n try:\n seconds = time_string[time_string_index:time_string_index + 2]\n seconds = int(seconds, 10)\n except ValueError:\n raise ValueError('Unable to parse day of seconds.')\n\n time_string_index += 2\n\n time_zone_string_index = time_string_index\n while time_zone_string_index < time_string_length:\n if time_string[time_zone_string_index] in ('+', '-'):\n break\n\n time_zone_string_index += 1\n\n # The calculations that follow rely on the time zone string index\n # to point beyond the string in case no time zone offset was defined.\n if time_zone_string_index == time_string_length - 1:\n time_zone_string_index += 1\n\n if (time_string_length > time_string_index and\n time_string[time_string_index] in ('.', ',')):\n time_string_index += 1\n time_fraction_length = time_zone_string_index - time_string_index\n\n try:\n time_fraction = time_string[time_string_index:time_zone_string_index]\n time_fraction = int(time_fraction, 10)\n time_fraction = (\n decimal.Decimal(time_fraction) /\n decimal.Decimal(10 ** time_fraction_length))\n except ValueError:\n raise ValueError('Unable to parse time fraction.')\n\n if minutes is None:\n time_fraction *= 60\n minutes = int(time_fraction)\n time_fraction -= minutes\n\n if seconds is None:\n time_fraction *= 60\n seconds = int(time_fraction)\n time_fraction -= seconds\n\n time_fraction *= definitions.MICROSECONDS_PER_SECOND\n microseconds = int(time_fraction)\n\n if minutes is not None and minutes not in range(0, 60):\n raise ValueError('Minutes value: {0:d} out of bounds.'.format(minutes))\n\n # TODO: support a leap second?\n if seconds is not None and seconds not in range(0, 60):\n raise ValueError('Seconds value: {0:d} out of bounds.'.format(seconds))\n\n if time_zone_string_index < time_string_length:\n if (time_string_length - time_zone_string_index != 6 or\n time_string[time_zone_string_index + 3] != ':'):\n raise ValueError('Invalid time string.')\n\n try:\n hours_from_utc = int(time_string[\n time_zone_string_index + 1:time_zone_string_index + 3])\n except ValueError:\n raise ValueError('Unable to parse time zone hours offset.')\n\n if hours_from_utc not in range(0, 15):\n raise ValueError('Time zone hours offset value out of bounds.')\n\n try:\n minutes_from_utc = int(time_string[\n time_zone_string_index + 4:time_zone_string_index + 6])\n except ValueError:\n raise ValueError('Unable to parse time zone minutes offset.')\n\n if minutes_from_utc not in range(0, 60):\n raise ValueError('Time zone minutes offset value out of bounds.')\n\n # pylint: disable=invalid-unary-operand-type\n time_zone_offset = (hours_from_utc * 60) + minutes_from_utc\n\n # Note that when the sign of the time zone offset is negative\n # the difference needs to be added. We do so by flipping the sign.\n if time_string[time_zone_string_index] != '-':\n time_zone_offset = -time_zone_offset\n\n return hours, minutes, seconds, microseconds, time_zone_offset"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CopyFromDateTimeString(self, time_string):\n date_time_values = self._CopyDateTimeFromString(time_string)\n\n self._CopyFromDateTimeValues(date_time_values)", "response": "Copies time elements from a date and time string."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CopyFromStringISO8601(self, time_string):\n date_time_values = self._CopyDateTimeFromStringISO8601(time_string)\n\n self._CopyFromDateTimeValues(date_time_values)", "response": "Copies time elements from an ISO 8601 date and time string."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CopyFromStringTuple(self, time_elements_tuple):\n if len(time_elements_tuple) < 6:\n raise ValueError((\n 'Invalid time elements tuple at least 6 elements required,'\n 'got: {0:d}').format(len(time_elements_tuple)))\n\n try:\n year = int(time_elements_tuple[0], 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid year value: {0!s}'.format(\n time_elements_tuple[0]))\n\n try:\n month = int(time_elements_tuple[1], 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid month value: {0!s}'.format(\n time_elements_tuple[1]))\n\n try:\n day_of_month = int(time_elements_tuple[2], 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid day of month value: {0!s}'.format(\n time_elements_tuple[2]))\n\n try:\n hours = int(time_elements_tuple[3], 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid hours value: {0!s}'.format(\n time_elements_tuple[3]))\n\n try:\n minutes = int(time_elements_tuple[4], 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid minutes value: {0!s}'.format(\n time_elements_tuple[4]))\n\n try:\n seconds = int(time_elements_tuple[5], 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid seconds value: {0!s}'.format(\n time_elements_tuple[5]))\n\n self._normalized_timestamp = None\n self._number_of_seconds = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n self._time_elements_tuple = (\n year, month, day_of_month, hours, minutes, seconds)", "response": "Copies time elements from a string - based time elements tuple."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CopyToDateTimeString(self):\n if self._number_of_seconds is None:\n return None\n\n return '{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}'.format(\n self._time_elements_tuple[0], self._time_elements_tuple[1],\n self._time_elements_tuple[2], self._time_elements_tuple[3],\n self._time_elements_tuple[4], self._time_elements_tuple[5])", "response": "Copies the time elements to a date and time string."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _CopyFromDateTimeValues(self, date_time_values):\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n microseconds = date_time_values.get('microseconds', 0)\n\n precision_helper = precisions.PrecisionHelperFactory.CreatePrecisionHelper(\n self._precision)\n\n fraction_of_second = precision_helper.CopyMicrosecondsToFractionOfSecond(\n microseconds)\n\n self._normalized_timestamp = None\n self._number_of_seconds = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n self._time_elements_tuple = (\n year, month, day_of_month, hours, minutes, seconds)\n self.fraction_of_second = fraction_of_second\n self.is_local_time = False", "response": "Copies time elements from a date and time values dictionary."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CopyFromStringTuple(self, time_elements_tuple):\n if len(time_elements_tuple) < 7:\n raise ValueError((\n 'Invalid time elements tuple at least 7 elements required,'\n 'got: {0:d}').format(len(time_elements_tuple)))\n\n super(TimeElementsWithFractionOfSecond, self).CopyFromStringTuple(\n time_elements_tuple)\n\n try:\n fraction_of_second = decimal.Decimal(time_elements_tuple[6])\n except (TypeError, ValueError):\n raise ValueError('Invalid fraction of second value: {0!s}'.format(\n time_elements_tuple[6]))\n\n if fraction_of_second < 0.0 or fraction_of_second >= 1.0:\n raise ValueError('Fraction of second value: {0:f} out of bounds.'.format(\n fraction_of_second))\n\n self.fraction_of_second = fraction_of_second", "response": "Copies time elements from string - based time elements tuple."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncopying the time elements to a date and time string.", "response": "def CopyToDateTimeString(self):\n \"\"\"Copies the time elements to a date and time string.\n\n Returns:\n str: date and time value formatted as: \"YYYY-MM-DD hh:mm:ss\" or\n \"YYYY-MM-DD hh:mm:ss.######\" or None if time elements are missing.\n\n Raises:\n ValueError: if the precision value is unsupported.\n \"\"\"\n if self._number_of_seconds is None or self.fraction_of_second is None:\n return None\n\n precision_helper = precisions.PrecisionHelperFactory.CreatePrecisionHelper(\n self._precision)\n\n return precision_helper.CopyToDateTimeString(\n self._time_elements_tuple, self.fraction_of_second)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncopies time elements from a string - based time elements tuple.", "response": "def CopyFromStringTuple(self, time_elements_tuple):\n \"\"\"Copies time elements from string-based time elements tuple.\n\n Args:\n time_elements_tuple (Optional[tuple[str, str, str, str, str, str, str]]):\n time elements, contains year, month, day of month, hours, minutes,\n seconds and milliseconds.\n\n Raises:\n ValueError: if the time elements tuple is invalid.\n \"\"\"\n if len(time_elements_tuple) < 7:\n raise ValueError((\n 'Invalid time elements tuple at least 7 elements required,'\n 'got: {0:d}').format(len(time_elements_tuple)))\n\n year, month, day_of_month, hours, minutes, seconds, milliseconds = (\n time_elements_tuple)\n\n try:\n milliseconds = int(milliseconds, 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid millisecond value: {0!s}'.format(milliseconds))\n\n if milliseconds < 0 or milliseconds >= definitions.MILLISECONDS_PER_SECOND:\n raise ValueError('Invalid number of milliseconds.')\n\n fraction_of_second = (\n decimal.Decimal(milliseconds) / definitions.MILLISECONDS_PER_SECOND)\n\n time_elements_tuple = (\n year, month, day_of_month, hours, minutes, seconds,\n str(fraction_of_second))\n\n super(TimeElementsInMilliseconds, self).CopyFromStringTuple(\n time_elements_tuple)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CopyFromStringTuple(self, time_elements_tuple):\n if len(time_elements_tuple) < 7:\n raise ValueError((\n 'Invalid time elements tuple at least 7 elements required,'\n 'got: {0:d}').format(len(time_elements_tuple)))\n\n year, month, day_of_month, hours, minutes, seconds, microseconds = (\n time_elements_tuple)\n\n try:\n microseconds = int(microseconds, 10)\n except (TypeError, ValueError):\n raise ValueError('Invalid microsecond value: {0!s}'.format(microseconds))\n\n if microseconds < 0 or microseconds >= definitions.MICROSECONDS_PER_SECOND:\n raise ValueError('Invalid number of microseconds.')\n\n fraction_of_second = (\n decimal.Decimal(microseconds) / definitions.MICROSECONDS_PER_SECOND)\n\n time_elements_tuple = (\n year, month, day_of_month, hours, minutes, seconds,\n str(fraction_of_second))\n\n super(TimeElementsInMicroseconds, self).CopyFromStringTuple(\n time_elements_tuple)", "response": "Copies time elements from string - based time elements tuple."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _GetNormalizedTimestamp(self):\n if self._normalized_timestamp is None:\n if self._number_of_seconds is not None:\n self._normalized_timestamp = (\n decimal.Decimal(self.milliseconds) /\n definitions.MILLISECONDS_PER_SECOND)\n self._normalized_timestamp += decimal.Decimal(self._number_of_seconds)\n\n return self._normalized_timestamp", "response": "Retrieves the normalized timestamp."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncopies a SYSTEMTIME structure from a date and time string.", "response": "def CopyFromDateTimeString(self, time_string):\n \"\"\"Copies a SYSTEMTIME structure from a date and time string.\n\n Args:\n time_string (str): date and time value formatted as:\n YYYY-MM-DD hh:mm:ss.######[+-]##:##\n\n Where # are numeric digits ranging from 0 to 9 and the seconds\n fraction can be either 3 or 6 digits. The time of day, seconds\n fraction and time zone offset are optional. The default time zone\n is UTC.\n\n Raises:\n ValueError: if the date string is invalid or not supported.\n \"\"\"\n date_time_values = self._CopyDateTimeFromString(time_string)\n\n year = date_time_values.get('year', 0)\n month = date_time_values.get('month', 0)\n day_of_month = date_time_values.get('day_of_month', 0)\n hours = date_time_values.get('hours', 0)\n minutes = date_time_values.get('minutes', 0)\n seconds = date_time_values.get('seconds', 0)\n\n microseconds = date_time_values.get('microseconds', 0)\n milliseconds, _ = divmod(\n microseconds, definitions.MICROSECONDS_PER_MILLISECOND)\n\n if year < 1601 or year > 30827:\n raise ValueError('Unsupported year value: {0:d}.'.format(year))\n\n self._normalized_timestamp = None\n self._number_of_seconds = self._GetNumberOfSecondsFromElements(\n year, month, day_of_month, hours, minutes, seconds)\n\n self.year = year\n self.month = month\n self.day_of_month = day_of_month\n # TODO: calculate day of week on demand.\n self.day_of_week = None\n self.hours = hours\n self.minutes = minutes\n self.seconds = seconds\n self.milliseconds = milliseconds\n\n self.is_local_time = False"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef send_messages(self, messages):\n\t\tmsg_count = 0\n\t\tfor message in messages: # .message() triggers header validation\n\t\t\tmessage.message()\n\t\t\tmsg_count += 1\n\t\tmail.outbox.extend(messages)\n\t\treturn msg_count", "response": "Redirect messages to the dummy outbox"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _prepare_template(self, obj, needs_request=False):\n if self.instance_name is None and self.template_name is None:\n raise SearchFieldError(\"This field requires either its instance_name variable to be populated or an explicit template_name in order to load the correct template.\")\n\n if self.template_name is not None:\n template_names = self.template_name\n\n if not isinstance(template_names, (list, tuple)):\n template_names = [template_names]\n else:\n template_names = ['search/indexes/%s/%s_%s.txt' % (obj._meta.app_label, obj._meta.module_name, self.instance_name)]\n\n t = loader.select_template(template_names)\n ctx = {'object': obj}\n if needs_request:\n request = rf.get(\"/\")\n request.session = {}\n ctx['request'] = request\n return t.render(Context(ctx))", "response": "This method is a copy of CharField. prepare_template that loads the template and renders it into the context."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef paragraph(node):\n text = ''\n if node.string_content is not None:\n text = node.string_content\n o = nodes.paragraph('', ' '.join(text))\n o.line = node.sourcepos[0][0]\n for n in MarkDown(node):\n o.append(n)\n\n return o", "response": "Process a paragraph node"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef reference(node):\n o = nodes.reference()\n o['refuri'] = node.destination\n if node.title:\n o['name'] = node.title\n for n in MarkDown(node):\n o += n\n return o", "response": "A hyperlink that references a node."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef emphasis(node):\n o = nodes.emphasis()\n for n in MarkDown(node):\n o += n\n return o", "response": "Returns the italicized section of the given node."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef strong(node):\n o = nodes.strong()\n for n in MarkDown(node):\n o += n\n return o", "response": "A simple node that is strongly connected to a node."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef literal(node):\n rendered = []\n try:\n if node.info is not None:\n l = Lexer(node.literal, node.info, tokennames=\"long\")\n for _ in l:\n rendered.append(node.inline(classes=_[0], text=_[1]))\n except:\n pass\n\n classes = ['code']\n if node.info is not None:\n classes.append(node.info)\n if len(rendered) > 0:\n o = nodes.literal(classes=classes)\n for element in rendered:\n o += element\n else:\n o = nodes.literal(text=node.literal, classes=classes)\n\n for n in MarkDown(node):\n o += n\n return o", "response": "Return a node that is a literal node."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd some raw html to the tree", "response": "def raw(node):\n \"\"\"\n Add some raw html (possibly as a block)\n \"\"\"\n o = nodes.raw(node.literal, node.literal, format='html')\n if node.sourcepos is not None:\n o.line = node.sourcepos[0][0]\n for n in MarkDown(node):\n o += n\n return o"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef block_quote(node):\n o = nodes.block_quote()\n o.line = node.sourcepos[0][0]\n for n in MarkDown(node):\n o += n\n return o", "response": "A block quote node"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef image(node):\n o = nodes.image(uri=node.destination)\n if node.first_child is not None:\n o['alt'] = node.first_child.literal\n return o", "response": "Return an image element."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef listItem(node):\n o = nodes.list_item()\n for n in MarkDown(node):\n o += n\n return o", "response": "Returns a list item"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MarkDown(node):\n cur = node.first_child\n\n # Go into each child, in turn\n output = []\n while cur is not None:\n t = cur.t\n if t == 'paragraph':\n output.append(paragraph(cur))\n elif t == 'text':\n output.append(text(cur))\n elif t == 'softbreak':\n output.append(softbreak(cur))\n elif t == 'linebreak':\n output.append(hardbreak(cur))\n elif t == 'link':\n output.append(reference(cur))\n elif t == 'heading':\n output.append(title(cur))\n elif t == 'emph':\n output.append(emphasis(cur))\n elif t == 'strong':\n output.append(strong(cur))\n elif t == 'code':\n output.append(literal(cur))\n elif t == 'code_block':\n output.append(literal_block(cur))\n elif t == 'html_inline' or t == 'html_block':\n output.append(raw(cur))\n elif t == 'block_quote':\n output.append(block_quote(cur))\n elif t == 'thematic_break':\n output.append(transition(cur))\n elif t == 'image':\n output.append(image(cur))\n elif t == 'list':\n output.append(listNode(cur))\n elif t == 'item':\n output.append(listItem(cur))\n elif t == 'MDsection':\n output.append(section(cur))\n else:\n print('Received unhandled type: {}. Full print of node:'.format(t))\n cur.pretty()\n\n cur = cur.nxt\n\n return output", "response": "Returns a list of CommonMark nodes converted to docutils nodes\n "} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef finalizeSection(section):\n cur = section.first_child\n last = section.last_child\n if last is not None:\n last.nxt = None\n\n while cur is not None:\n cur.parent = section\n cur = cur.nxt", "response": "Correct the nxt and parent for each child\n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nnesting sections in a CommonMark block.", "response": "def nestSections(block, level=1):\n \"\"\"\n Sections aren't handled by CommonMark at the moment.\n This function adds sections to a block of nodes.\n 'title' nodes with an assigned level below 'level' will be put in a child section.\n If there are no child nodes with titles of level 'level' then nothing is done\n \"\"\"\n cur = block.first_child\n if cur is not None:\n children = []\n # Do we need to do anything?\n nest = False\n while cur is not None:\n if cur.t == 'heading' and cur.level == level:\n nest = True\n break\n cur = cur.nxt\n if not nest:\n return\n\n section = Node('MDsection', 0)\n section.parent = block\n cur = block.first_child\n while cur is not None:\n if cur.t == 'heading' and cur.level == level:\n # Found a split point, flush the last section if needed\n if section.first_child is not None:\n finalizeSection(section)\n children.append(section)\n section = Node('MDsection', 0)\n nxt = cur.nxt\n # Avoid adding sections without titles at the start\n if section.first_child is None:\n if cur.t == 'heading' and cur.level == level:\n section.append_child(cur)\n else:\n children.append(cur)\n else:\n section.append_child(cur)\n cur = nxt\n\n # If there's only 1 child then don't bother\n if section.first_child is not None:\n finalizeSection(section)\n children.append(section)\n\n block.first_child = None\n block.last_child = None\n nextLevel = level + 1\n for child in children:\n # Handle nesting\n if child.t == 'MDsection':\n nestSections(child, level=nextLevel)\n\n # Append\n if block.first_child is None:\n block.first_child = child\n else:\n block.last_child.nxt = child\n child.parent = block\n child.nxt = None\n child.prev = block.last_child\n block.last_child = child"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef parseMarkDownBlock(text):\n block = Parser().parse(text)\n # CommonMark can't nest sections, so do it manually\n nestSections(block)\n\n return MarkDown(block)", "response": "Parses a block of text returning a list of docutils nodes"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngiving a list of reStructuredText or MarkDown sections return a docutils node list", "response": "def renderList(l, markDownHelp, settings=None):\n \"\"\"\n Given a list of reStructuredText or MarkDown sections, return a docutils node list\n \"\"\"\n if len(l) == 0:\n return []\n if markDownHelp:\n from sphinxarg.markdown import parseMarkDownBlock\n return parseMarkDownBlock('\\n\\n'.join(l) + '\\n')\n else:\n all_children = []\n for element in l:\n if isinstance(element, str):\n if settings is None:\n settings = OptionParser(components=(Parser,)).get_default_values()\n document = new_document(None, settings)\n Parser().parse(element + '\\n', document)\n all_children += document.children\n elif isinstance(element, nodes.definition):\n all_children += element\n\n return all_children"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprinting all action groups in nested content.", "response": "def print_action_groups(data, nested_content, markDownHelp=False, settings=None):\n \"\"\"\n Process all 'action groups', which are also include 'Options' and 'Required\n arguments'. A list of nodes is returned.\n \"\"\"\n definitions = map_nested_definitions(nested_content)\n nodes_list = []\n if 'action_groups' in data:\n for action_group in data['action_groups']:\n # Every action group is comprised of a section, holding a title, the description, and the option group (members)\n section = nodes.section(ids=[action_group['title']])\n section += nodes.title(action_group['title'], action_group['title'])\n\n desc = []\n if action_group['description']:\n desc.append(action_group['description'])\n # Replace/append/prepend content to the description according to nested content\n subContent = []\n if action_group['title'] in definitions:\n classifier, s, subContent = definitions[action_group['title']]\n if classifier == '@replace':\n desc = [s]\n elif classifier == '@after':\n desc.append(s)\n elif classifier == '@before':\n desc.insert(0, s)\n elif classifier == '@skip':\n continue\n if len(subContent) > 0:\n for k, v in map_nested_definitions(subContent).items():\n definitions[k] = v\n # Render appropriately\n for element in renderList(desc, markDownHelp):\n section += element\n\n localDefinitions = definitions\n if len(subContent) > 0:\n localDefinitions = {k: v for k, v in definitions.items()}\n for k, v in map_nested_definitions(subContent).items():\n localDefinitions[k] = v\n\n items = []\n # Iterate over action group members\n for entry in action_group['options']:\n \"\"\"\n Members will include:\n default\tThe default value. This may be ==SUPPRESS==\n name\tA list of option names (e.g., ['-h', '--help']\n help\tThe help message string\n There may also be a 'choices' member.\n \"\"\"\n # Build the help text\n arg = []\n if 'choices' in entry:\n arg.append('Possible choices: {}\\n'.format(\", \".join([str(c) for c in entry['choices']])))\n if 'help' in entry:\n arg.append(entry['help'])\n if entry['default'] is not None and entry['default'] not in ['\"==SUPPRESS==\"', '==SUPPRESS==']:\n if entry['default'] == '':\n arg.append('Default: \"\"')\n else:\n arg.append('Default: {}'.format(entry['default']))\n\n # Handle nested content, the term used in the dict has the comma removed for simplicity\n desc = arg\n term = ' '.join(entry['name'])\n if term in localDefinitions:\n classifier, s, subContent = localDefinitions[term]\n if classifier == '@replace':\n desc = [s]\n elif classifier == '@after':\n desc.append(s)\n elif classifier == '@before':\n desc.insert(0, s)\n term = ', '.join(entry['name'])\n\n n = nodes.option_list_item('',\n nodes.option_group('', nodes.option_string(text=term)),\n nodes.description('', *renderList(desc, markDownHelp, settings)))\n items.append(n)\n\n section += nodes.option_list('', *items)\n nodes_list.append(section)\n\n return nodes_list"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprint all the subcommands in the nested content.", "response": "def print_subcommands(data, nested_content, markDownHelp=False, settings=None):\n \"\"\"\n Each subcommand is a dictionary with the following keys:\n\n ['usage', 'action_groups', 'bare_usage', 'name', 'help']\n\n In essence, this is all tossed in a new section with the title 'name'.\n Apparently there can also be a 'description' entry.\n \"\"\"\n\n definitions = map_nested_definitions(nested_content)\n items = []\n if 'children' in data:\n subCommands = nodes.section(ids=[\"Sub-commands:\"])\n subCommands += nodes.title('Sub-commands:', 'Sub-commands:')\n\n for child in data['children']:\n sec = nodes.section(ids=[child['name']])\n sec += nodes.title(child['name'], child['name'])\n\n if 'description' in child and child['description']:\n desc = [child['description']]\n elif child['help']:\n desc = [child['help']]\n else:\n desc = ['Undocumented']\n\n # Handle nested content\n subContent = []\n if child['name'] in definitions:\n classifier, s, subContent = definitions[child['name']]\n if classifier == '@replace':\n desc = [s]\n elif classifier == '@after':\n desc.append(s)\n elif classifier == '@before':\n desc.insert(0, s)\n\n for element in renderList(desc, markDownHelp):\n sec += element\n sec += nodes.literal_block(text=child['bare_usage'])\n for x in print_action_groups(child, nested_content + subContent, markDownHelp,\n settings=settings):\n sec += x\n\n for x in print_subcommands(child, nested_content + subContent, markDownHelp,\n settings=settings):\n sec += x\n\n if 'epilog' in child and child['epilog']:\n for element in renderList([child['epilog']], markDownHelp):\n sec += element\n\n subCommands += sec\n items.append(subCommands)\n\n return items"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ensureUniqueIDs(items):\n s = set()\n for item in items:\n for n in item.traverse(descend=True, siblings=True, ascend=False):\n if isinstance(n, nodes.section):\n ids = n['ids']\n for idx, id in enumerate(ids):\n if id not in s:\n s.add(id)\n else:\n i = 1\n while \"{}_repeat{}\".format(id, i) in s:\n i += 1\n ids[idx] = \"{}_repeat{}\".format(id, i)\n s.add(ids[idx])\n n['ids'] = ids", "response": "This function updates the title IDs of the action groups that are repeated and the links in the table of contents will have unique IDs."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _construct_manpage_specific_structure(self, parser_info):\n items = []\n # SYNOPSIS section\n synopsis_section = nodes.section(\n '',\n nodes.title(text='Synopsis'),\n nodes.literal_block(text=parser_info[\"bare_usage\"]),\n ids=['synopsis-section'])\n items.append(synopsis_section)\n # DESCRIPTION section\n if 'nodescription' not in self.options:\n description_section = nodes.section(\n '',\n nodes.title(text='Description'),\n nodes.paragraph(text=parser_info.get(\n 'description', parser_info.get(\n 'help', \"undocumented\").capitalize())),\n ids=['description-section'])\n nested_parse_with_titles(\n self.state, self.content, description_section)\n items.append(description_section)\n if parser_info.get('epilog') and 'noepilog' not in self.options:\n # TODO: do whatever sphinx does to understand ReST inside\n # docstrings magically imported from other places. The nested\n # parse method invoked above seem to be able to do this but\n # I haven't found a way to do it for arbitrary text\n if description_section:\n description_section += nodes.paragraph(\n text=parser_info['epilog'])\n else:\n description_section = nodes.paragraph(\n text=parser_info['epilog'])\n items.append(description_section)\n # OPTIONS section\n options_section = nodes.section(\n '',\n nodes.title(text='Options'),\n ids=['options-section'])\n if 'args' in parser_info:\n options_section += nodes.paragraph()\n options_section += nodes.subtitle(text='Positional arguments:')\n options_section += self._format_positional_arguments(parser_info)\n for action_group in parser_info['action_groups']:\n if 'options' in parser_info:\n options_section += nodes.paragraph()\n options_section += nodes.subtitle(text=action_group['title'])\n options_section += self._format_optional_arguments(action_group)\n\n # NOTE: we cannot generate NAME ourselves. It is generated by\n # docutils.writers.manpage\n # TODO: items.append(files)\n # TODO: items.append(see also)\n # TODO: items.append(bugs)\n\n if len(options_section.children) > 1:\n items.append(options_section)\n if 'nosubcommands' not in self.options:\n # SUBCOMMANDS section (non-standard)\n subcommands_section = nodes.section(\n '',\n nodes.title(text='Sub-Commands'),\n ids=['subcommands-section'])\n if 'children' in parser_info:\n subcommands_section += self._format_subcommands(parser_info)\n if len(subcommands_section) > 1:\n items.append(subcommands_section)\n if os.getenv(\"INCLUDE_DEBUG_SECTION\"):\n import json\n # DEBUG section (non-standard)\n debug_section = nodes.section(\n '',\n nodes.title(text=\"Argparse + Sphinx Debugging\"),\n nodes.literal_block(text=json.dumps(parser_info, indent=' ')),\n ids=['debug-section'])\n items.append(debug_section)\n return items", "response": "Construct a typical man page consisting of the following elements."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef select(self, item):\n self._on_unselect[self._selected]()\n self.selected().unfocus()\n\n if isinstance(item, int):\n self._selected = item % len(self)\n else:\n self._selected = self.items.index(item)\n\n self.selected().focus()\n self._on_select[self._selected]()", "response": "Select an arbitrary item by possition or by reference."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds an action to make when an object is selected.", "response": "def on_select(self, item, action):\n \"\"\"\n Add an action to make when an object is selected.\n Only one action can be stored this way.\n \"\"\"\n\n if not isinstance(item, int):\n item = self.items.index(item)\n\n self._on_select[item] = action"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef on_unselect(self, item, action):\n if not isinstance(item, int):\n item = self.items.index(item)\n\n self._on_unselect[item] = action", "response": "Add an action to make when an object is unfocused."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add(self, widget, condition=lambda: 42):\n\n assert callable(condition)\n assert isinstance(widget, BaseWidget)\n self._widgets.append((widget, condition))\n\n return widget", "response": "Add a widget to the widows."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef remove(self, widget):\n for i, (wid, _) in enumerate(self._widgets):\n if widget is wid:\n del self._widgets[i]\n return True\n\n raise ValueError('Widget not in list')", "response": "Remove a widget from the window."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_on_event(self, e):\n if e.type == QUIT:\n self.running = False\n\n elif e.type == KEYDOWN:\n if e.key == K_ESCAPE:\n self.running = False\n\n elif e.key == K_F4 and e.mod & KMOD_ALT: # Alt+F4 --> quits\n self.running = False\n\n elif e.type == VIDEORESIZE:\n self.SCREEN_SIZE = e.size\n self.screen = self.new_screen()", "response": "Process a single event."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget all events and process them by calling update_on_event", "response": "def update(self):\n \"\"\"Get all events and process them by calling update_on_event()\"\"\"\n events = pygame.event.get()\n for e in events:\n self.update_on_event(e)\n\n for wid, cond in self._widgets:\n if cond():\n wid.update(events)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nrenders the screen. Here you must draw everything.", "response": "def render(self):\n \"\"\"Render the screen. Here you must draw everything.\"\"\"\n self.screen.fill(self.BACKGROUND_COLOR)\n\n for wid, cond in self._widgets:\n if cond():\n wid.render(self.screen)\n\n if self.BORDER_COLOR is not None:\n pygame.draw.rect(self.screen, self.BORDER_COLOR, ((0, 0), self.SCREEN_SIZE), 1)\n\n if self.SHOW_FPS:\n self.fps.render(self.screen)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update_screen(self):\n self.clock.tick(self.FPS)\n pygame.display.update()", "response": "Refresh the screen. You don't need to override this except to update only small portins of the screen."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nmaking a new screen with a size of SCREEN_SIZE and VIDEO_OPTION as flags. Sets the windows name to NAME.", "response": "def new_screen(self):\n \"\"\"Makes a new screen with a size of SCREEN_SIZE, and VIDEO_OPTION as flags. Sets the windows name to NAME.\"\"\"\n os.environ['SDL_VIDEO_CENTERED'] = '1'\n pygame.display.set_caption(self.NAME)\n\n screen_s = self.SCREEN_SIZE\n video_options = self.VIDEO_OPTIONS\n if FULLSCREEN & self.VIDEO_OPTIONS:\n video_options ^= FULLSCREEN\n video_options |= NOFRAME\n screen_s = (0, 0)\n\n screen = pygame.display.set_mode(screen_s, video_options)\n\n if FULLSCREEN & self.VIDEO_OPTIONS:\n self.SCREEN_SIZE = screen.get_size()\n\n if not QUIT in self.EVENT_ALLOWED:\n self.EVENT_ALLOWED = list(self.EVENT_ALLOWED)\n self.EVENT_ALLOWED.append(QUIT)\n\n pygame.event.set_allowed(self.EVENT_ALLOWED)\n\n return screen"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef merge_rects(rect1, rect2):\n r = pygame.Rect(rect1)\n t = pygame.Rect(rect2)\n\n right = max(r.right, t.right)\n bot = max(r.bottom, t.bottom)\n x = min(t.x, r.x)\n y = min(t.y, r.y)\n\n return pygame.Rect(x, y, right - x, bot - y)", "response": "Return the smallest rect containning two rects"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a vecor noraml to this one with a norm of one", "response": "def normnorm(self):\n \"\"\"\n Return a vecor noraml to this one with a norm of one\n\n :return: V2\n \"\"\"\n\n n = self.norm()\n return V2(-self.y / n, self.x / n)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef line(surf, start, end, color=BLACK, width=1, style=FLAT):\n\n width = round(width, 1)\n if width == 1:\n # return pygame.draw.aaline(surf, color, start, end)\n return gfxdraw.line(surf, *start, *end, color)\n\n start = V2(*start)\n end = V2(*end)\n\n line_vector = end - start\n half_side = line_vector.normnorm() * width / 2\n\n point1 = start + half_side\n point2 = start - half_side\n point3 = end - half_side\n point4 = end + half_side\n\n # noinspection PyUnresolvedReferences\n liste = [\n (point1.x, point1.y),\n (point2.x, point2.y),\n (point3.x, point3.y),\n (point4.x, point4.y)\n ]\n\n rect = polygon(surf, liste, color)\n\n if style == ROUNDED:\n _ = circle(surf, start, width / 2, color)\n rect = merge_rects(rect, _)\n _ = circle(surf, end, width / 2, color)\n rect = merge_rects(rect, _)\n\n return rect", "response": "Draws an antialiased line on the surface."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef circle(surf, xy, r, color=BLACK):\n\n x, y = xy\n\n x = round(x)\n y = round(y)\n r = round(r)\n\n gfxdraw.filled_circle(surf, x, y, r, color)\n gfxdraw.aacircle(surf, x, y, r, color)\n\n r += 1\n return pygame.Rect(x - r, y - r, 2 * r, 2 * r)", "response": "Draw an antialiased filled circle on the given surface."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndraws a ring of the image.", "response": "def ring(surf, xy, r, width, color):\n \"\"\"Draws a ring\"\"\"\n\n r2 = r - width\n\n x0, y0 = xy\n x = r2\n y = 0\n err = 0\n\n # collect points of the inner circle\n right = {}\n while x >= y:\n right[x] = y\n right[y] = x\n right[-x] = y\n right[-y] = x\n\n y += 1\n if err <= 0:\n err += 2 * y + 1\n if err > 0:\n x -= 1\n err -= 2 * x + 1\n\n def h_fill_the_circle(surf, color, x, y, right):\n if -r2 <= y <= r2:\n pygame.draw.line(surf, color, (x0 + right[y], y0 + y), (x0 + x, y0 + y))\n pygame.draw.line(surf, color, (x0 - right[y], y0 + y), (x0 - x, y0 + y))\n else:\n pygame.draw.line(surf, color, (x0 - x, y0 + y), (x0 + x, y0 + y))\n\n x = r\n y = 0\n err = 0\n\n while x >= y:\n\n h_fill_the_circle(surf, color, x, y, right)\n h_fill_the_circle(surf, color, x, -y, right)\n h_fill_the_circle(surf, color, y, x, right)\n h_fill_the_circle(surf, color, y, -x, right)\n\n y += 1\n if err < 0:\n err += 2 * y + 1\n if err >= 0:\n x -= 1\n err -= 2 * x + 1\n\n gfxdraw.aacircle(surf, x0, y0, r, color)\n gfxdraw.aacircle(surf, x0, y0, r2, color)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef roundrect(surface, rect, color, rounding=5, unit=PIXEL):\n\n if unit == PERCENT:\n rounding = int(min(rect.size) / 2 * rounding / 100)\n\n rect = pygame.Rect(rect)\n color = pygame.Color(*color)\n alpha = color.a\n color.a = 0\n pos = rect.topleft\n rect.topleft = 0, 0\n rectangle = pygame.Surface(rect.size, SRCALPHA)\n\n circle = pygame.Surface([min(rect.size) * 3] * 2, SRCALPHA)\n pygame.draw.ellipse(circle, (0, 0, 0), circle.get_rect(), 0)\n circle = pygame.transform.smoothscale(circle, (rounding, rounding))\n\n rounding = rectangle.blit(circle, (0, 0))\n rounding.bottomright = rect.bottomright\n rectangle.blit(circle, rounding)\n rounding.topright = rect.topright\n rectangle.blit(circle, rounding)\n rounding.bottomleft = rect.bottomleft\n rectangle.blit(circle, rounding)\n\n rectangle.fill((0, 0, 0), rect.inflate(-rounding.w, 0))\n rectangle.fill((0, 0, 0), rect.inflate(0, -rounding.h))\n\n rectangle.fill(color, special_flags=BLEND_RGBA_MAX)\n rectangle.fill((255, 255, 255, alpha), special_flags=BLEND_RGBA_MIN)\n\n return surface.blit(rectangle, pos)", "response": "Draw an antialiased round rectangle on the surface."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef polygon(surf, points, color):\n\n gfxdraw.aapolygon(surf, points, color)\n gfxdraw.filled_polygon(surf, points, color)\n\n x = min([x for (x, y) in points])\n y = min([y for (x, y) in points])\n xm = max([x for (x, y) in points])\n ym = max([y for (x, y) in points])\n\n return pygame.Rect(x, y, xm - x, ym - y)", "response": "Draw an antialiased filled polygon on a surface"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalling when the button is pressed. This start the callback function in a thread If :milis is given, will release the button after :milis miliseconds", "response": "def click(self, force_no_call=False, milis=None):\n \"\"\"\n Call when the button is pressed. This start the callback function in a thread\n If :milis is given, will release the button after :milis miliseconds\n \"\"\"\n\n if self.clicked:\n return False\n\n if not force_no_call and self.flags & self.CALL_ON_PRESS:\n if self.flags & self.THREADED_CALL:\n start_new_thread(self.func, ())\n else:\n self.func()\n\n super().click()\n\n if milis is not None:\n start_new_thread(self.release, (), {'milis': milis})"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the color of the button depending on its state", "response": "def _get_color(self):\n \"\"\"Return the color of the button, depending on its state\"\"\"\n if self.clicked and self.hovered: # the mouse is over the button\n color = mix(self.color, BLACK, 0.8)\n\n elif self.hovered and not self.flags & self.NO_HOVER:\n color = mix(self.color, BLACK, 0.93)\n\n else:\n color = self.color\n\n self.text.bg_color = color\n return color"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _front_delta(self):\n if self.flags & self.NO_MOVE:\n return Separator(0, 0)\n\n if self.clicked and self.hovered: # the mouse is over the button\n delta = 2\n\n elif self.hovered and not self.flags & self.NO_HOVER:\n delta = 0\n\n else:\n delta = 0\n\n return Separator(delta, delta)", "response": "Return the offset of the colored part."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update(self, event_or_list):\n\n for e in super().update(event_or_list):\n if e.type == MOUSEBUTTONDOWN:\n if e.pos in self:\n self.click()\n else:\n self.release(force_no_call=True)\n\n elif e.type == MOUSEBUTTONUP:\n self.release(force_no_call=e.pos not in self)\n\n elif e.type == MOUSEMOTION:\n if e.pos in self:\n self.hovered = True\n else:\n self.hovered = False", "response": "Update the button with the events."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrendering the button on a surface.", "response": "def render(self, surf):\n \"\"\"Render the button on a surface.\"\"\"\n pos, size = self.topleft, self.size\n\n if not self.flags & self.NO_SHADOW:\n if self.flags & self.NO_ROUNDING:\n pygame.draw.rect(surf, LIGHT_GREY, (pos + self._bg_delta, size))\n else:\n roundrect(surf, (pos + self._bg_delta, size), LIGHT_GREY + (100,), 5)\n\n if self.flags & self.NO_ROUNDING:\n pygame.draw.rect(surf, self._get_color(), (pos + self._front_delta, size))\n else:\n roundrect(surf, (pos + self._front_delta, size), self._get_color(), 5)\n\n self.text.center = self.center + self._front_delta\n self.text.render(surf)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef render(self, surf):\n if not self.flags & self.NO_SHADOW:\n circle(surf, self.center + self._bg_delta, self.width / 2, LIGHT_GREY)\n circle(surf, self.center + self._front_delta, self.width / 2, self._get_color())\n\n self.text.center = self.center + self._front_delta\n self.text.render(surf)", "response": "Draw the button on the surface."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_darker_image(self):\n icon_pressed = self.icon.copy()\n\n for x in range(self.w):\n for y in range(self.h):\n r, g, b, *_ = tuple(self.icon.get_at((x, y)))\n const = 0.8\n r = int(const * r)\n g = int(const * g)\n b = int(const * b)\n icon_pressed.set_at((x, y), (r, g, b))\n\n return icon_pressed", "response": "Returns an icon 80% more dark"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set(self, value):\n value = min(self.max, max(self.min, value))\n self._value = value\n start_new_thread(self.func, (self.get(),))", "response": "Set the value of the bar."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nstart checking if the SB is shifted", "response": "def _start(self):\n \"\"\"Starts checking if the SB is shifted\"\"\"\n\n # TODO : make an update method instead\n\n last_call = 42\n while self._focus:\n sleep(1 / 100)\n\n mouse = pygame.mouse.get_pos()\n last_value = self.get()\n self.value_px = mouse[0]\n\n # we do not need to do anything when it the same value\n if self.get() == last_value:\n continue\n\n if last_call + self.interval / 1000 < time():\n last_call = time()\n self.func(self.get())"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef render(self, display):\n\n # the bar\n bar_rect = pygame.Rect(0, 0, self.width, self.height // 3)\n bar_rect.center = self.center\n display.fill(self.bg_color, bar_rect)\n\n # the cursor\n circle(display, (self.value_px, self.centery), self.height // 2, self.color)\n\n # the value\n if self.show_val:\n self.text_val.render(display)", "response": "Renders the bar on the display"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __update(self):\n\n # I can not set the size attr because it is my property, so I set the width and height separately\n width, height = self.size\n super(BaseWidget, self).__setattr__(\"width\", width)\n super(BaseWidget, self).__setattr__(\"height\", height)\n super(BaseWidget, self).__setattr__(self.anchor, self.pos)", "response": "Update the size and anchor attributes of the base widget."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nmaking a compatible version of pip importable. Raise a RuntimeError if we couldn t.", "response": "def activate(specifier):\n \"\"\"Make a compatible version of pip importable. Raise a RuntimeError if we\n couldn't.\"\"\"\n try:\n for distro in require(specifier):\n distro.activate()\n except (VersionConflict, DistributionNotFound):\n raise RuntimeError('The installed version of pip is too old; peep '\n 'requires ' + specifier)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the path and line number of the file from which an was came.", "response": "def path_and_line(req):\n \"\"\"Return the path and line number of the file from which an\n InstallRequirement came.\n\n \"\"\"\n path, line = (re.match(r'-r (.*) \\(line (\\d+)\\)$',\n req.comes_from).groups())\n return path, int(line)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nyields hashes from contiguous comment lines before line line_number.", "response": "def hashes_above(path, line_number):\n \"\"\"Yield hashes from contiguous comment lines before line ``line_number``.\n\n \"\"\"\n def hash_lists(path):\n \"\"\"Yield lists of hashes appearing between non-comment lines.\n\n The lists will be in order of appearance and, for each non-empty\n list, their place in the results will coincide with that of the\n line number of the corresponding result from `parse_requirements`\n (which changed in pip 7.0 to not count comments).\n\n \"\"\"\n hashes = []\n with open(path) as file:\n for lineno, line in enumerate(file, 1):\n match = HASH_COMMENT_RE.match(line)\n if match: # Accumulate this hash.\n hashes.append(match.groupdict()['hash'])\n if not IGNORED_LINE_RE.match(line):\n yield hashes # Report hashes seen so far.\n hashes = []\n elif PIP_COUNTS_COMMENTS:\n # Comment: count as normal req but have no hashes.\n yield []\n\n return next(islice(hash_lists(path), line_number - 1, None))"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndelegates to pip and raise PipException if something goes wrong.", "response": "def run_pip(initial_args):\n \"\"\"Delegate to pip the given args (starting with the subcommand), and raise\n ``PipException`` if something goes wrong.\"\"\"\n status_code = pip.main(initial_args)\n\n # Clear out the registrations in the pip \"logger\" singleton. Otherwise,\n # loggers keep getting appended to it with every run. Pip assumes only one\n # command invocation will happen per interpreter lifetime.\n logger.consumers = []\n\n if status_code:\n raise PipException(status_code)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the hash of a downloaded file.", "response": "def hash_of_file(path):\n \"\"\"Return the hash of a downloaded file.\"\"\"\n with open(path, 'rb') as archive:\n sha = sha256()\n while True:\n data = archive.read(2 ** 20)\n if not data:\n break\n sha.update(data)\n return encoded_hash(sha)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef requirement_args(argv, want_paths=False, want_other=False):\n was_r = False\n for arg in argv:\n # Allow for requirements files named \"-r\", don't freak out if there's a\n # trailing \"-r\", etc.\n if was_r:\n if want_paths:\n yield arg\n was_r = False\n elif arg in ['-r', '--requirement']:\n was_r = True\n else:\n if want_other:\n yield arg", "response": "Yields the arguments that are passed to the requirement - file."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the peep hash of one or more files returning a shell status code otherwise raising a PipException.", "response": "def peep_hash(argv):\n \"\"\"Return the peep hash of one or more files, returning a shell status code\n or raising a PipException.\n\n :arg argv: The commandline args, starting after the subcommand\n\n \"\"\"\n parser = OptionParser(\n usage='usage: %prog hash file [file ...]',\n description='Print a peep hash line for one or more files: for '\n 'example, \"# sha256: '\n 'oz42dZy6Gowxw8AelDtO4gRgTW_xPdooH484k7I5EOY\".')\n _, paths = parser.parse_args(args=argv)\n if paths:\n for path in paths:\n print('# sha256:', hash_of_file(path))\n return ITS_FINE_ITS_FINE\n else:\n parser.print_usage()\n return COMMAND_LINE_ERROR"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef memoize(func):\n @wraps(func)\n def memoizer(self):\n if not hasattr(self, '_cache'):\n self._cache = {}\n if func.__name__ not in self._cache:\n self._cache[func.__name__] = func(self)\n return self._cache[func.__name__]\n return memoizer", "response": "Memoize a method that should return the same result every time on a\n given instance."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a PackageFinder that will find all the packages that are available on the private node.", "response": "def package_finder(argv):\n \"\"\"Return a PackageFinder respecting command-line options.\n\n :arg argv: Everything after the subcommand\n\n \"\"\"\n # We instantiate an InstallCommand and then use some of its private\n # machinery--its arg parser--for our own purposes, like a virus. This\n # approach is portable across many pip versions, where more fine-grained\n # ones are not. Ignoring options that don't exist on the parser (for\n # instance, --use-wheel) gives us a straightforward method of backward\n # compatibility.\n try:\n command = InstallCommand()\n except TypeError:\n # This is likely pip 1.3.0's \"__init__() takes exactly 2 arguments (1\n # given)\" error. In that version, InstallCommand takes a top=level\n # parser passed in from outside.\n from pip.baseparser import create_main_parser\n command = InstallCommand(create_main_parser())\n # The downside is that it essentially ruins the InstallCommand class for\n # further use. Calling out to pip.main() within the same interpreter, for\n # example, would result in arguments parsed this time turning up there.\n # Thus, we deepcopy the arg parser so we don't trash its singletons. Of\n # course, deepcopy doesn't work on these objects, because they contain\n # uncopyable regex patterns, so we pickle and unpickle instead. Fun!\n options, _ = loads(dumps(command.parser)).parse_args(argv)\n\n # Carry over PackageFinder kwargs that have [about] the same names as\n # options attr names:\n possible_options = [\n 'find_links',\n FORMAT_CONTROL_ARG,\n ('allow_all_prereleases', 'pre'),\n 'process_dependency_links'\n ]\n kwargs = {}\n for option in possible_options:\n kw, attr = option if isinstance(option, tuple) else (option, option)\n value = getattr(options, attr, MARKER)\n if value is not MARKER:\n kwargs[kw] = value\n\n # Figure out index_urls:\n index_urls = [options.index_url] + options.extra_index_urls\n if options.no_index:\n index_urls = []\n index_urls += getattr(options, 'mirrors', [])\n\n # If pip is new enough to have a PipSession, initialize one, since\n # PackageFinder requires it:\n if hasattr(command, '_build_session'):\n kwargs['session'] = command._build_session(options)\n\n return PackageFinder(index_urls=index_urls, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a map of key - > list of things.", "response": "def bucket(things, key):\n \"\"\"Return a map of key -> list of things.\"\"\"\n ret = defaultdict(list)\n for thing in things:\n ret[key(thing)].append(thing)\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef first_every_last(iterable, first, every, last):\n did_first = False\n for item in iterable:\n if not did_first:\n did_first = True\n first(item)\n every(item)\n if did_first:\n last(item)", "response": "Execute something before the first item of iter and something else for each\nTaxonomy item and a third thing after the last item."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef downloaded_reqs_from_path(path, argv):\n finder = package_finder(argv)\n return [DownloadedReq(req, argv, finder) for req in\n _parse_requirements(path, finder)]", "response": "Return a list of DownloadedReqs representing the requirements parsed from a given requirements file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nperforming the peep install subcommand returning a shell status code.", "response": "def peep_install(argv):\n \"\"\"Perform the ``peep install`` subcommand, returning a shell status code\n or raising a PipException.\n\n :arg argv: The commandline args, starting after the subcommand\n\n \"\"\"\n output = []\n out = output.append\n reqs = []\n try:\n req_paths = list(requirement_args(argv, want_paths=True))\n if not req_paths:\n out(\"You have to specify one or more requirements files with the -r option, because\\n\"\n \"otherwise there's nowhere for peep to look up the hashes.\\n\")\n return COMMAND_LINE_ERROR\n\n # We're a \"peep install\" command, and we have some requirement paths.\n reqs = list(chain.from_iterable(\n downloaded_reqs_from_path(path, argv)\n for path in req_paths))\n buckets = bucket(reqs, lambda r: r.__class__)\n\n # Skip a line after pip's \"Cleaning up...\" so the important stuff\n # stands out:\n if any(buckets[b] for b in ERROR_CLASSES):\n out('\\n')\n\n printers = (lambda r: out(r.head()),\n lambda r: out(r.error() + '\\n'),\n lambda r: out(r.foot()))\n for c in ERROR_CLASSES:\n first_every_last(buckets[c], *printers)\n\n if any(buckets[b] for b in ERROR_CLASSES):\n out('-------------------------------\\n'\n 'Not proceeding to installation.\\n')\n return SOMETHING_WENT_WRONG\n else:\n for req in buckets[InstallableReq]:\n req.install()\n\n first_every_last(buckets[SatisfiedReq], *printers)\n\n return ITS_FINE_ITS_FINE\n except (UnsupportedRequirementError, InstallationError, DownloadError) as exc:\n out(str(exc))\n return SOMETHING_WENT_WRONG\n finally:\n for req in reqs:\n req.dispose()\n print(''.join(output))"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef peep_port(paths):\n if not paths:\n print('Please specify one or more requirements files so I have '\n 'something to port.\\n')\n return COMMAND_LINE_ERROR\n\n comes_from = None\n for req in chain.from_iterable(\n _parse_requirements(path, package_finder(argv)) for path in paths):\n req_path, req_line = path_and_line(req)\n hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii')\n for hash in hashes_above(req_path, req_line)]\n if req_path != comes_from:\n print()\n print('# from %s' % req_path)\n print()\n comes_from = req_path\n\n if not hashes:\n print(req.req)\n else:\n print('%s' % (req.link if getattr(req, 'link', None) else req.req), end='')\n for hash in hashes:\n print(' \\\\')\n print(' --hash=sha256:%s' % hash, end='')\n print()", "response": "Convert a peep requirements file to one compatble with pip - 8 hashing."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef main():\n commands = {'hash': peep_hash,\n 'install': peep_install,\n 'port': peep_port}\n try:\n if len(argv) >= 2 and argv[1] in commands:\n return commands[argv[1]](argv[2:])\n else:\n # Fall through to top-level pip main() for everything else:\n return pip.main()\n except PipException as exc:\n return exc.error_code", "response": "Be the top - level entrypoint. Return a shell status code."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _version(self):\n # TODO: Can we delete this method and just print the line from the\n # reqs file verbatim instead?\n def version_of_archive(filename, package_name):\n # Since we know the project_name, we can strip that off the left, strip\n # any archive extensions off the right, and take the rest as the\n # version.\n for ext in ARCHIVE_EXTENSIONS:\n if filename.endswith(ext):\n filename = filename[:-len(ext)]\n break\n # Handle github sha tarball downloads.\n if is_git_sha(filename):\n filename = package_name + '-' + filename\n if not filename.lower().replace('_', '-').startswith(package_name.lower()):\n # TODO: Should we replace runs of [^a-zA-Z0-9.], not just _, with -?\n give_up(filename, package_name)\n return filename[len(package_name) + 1:] # Strip off '-' before version.\n\n def version_of_wheel(filename, package_name):\n # For Wheel files (http://legacy.python.org/dev/peps/pep-0427/#file-\n # name-convention) we know the format bits are '-' separated.\n whl_package_name, version, _rest = filename.split('-', 2)\n # Do the alteration to package_name from PEP 427:\n our_package_name = re.sub(r'[^\\w\\d.]+', '_', package_name, re.UNICODE)\n if whl_package_name != our_package_name:\n give_up(filename, whl_package_name)\n return version\n\n def give_up(filename, package_name):\n raise RuntimeError(\"The archive '%s' didn't start with the package name \"\n \"'%s', so I couldn't figure out the version number. \"\n \"My bad; improve me.\" %\n (filename, package_name))\n\n get_version = (version_of_wheel\n if self._downloaded_filename().endswith('.whl')\n else version_of_archive)\n return get_version(self._downloaded_filename(), self._project_name())", "response": "Deduce the version number of the downloaded package from its filename."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _is_always_unsatisfied(self):\n # If this is a github sha tarball, then it is always unsatisfied\n # because the url has a commit sha in it and not the version\n # number.\n url = self._url()\n if url:\n filename = filename_from_url(url)\n if filename.endswith(ARCHIVE_EXTENSIONS):\n filename, ext = splitext(filename)\n if is_git_sha(filename):\n return True\n return False", "response": "Returns whether this requirement is always unsatisfied."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _download(self, link):\n # Based on pip 1.4.1's URLOpener but with cert verification removed\n def opener(is_https):\n if is_https:\n opener = build_opener(HTTPSHandler())\n # Strip out HTTPHandler to prevent MITM spoof:\n for handler in opener.handlers:\n if isinstance(handler, HTTPHandler):\n opener.handlers.remove(handler)\n else:\n opener = build_opener()\n return opener\n\n # Descended from unpack_http_url() in pip 1.4.1\n def best_filename(link, response):\n \"\"\"Return the most informative possible filename for a download,\n ideally with a proper extension.\n\n \"\"\"\n content_type = response.info().get('content-type', '')\n filename = link.filename # fallback\n # Have a look at the Content-Disposition header for a better guess:\n content_disposition = response.info().get('content-disposition')\n if content_disposition:\n type, params = cgi.parse_header(content_disposition)\n # We use ``or`` here because we don't want to use an \"empty\" value\n # from the filename param:\n filename = params.get('filename') or filename\n ext = splitext(filename)[1]\n if not ext:\n ext = mimetypes.guess_extension(content_type)\n if ext:\n filename += ext\n if not ext and link.url != response.geturl():\n ext = splitext(response.geturl())[1]\n if ext:\n filename += ext\n return filename\n\n # Descended from _download_url() in pip 1.4.1\n def pipe_to_file(response, path, size=0):\n \"\"\"Pull the data off an HTTP response, shove it in a new file, and\n show progress.\n\n :arg response: A file-like object to read from\n :arg path: The path of the new file\n :arg size: The expected size, in bytes, of the download. 0 for\n unknown or to suppress progress indication (as for cached\n downloads)\n\n \"\"\"\n def response_chunks(chunk_size):\n while True:\n chunk = response.read(chunk_size)\n if not chunk:\n break\n yield chunk\n\n print('Downloading %s%s...' % (\n self._req.req,\n (' (%sK)' % (size / 1000)) if size > 1000 else ''))\n progress_indicator = (DownloadProgressBar(max=size).iter if size\n else DownloadProgressSpinner().iter)\n with open(path, 'wb') as file:\n for chunk in progress_indicator(response_chunks(4096), 4096):\n file.write(chunk)\n\n url = link.url.split('#', 1)[0]\n try:\n response = opener(urlparse(url).scheme != 'http').open(url)\n except (HTTPError, IOError) as exc:\n raise DownloadError(link, exc)\n filename = best_filename(link, response)\n try:\n size = int(response.headers['content-length'])\n except (ValueError, KeyError, TypeError):\n size = 0\n pipe_to_file(response, join(self._temp_path, filename), size=size)\n return filename", "response": "Download a file and return its name within my temp dir."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _downloaded_filename(self):\n # Peep doesn't support requirements that don't come down as a single\n # file, because it can't hash them. Thus, it doesn't support editable\n # requirements, because pip itself doesn't support editable\n # requirements except for \"local projects or a VCS url\". Nor does it\n # support VCS requirements yet, because we haven't yet come up with a\n # portable, deterministic way to hash them. In summary, all we support\n # is == requirements and tarballs/zips/etc.\n\n # TODO: Stop on reqs that are editable or aren't ==.\n\n # If the requirement isn't already specified as a URL, get a URL\n # from an index:\n link = self._link() or self._finder.find_requirement(self._req, upgrade=False)\n\n if link:\n lower_scheme = link.scheme.lower() # pip lower()s it for some reason.\n if lower_scheme == 'http' or lower_scheme == 'https':\n file_path = self._download(link)\n return basename(file_path)\n elif lower_scheme == 'file':\n # The following is inspired by pip's unpack_file_url():\n link_path = url_to_path(link.url_without_fragment)\n if isdir(link_path):\n raise UnsupportedRequirementError(\n \"%s: %s is a directory. So that it can compute \"\n \"a hash, peep supports only filesystem paths which \"\n \"point to files\" %\n (self._req, link.url_without_fragment))\n else:\n copy(link_path, self._temp_path)\n return basename(link_path)\n else:\n raise UnsupportedRequirementError(\n \"%s: The download link, %s, would not result in a file \"\n \"that can be hashed. Peep supports only == requirements, \"\n \"file:// URLs pointing to files (not folders), and \"\n \"http:// and https:// URLs pointing to tarballs, zips, \"\n \"etc.\" % (self._req, link.url))\n else:\n raise UnsupportedRequirementError(\n \"%s: couldn't determine where to download this requirement from.\"\n % (self._req,))", "response": "Download the package s archive if necessary and return its\n filename."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninstalling the package I represent without dependencies.", "response": "def install(self):\n \"\"\"Install the package I represent, without dependencies.\n\n Obey typical pip-install options passed in on the command line.\n\n \"\"\"\n other_args = list(requirement_args(self._argv, want_other=True))\n archive_path = join(self._temp_path, self._downloaded_filename())\n # -U so it installs whether pip deems the requirement \"satisfied\" or\n # not. This is necessary for GitHub-sourced zips, which change without\n # their version numbers changing.\n run_pip(['install'] + other_args + ['--no-deps', '-U', archive_path])"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the inner Requirement s safe name. Raise ValueError if there is no name.", "response": "def _project_name(self):\n \"\"\"Return the inner Requirement's \"unsafe name\".\n\n Raise ValueError if there is no name.\n\n \"\"\"\n name = getattr(self._req.req, 'project_name', '')\n if name:\n return name\n name = getattr(self._req.req, 'name', '')\n if name:\n return safe_name(name)\n raise ValueError('Requirement has no project_name.')"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the class I should be spanning a continuum of goodness.", "response": "def _class(self):\n \"\"\"Return the class I should be, spanning a continuum of goodness.\"\"\"\n try:\n self._project_name()\n except ValueError:\n return MalformedReq\n if self._is_satisfied():\n return SatisfiedReq\n if not self._expected_hashes():\n return MissingReq\n if self._actual_hash() not in self._expected_hashes():\n return MismatchedReq\n return InstallableReq"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef check(self):\n for k in iter(self.__dict__.keys()):\n if k not in self.__allowed:\n raise TypeError(\"Parameter not allowed {}\".format(k))\n\n for k in self.__requiered:\n if k not in self.__dict__:\n raise TypeError(\"Requiered parameter not found {}\".format(k))\n\n if not isinstance(self.units, (int, float)):\n msg = \"Unit must be either int or float, '{}'' found\".format(\n type(self.units))\n raise TypeError(msg)\n\n if self.side not in self.__side:\n msg = \"Side must be in {1}, '{0}' found\".format(\n self.side, self.__side)\n raise TypeError(msg)\n\n if self.type not in self.__type:\n msg = \"Type must be in {1}, '{0}' found\".format(\n self.type, self.__type)\n raise TypeError(msg)\n\n if not self.type == \"market\" and (\n not hasattr(self, \"expiry\") or not hasattr(self, \"price\")):\n msg = \"As type is {}, expiry and price must be provided\".format(\n self.type)\n raise TypeError(msg)\n if hasattr(self, \"expiry\") and not isinstance(self.expiry, datetime):\n msg = \"Expiry must be {1}, '{0}' found\".format(\n type(self.expiry), datetime)\n raise TypeError(msg)\n\n if hasattr(self, \"price\"):\n try:\n Decimal(self.price)\n except InvalidOperation:\n msg = \"Expiry must be int or float, '{0}' found\".format(\n type(self.price))\n raise TypeError(msg)\n\n return True", "response": "Checks that the current object is valid."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef gui():\n\n global SCREEN_SIZE\n\n # #######\n # setup all objects\n # #######\n\n os.environ['SDL_VIDEO_CENTERED'] = '1' # centers the windows\n\n screen = new_screen()\n pygame.display.set_caption('Empty project')\n pygame.event.set_allowed([QUIT, KEYDOWN, MOUSEBUTTONDOWN])\n clock = pygame.time.Clock()\n fps = FPSIndicator(clock)\n\n while True:\n\n # #######\n # Input loop\n # #######\n\n # mouse = pygame.mouse.get_pos()\n for e in pygame.event.get():\n if e.type == QUIT:\n return 0\n\n elif e.type == KEYDOWN:\n if e.key == K_ESCAPE:\n return 0\n\n if e.key == K_F4 and e.mod & KMOD_ALT: # Alt+F4 --> quits\n return 0\n\n if e.type == VIDEORESIZE:\n SCREEN_SIZE = e.size\n screen = new_screen()\n\n # #######\n # Draw all\n # #######\n\n screen.fill(WHITE)\n fps.render(screen)\n\n pygame.display.update()\n clock.tick(FPS)", "response": "Main function for the main window."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a requests. Response object with the given params and option", "response": "def __get_response(self, uri, params=None, method=\"get\", stream=False):\n \"\"\"Creates a response object with the given params and option\n\n Parameters\n ----------\n url : string\n The full URL to request.\n params: dict\n A list of parameters to send with the request. This\n will be sent as data for methods that accept a request\n body and will otherwise be sent as query parameters.\n method : str\n The HTTP method to use.\n stream : bool\n Whether to stream the response.\n\n Returns a requests.Response object.\n \"\"\"\n if not hasattr(self, \"session\") or not self.session:\n self.session = requests.Session()\n if self.access_token:\n self.session.headers.update(\n {'Authorization': 'Bearer {}'.format(self.access_token)}\n )\n\n # Remove empty params\n if params:\n params = {k: v for k, v in params.items() if v is not None}\n\n kwargs = {\n \"url\": uri,\n \"verify\": True,\n \"stream\": stream\n }\n\n kwargs[\"params\" if method == \"get\" else \"data\"] = params\n\n return getattr(self.session, method)(**kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __call_stream(self, uri, params=None, method=\"get\"):\n try:\n resp = self.__get_response(uri, params, method, True)\n assert resp.ok\n except AssertionError:\n raise BadRequest(resp.status_code)\n except Exception as e:\n log.error(\"Bad response: {}\".format(e), exc_info=True)\n else:\n return resp", "response": "Returns an HTTP response for a given uri."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_instruments(self):\n url = \"{0}/{1}/instruments\".format(self.domain, self.API_VERSION)\n params = {\"accountId\": self.account_id}\n try:\n response = self._Client__call(uri=url, params=params)\n assert len(response) > 0\n return response\n except RequestException:\n return False\n except AssertionError:\n return False", "response": "Get the list of instrument names for the current account."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_prices(self, instruments, stream=True):\n url = \"{0}/{1}/prices\".format(\n self.domain_stream if stream else self.domain,\n self.API_VERSION\n )\n params = {\"accountId\": self.account_id, \"instruments\": instruments}\n\n call = {\"uri\": url, \"params\": params, \"method\": \"get\"}\n\n try:\n if stream:\n return self._Client__call_stream(**call)\n else:\n return self._Client__call(**call)\n except RequestException:\n return False\n except AssertionError:\n return False", "response": "Get the current price for a given set of instruments."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_orders(self, instrument=None, count=50):\n url = \"{0}/{1}/accounts/{2}/orders\".format(\n self.domain,\n self.API_VERSION,\n self.account_id\n )\n params = {\"instrument\": instrument, \"count\": count}\n try:\n return self._Client__call(uri=url, params=params, method=\"get\")\n except RequestException:\n return False\n except AssertionError:\n return False", "response": "Get the orders for an account."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_order(self, order_id):\n url = \"{0}/{1}/accounts/{2}/orders/{3}\".format(\n self.domain,\n self.API_VERSION,\n self.account_id,\n order_id\n )\n try:\n return self._Client__call(uri=url, method=\"get\")\n except RequestException:\n return False\n except AssertionError:\n return False", "response": "Get the order information for a specific order."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a new order in the order store.", "response": "def create_order(self, order):\n \"\"\"\n See more:\n http://developer.oanda.com/rest-live/orders/#createNewOrder\n \"\"\"\n url = \"{0}/{1}/accounts/{2}/orders\".format(\n self.domain,\n self.API_VERSION,\n self.account_id\n )\n try:\n return self._Client__call(\n uri=url,\n params=order.__dict__,\n method=\"post\"\n )\n except RequestException:\n return False\n except AssertionError:\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_trades(self, max_id=None, count=None, instrument=None, ids=None):\n url = \"{0}/{1}/accounts/{2}/trades\".format(\n self.domain,\n self.API_VERSION,\n self.account_id\n )\n params = {\n \"maxId\": int(max_id) if max_id and max_id > 0 else None,\n \"count\": int(count) if count and count > 0 else None,\n \"instrument\": instrument,\n \"ids\": ','.join(ids) if ids else None\n }\n\n try:\n return self._Client__call(uri=url, params=params, method=\"get\")\n except RequestException:\n return False\n except AssertionError:\n return False", "response": "This method returns a list of open trades for a specific instrument."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmodifies an existing trade.", "response": "def update_trade(\n self,\n trade_id,\n stop_loss=None,\n take_profit=None,\n trailing_stop=None\n ):\n \"\"\" Modify an existing trade.\n\n Note: Only the specified parameters will be modified. All\n other parameters will remain unchanged. To remove an\n optional parameter, set its value to 0.\n\n Parameters\n ----------\n trade_id : int\n The id of the trade to modify.\n stop_loss : number\n Stop Loss value.\n take_profit : number\n Take Profit value.\n trailing_stop : number\n Trailing Stop distance in pips, up to one decimal place\n\n See more:\n http://developer.oanda.com/rest-live/trades/#modifyExistingTrade\n \"\"\"\n url = \"{0}/{1}/accounts/{2}/trades/{3}\".format(\n self.domain,\n self.API_VERSION,\n self.account_id,\n trade_id\n )\n params = {\n \"stopLoss\": stop_loss,\n \"takeProfit\": take_profit,\n \"trailingStop\": trailing_stop\n }\n try:\n return self._Client__call(uri=url, params=params, method=\"patch\")\n except RequestException:\n return False\n except AssertionError:\n return False\n raise NotImplementedError()"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nrequests full account history.", "response": "def request_transaction_history(self):\n \"\"\" Request full account history.\n\n Submit a request for a full transaction history. A\n successfully accepted submission results in a response\n containing a URL in the Location header to a file that will\n be available once the request is served. Response for the\n URL will be HTTP 404 until the file is ready. Once served\n the URL will be valid for a certain amount of time.\n\n See more:\n http://developer.oanda.com/rest-live/transaction-history/#getFullAccountHistory\n http://developer.oanda.com/rest-live/transaction-history/#transactionTypes\n \"\"\"\n url = \"{0}/{1}/accounts/{2}/alltransactions\".format(\n self.domain,\n self.API_VERSION,\n self.account_id\n )\n try:\n resp = self.__get_response(url)\n return resp.headers['location']\n except RequestException:\n return False\n except AssertionError:\n return False"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndownloading full account history.", "response": "def get_transaction_history(self, max_wait=5.0):\n \"\"\" Download full account history.\n\n Uses request_transaction_history to get the transaction\n history URL, then polls the given URL until it's ready (or\n the max_wait time is reached) and provides the decoded\n response.\n\n Parameters\n ----------\n max_wait : float\n The total maximum time to spend waiting for the file to\n be ready; if this is exceeded a failed response will be\n returned. This is not guaranteed to be strictly\n followed, as one last attempt will be made to check the\n file before giving up.\n\n See more:\n http://developer.oanda.com/rest-live/transaction-history/#getFullAccountHistory\n http://developer.oanda.com/rest-live/transaction-history/#transactionTypes\n \"\"\"\n url = self.request_transaction_history()\n if not url:\n return False\n\n ready = False\n start = time()\n delay = 0.1\n while not ready and delay:\n response = requests.head(url)\n ready = response.ok\n if not ready:\n sleep(delay)\n time_remaining = max_wait - time() + start\n max_delay = max(0., time_remaining - .1)\n delay = min(delay * 2, max_delay)\n\n if not ready:\n return False\n\n response = requests.get(url)\n try:\n with ZipFile(BytesIO(response.content)) as container:\n files = container.namelist()\n if not files:\n log.error('Transaction ZIP has no files.')\n return False\n history = container.open(files[0])\n raw = history.read().decode('ascii')\n except BadZipfile:\n log.error('Response is not a valid ZIP file', exc_info=True)\n return False\n\n return json.loads(raw, **self.json_options)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_account(self, currency=None):\n url = \"{0}/{1}/accounts\".format(self.domain, self.API_VERSION)\n params = {\"currency\": currency}\n try:\n return self._Client__call(uri=url, params=params, method=\"post\")\n except RequestException:\n return False\n except AssertionError:\n return False", "response": "Create a new account."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_accounts(self, username=None):\n url = \"{0}/{1}/accounts\".format(self.domain, self.API_VERSION)\n params = {\"username\": username}\n try:\n return self._Client__call(uri=url, params=params, method=\"get\")\n except RequestException:\n return False\n except AssertionError:\n return False", "response": "Get a list of accounts owned by a user."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef choose(self):\n if not self.choosed:\n self.choosed = True\n self.pos = self.pos + Sep(5, 0)", "response": "Marks the item as the one the user is in."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef stop_choose(self):\n if self.choosed:\n self.choosed = False\n self.pos = self.pos + Sep(-5, 0)", "response": "Marks the item as the one the user is not in."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef gui():\n\n # #######\n # setup all objects\n # #######\n os.environ['SDL_VIDEO_CENTERED'] = '1'\n\n clock = pygame.time.Clock()\n screen = pygame.display.set_mode(SCREEN_SIZE, DOUBLEBUF | NOFRAME)\n pygame.event.set_allowed([QUIT, KEYDOWN, MOUSEBUTTONDOWN])\n\n game = Morpion()\n\n run = True\n while run:\n\n # #######\n # Input loop\n # #######\n\n mouse = pygame.mouse.get_pos()\n for e in pygame.event.get():\n if e.type == QUIT:\n run = False\n\n elif e.type == KEYDOWN:\n if e.key == K_ESCAPE:\n run = False\n\n if e.key == K_F4 and e.mod & KMOD_ALT:\n return 0\n\n elif e.type == MOUSEBUTTONDOWN:\n if e.button == 1:\n if pos_from_mouse(mouse):\n\n if game.is_full() or game.is_won():\n game = Morpion()\n continue\n\n x, y = pos_from_mouse(mouse)\n\n try:\n game.play(x, y)\n except IndexError:\n pass\n\n if pos_from_mouse(mouse):\n x, y = pos_from_mouse(mouse)\n game.hint(x, y)\n\n # #######\n # Draw all\n # #######\n\n screen.fill(WHITE)\n game.render(screen)\n\n pygame.display.update()\n clock.tick(FPS)", "response": "Main function for the main loop of the main loop."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts a size in pxel to a size in points.", "response": "def px_to_pt(self, px):\n \"\"\"Convert a size in pxel to a size in points.\"\"\"\n if px < 200:\n pt = self.PX_TO_PT[px]\n else:\n pt = int(floor((px - 1.21) / 1.332))\n\n return pt"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_size(self, pt=None, px=None):\n\n assert (pt, px) != (None, None)\n\n if pt is not None:\n self.__init__(pt, self.font_name)\n else:\n self.__init__(self.px_to_pt(px), self.font_name)", "response": "Set the size of the font in pt or px."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef text(self):\n if callable(self._text):\n return str(self._text())\n return str(self._text)", "response": "Return the string to render."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef color(self, value):\n\n if value != self.color:\n self._color = value\n self._render()", "response": "Set the color to a new value. Renders the text if needed."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the color of the log entry.", "response": "def bg_color(self, value):\n \"\"\"Sets the color to a new value (tuple). Renders the text if needed.\"\"\"\n\n if value != self.bg_color:\n self._bg_color = value\n self._render()"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_font_size(self, pt=None, px=None):\n self.font.set_size(pt, px)\n\n self._render()", "response": "Set the font size in pt or px."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _render(self):\n\n self._last_text = self.text\n\n self._surface = self.font.render(self.text, True, self.color, self.bg_color)\n rect = self._surface.get_rect()\n\n self.size = rect.size", "response": "Render the text.\n\n Avoid using this fonction too many time as it is slow as it is low to render text and blit it."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef render(self, display):\n # to handle changing objects / callable\n if self.text != self._last_text:\n self._render()\n\n display.blit(self._surface, (self.topleft, self.size))", "response": "Render basicly the text."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef move_cursor_one_letter(self, letter=RIGHT):\n assert letter in (self.RIGHT, self.LEFT)\n\n if letter == self.RIGHT:\n self.cursor += 1\n if self.cursor > len(self.text):\n self.cursor -= 1\n else:\n self.cursor -= 1\n if self.cursor < 0:\n self.cursor += 1", "response": "Move the cursor of one letter to the right or the left."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef move_cursor_one_word(self, word=LEFT):\n\n assert word in (self.RIGHT, self.LEFT)\n\n if word == self.RIGHT:\n papy = self.text.find(' ', self.cursor) + 1\n if not papy:\n papy = len(self)\n self.cursor = papy\n else:\n papy = self.text.rfind(' ', 0, self.cursor)\n if papy == -1:\n papy = 0\n self.cursor = papy", "response": "Move the cursor of one word to the right or the left."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndeletes one letter the right or the left of the cursor.", "response": "def delete_one_letter(self, letter=RIGHT):\n \"\"\"Delete one letter the right or the the left of the cursor.\"\"\"\n\n assert letter in (self.RIGHT, self.LEFT)\n\n if letter == self.LEFT:\n papy = self.cursor\n self.text = self.text[:self.cursor - 1] + self.text[self.cursor:]\n self.cursor = papy - 1\n\n else:\n self.text = self.text[:self.cursor] + self.text[self.cursor + 1:]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef delete_one_word(self, word=RIGHT):\n\n assert word in (self.RIGHT, self.LEFT)\n\n if word == self.RIGHT:\n papy = self.text.find(' ', self.cursor) + 1\n if not papy:\n papy = len(self.text)\n self.text = self.text[:self.cursor] + self.text[papy:]\n\n else:\n papy = self.text.rfind(' ', 0, self.cursor)\n if papy == -1:\n papy = 0\n self.text = self.text[:papy] + self.text[self.cursor:]\n self.cursor = papy", "response": "Delete one word the right or the left of the cursor."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a letter at the cursor pos.", "response": "def add_letter(self, letter):\n \"\"\"Add a letter at the cursor pos.\"\"\"\n assert isinstance(letter, str)\n assert len(letter) == 1\n\n self.text = self.text[:self.cursor] + letter + self.text[self.cursor:]\n self.cursor += 1"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update(self, event_or_list):\n\n event_or_list = super().update(event_or_list)\n\n for e in event_or_list:\n if e.type == KEYDOWN:\n if e.key == K_RIGHT:\n if e.mod * KMOD_CTRL:\n self.move_cursor_one_word(self.RIGHT)\n else:\n self.move_cursor_one_letter(self.RIGHT)\n\n elif e.key == K_LEFT:\n if e.mod * KMOD_CTRL:\n self.move_cursor_one_word(self.LEFT)\n else:\n self.move_cursor_one_letter(self.LEFT)\n\n elif e.key == K_BACKSPACE:\n if self.cursor == 0:\n continue\n\n if e.mod & KMOD_CTRL:\n self.delete_one_word(self.LEFT)\n else:\n self.delete_one_letter(self.LEFT)\n\n elif e.key == K_DELETE:\n if e.mod & KMOD_CTRL:\n self.delete_one_word(self.RIGHT)\n else:\n self.delete_one_letter(self.RIGHT)\n\n elif e.unicode != '' and e.unicode.isprintable():\n self.add_letter(e.unicode)", "response": "Update the text and position of cursor according to the event passed."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _render(self):\n\n self._last_text = self.text\n\n self._surface = self.font.render(self.text, True, self.color, self.bg_color)\n size = self.width, self._surface.get_height()\n self.size = size", "response": "Render the text.\n\n Avoid using this fonction too many times as it is slow as it is slow to render text and blit it."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef shawn_text(self):\n\n if len(self._shawn_text) == len(self):\n return self._shawn_text\n\n if self.style == self.DOTS:\n return chr(0x2022) * len(self)\n\n ranges = [\n (902, 1366),\n (192, 683),\n (33, 122)\n ]\n\n s = ''\n while len(s) < len(self.text):\n apolo = randint(33, 1366)\n for a, b in ranges:\n if a <= apolo <= b:\n s += chr(apolo)\n break\n\n self._shawn_text = s\n return s", "response": "The text displayed instead of the real one."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cursor_pos(self):\n if len(self) == 0:\n return self.left + self.default_text.get_width()\n\n papy = self._surface.get_width()\n if papy > self.w:\n shift = papy - self.width\n else:\n shift = 0\n\n return self.left + self.font.size(self.shawn_text[:self.cursor])[0] - shift", "response": "The cursor position in pixels."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _render(self):\n\n self._last_text = self.shawn_text\n\n self._surface = self.font.render(self.shawn_text, True, self.color, self.bg_color)\n size = self.w, self._surface.get_height()\n self.size = size", "response": "Render the text.\n\n Avoid using this fonction too many times as it is slow as it is slow to render text and blit it."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nrenders basicly the text.", "response": "def render(self, display):\n \"\"\"Render basicly the text.\"\"\"\n\n # to handle changing objects / callable\n if self.shawn_text != self._last_text:\n self._render()\n\n if self.text:\n papy = self._surface.get_width()\n if papy <= self.width:\n display.blit(self._surface, (self.topleft, self.size))\n else:\n display.blit(self._surface, (self.topleft, self.size), ((papy - self.w, 0), self.size))\n else:\n display.blit(self.default_text, (self.topleft, self.size))\n\n if self._focus:\n groom = self.cursor_pos()\n line(display, (groom, self.top), (groom, self.bottom), CONCRETE)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a pygame image from a latex template.", "response": "def latex_to_img(tex):\n \"\"\"Return a pygame image from a latex template.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdirname:\n with open(tmpdirname + r'\\tex.tex', 'w') as f:\n f.write(tex)\n\n os.system(r\"latex {0}\\tex.tex -halt-on-error -interaction=batchmode -disable-installer -aux-directory={0} \"\n r\"-output-directory={0}\".format(tmpdirname))\n os.system(r\"dvipng -T tight -z 9 --truecolor -o {0}\\tex.png {0}\\tex.dvi\".format(tmpdirname))\n # os.system(r'latex2png ' + tmpdirname)\n\n image = pygame.image.load(tmpdirname + r'\\tex.png')\n\n return image"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef mix(color1, color2, pos=0.5):\n opp_pos = 1 - pos\n\n red = color1[0] * pos + color2[0] * opp_pos\n green = color1[1] * pos + color2[1] * opp_pos\n blue = color1[2] * pos + color2[2] * opp_pos\n return int(red), int(green), int(blue)", "response": "Return the mix of two colors at a state of pos"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert the name of a color into its RGB value", "response": "def name2rgb(name):\n \"\"\"Convert the name of a color into its RGB value\"\"\"\n try:\n import colour\n except ImportError:\n raise ImportError('You need colour to be installed: pip install colour')\n\n c = colour.Color(name)\n color = int(c.red * 255), int(c.green * 255), int(c.blue * 255)\n return color"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse the command man page.", "response": "def parse_page(page):\n \"\"\"Parse the command man page.\"\"\"\n colors = get_config()['colors']\n with io.open(page, encoding='utf-8') as f:\n lines = f.readlines()\n output_lines = []\n for line in lines[1:]:\n if is_headline(line):\n continue\n elif is_description(line):\n output_lines.append(click.style(line.replace('>', ' '),\n fg=colors['description']))\n elif is_old_usage(line):\n output_lines.append(click.style(line, fg=colors['usage']))\n elif is_code_example(line):\n line = ' ' + line if line.startswith('`') else line[2:]\n output_lines.append(click.style(line.replace('`', ''),\n fg=colors['command']))\n elif is_line_break(line):\n output_lines.append(click.style(line))\n else:\n output_lines.append(click.style('- ' + line, fg=colors['usage']))\n return output_lines"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef configure_logging(level=logging.DEBUG):\n if level == logging.DEBUG:\n # For debugging purposes, log from everyone!\n logging.basicConfig(\n level=logging.DEBUG,\n format='%(asctime)s - %(levelname)s - %(message)s'\n )\n return logging\n\n logger = logging.getLogger(__name__)\n logger.setLevel(level)\n formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n ch = logging.StreamHandler()\n ch.setLevel(level)\n ch.setFormatter(formatter)\n logger.addHandler(ch)\n return logger", "response": "Configure the module logging engine."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nwraps around os. path. join.", "response": "def path_join(*args):\n \"\"\"\n Wrapper around `os.path.join`.\n Makes sure to join paths of the same type (bytes).\n \"\"\"\n args = (paramiko.py3compat.u(arg) for arg in args)\n return os.path.join(*args)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_username_password_hostname(remote_url):\n assert remote_url\n assert ':' in remote_url\n\n if '@' in remote_url:\n username, hostname = remote_url.rsplit('@', 1)\n else:\n username, hostname = None, remote_url\n\n hostname, remote_path = hostname.split(':', 1)\n\n password = None\n if username and ':' in username:\n username, password = username.split(':', 1)\n\n assert hostname\n assert remote_path\n return username, password, hostname, remote_path", "response": "Parse a command line string and return username password hostname and remote path."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_ssh_agent_keys(logger):\n agent, agent_keys = None, None\n\n try:\n agent = paramiko.agent.Agent()\n _agent_keys = agent.get_keys()\n\n if not _agent_keys:\n agent.close()\n logger.error(\n \"SSH agent didn't provide any valid key. Trying to continue...\"\n )\n else:\n agent_keys = tuple(k for k in _agent_keys)\n except paramiko.SSHException:\n if agent:\n agent.close()\n agent = None\n logger.error(\"SSH agent speaks a non-compatible protocol. Ignoring it.\")\n finally:\n return agent, agent_keys", "response": "Ask the SSH agent for a list of keys and return it."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_parser():\n parser = argparse.ArgumentParser(\n description='Sync a local and a remote folder through SFTP.'\n )\n\n parser.add_argument(\n \"path\",\n type=str,\n metavar=\"local-path\",\n help=\"the path of the local folder\",\n )\n\n parser.add_argument(\n \"remote\",\n type=str,\n metavar=\"user[:password]@hostname:remote-path\",\n help=\"the ssh-url ([user[:password]@]hostname:remote-path) of the remote folder. \"\n \"The hostname can be specified as a ssh_config's hostname too. \"\n \"Every missing information will be gathered from there\",\n )\n\n parser.add_argument(\n \"-k\",\n \"--key\",\n metavar=\"identity-path\",\n action=\"append\",\n help=\"private key identity path (defaults to ~/.ssh/id_rsa)\"\n )\n\n parser.add_argument(\n \"-l\",\n \"--logging\",\n choices=['CRITICAL',\n 'ERROR',\n 'WARNING',\n 'INFO',\n 'DEBUG',\n 'NOTSET'],\n default='ERROR',\n help=\"set logging level\"\n )\n\n parser.add_argument(\n \"-p\",\n \"--port\",\n default=22,\n type=int,\n help=\"SSH remote port (defaults to 22)\"\n )\n\n parser.add_argument(\n \"-f\",\n \"--fix-symlinks\",\n action=\"store_true\",\n help=\"fix symbolic links on remote side\"\n )\n\n parser.add_argument(\n \"-a\",\n \"--ssh-agent\",\n action=\"store_true\",\n help=\"enable ssh-agent support\"\n )\n\n parser.add_argument(\n \"-c\",\n \"--ssh-config\",\n metavar=\"ssh_config path\",\n default=\"~/.ssh/config\",\n type=str,\n help=\"path to the ssh-configuration file (default to ~/.ssh/config)\"\n )\n\n parser.add_argument(\n \"-n\",\n \"--known-hosts\",\n metavar=\"known_hosts path\",\n default=\"~/.ssh/known_hosts\",\n type=str,\n help=\"path to the openSSH known_hosts file\"\n )\n\n parser.add_argument(\n \"-d\",\n \"--disable-known-hosts\",\n action=\"store_true\",\n help=\"disable known_hosts fingerprint checking (security warning!)\"\n )\n\n parser.add_argument(\n \"-e\",\n \"--exclude-from\",\n metavar=\"exclude-from-file-path\",\n type=str,\n help=\"exclude files matching pattern in exclude-from-file-path\"\n )\n\n parser.add_argument(\n \"-t\",\n \"--do-not-delete\",\n action=\"store_true\",\n help=\"do not delete remote files missing from local folder\"\n )\n\n parser.add_argument(\n \"-o\",\n \"--allow-unknown\",\n action=\"store_true\",\n help=\"allow connection to unknown hosts\"\n )\n\n parser.add_argument(\n \"-r\",\n \"--create-remote-directory\",\n action=\"store_true\",\n help=\"Create remote base directory if missing on remote\"\n )\n\n return parser", "response": "Create the CLI argument parser."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef main(args=None):\n parser = create_parser()\n args = vars(parser.parse_args(args))\n\n log_mapping = {\n 'CRITICAL': logging.CRITICAL,\n 'ERROR': logging.ERROR,\n 'WARNING': logging.WARNING,\n 'INFO': logging.INFO,\n 'DEBUG': logging.DEBUG,\n 'NOTSET': logging.NOTSET,\n }\n log_level = log_mapping[args['logging']]\n del(args['logging'])\n\n global logger\n logger = configure_logging(log_level)\n\n args_mapping = {\n \"path\": \"local_path\",\n \"remote\": \"remote_url\",\n \"ssh_config\": \"ssh_config_path\",\n \"exclude_from\": \"exclude_file\",\n \"known_hosts\": \"known_hosts_path\",\n \"do_not_delete\": \"delete\",\n \"key\": \"identity_files\",\n }\n\n kwargs = { # convert the argument names to class constructor parameters\n args_mapping[k]: v\n for k, v in args.items()\n if v and k in args_mapping\n }\n\n kwargs.update({\n k: v\n for k, v in args.items()\n if v and k not in args_mapping\n })\n\n # Special case: disable known_hosts check\n if args['disable_known_hosts']:\n kwargs['known_hosts_path'] = None\n del(kwargs['disable_known_hosts'])\n\n # Toggle `do_not_delete` flag\n if \"delete\" in kwargs:\n kwargs[\"delete\"] = not kwargs[\"delete\"]\n\n # Manually set the default identity file.\n kwargs[\"identity_files\"] = kwargs.get(\"identity_files\", None) or [\"~/.ssh/id_rsa\"]\n\n sync = SFTPClone(\n **kwargs\n )\n sync.run()", "response": "The main function of the main function."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _must_be_deleted(local_path, r_st):\n # if the file doesn't exists\n if not os.path.lexists(local_path):\n return True\n\n # or if the file type is different\n l_st = os.lstat(local_path)\n if S_IFMT(r_st.st_mode) != S_IFMT(l_st.st_mode):\n return True\n\n return False", "response": "Return True if the remote correspondent of local_path has to be deleted."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _match_modes(self, remote_path, l_st):\n self.sftp.chmod(remote_path, S_IMODE(l_st.st_mode))\n self.sftp.utime(remote_path, (l_st.st_atime, l_st.st_mtime))\n\n if self.chown:\n self.sftp.chown(remote_path, l_st.st_uid, l_st.st_gid)", "response": "Match mod utime and uid and gid with locals one."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef file_upload(self, local_path, remote_path, l_st):\n self.sftp.put(local_path, remote_path)\n self._match_modes(remote_path, l_st)", "response": "Upload local_path to remote_path and set permission and mtime."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nremove the remote file node.", "response": "def remote_delete(self, remote_path, r_st):\n \"\"\"Remove the remote directory node.\"\"\"\n # If it's a directory, then delete content and directory\n if S_ISDIR(r_st.st_mode):\n for item in self.sftp.listdir_attr(remote_path):\n full_path = path_join(remote_path, item.filename)\n self.remote_delete(full_path, item)\n self.sftp.rmdir(remote_path)\n\n # Or simply delete files\n else:\n try:\n self.sftp.remove(remote_path)\n except FileNotFoundError as e:\n self.logger.error(\n \"error while removing {}. trace: {}\".format(remote_path, e)\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntraverses the entire remote_path tree and check if any files in the local folder are deleted.", "response": "def check_for_deletion(self, relative_path=None):\n \"\"\"Traverse the entire remote_path tree.\n\n Find files/directories that need to be deleted,\n not being present in the local folder.\n \"\"\"\n if not relative_path:\n relative_path = str() # root of shared directory tree\n\n remote_path = path_join(self.remote_path, relative_path)\n local_path = path_join(self.local_path, relative_path)\n\n for remote_st in self.sftp.listdir_attr(remote_path):\n r_lstat = self.sftp.lstat(path_join(remote_path, remote_st.filename))\n\n inner_remote_path = path_join(remote_path, remote_st.filename)\n inner_local_path = path_join(local_path, remote_st.filename)\n\n # check if remote_st is a symlink\n # otherwise could delete file outside shared directory\n if S_ISLNK(r_lstat.st_mode):\n if self._must_be_deleted(inner_local_path, r_lstat):\n self.remote_delete(inner_remote_path, r_lstat)\n continue\n\n if self._must_be_deleted(inner_local_path, remote_st):\n self.remote_delete(inner_remote_path, remote_st)\n elif S_ISDIR(remote_st.st_mode):\n self.check_for_deletion(\n path_join(relative_path, remote_st.filename)\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a new symlink pointing to link_destination in remote_path position.", "response": "def create_update_symlink(self, link_destination, remote_path):\n \"\"\"Create a new link pointing to link_destination in remote_path position.\"\"\"\n try: # if there's anything, delete it\n self.sftp.remove(remote_path)\n except IOError: # that's fine, nothing exists there!\n pass\n finally: # and recreate the link\n try:\n self.sftp.symlink(link_destination, remote_path)\n except OSError as e:\n # Sometimes, if links are \"too\" different, symlink fails.\n # Sadly, nothing we can do about it.\n self.logger.error(\"error while symlinking {} to {}: {}\".format(\n remote_path, link_destination, e))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks if the given file tree node has to be uploaded or created on the remote folder.", "response": "def node_check_for_upload_create(self, relative_path, f):\n \"\"\"Check if the given directory tree node has to be uploaded/created on the remote folder.\"\"\"\n if not relative_path:\n # we're at the root of the shared directory tree\n relative_path = str()\n\n # the (absolute) local address of f.\n local_path = path_join(self.local_path, relative_path, f)\n try:\n l_st = os.lstat(local_path)\n except OSError as e:\n \"\"\"A little background here.\n Sometimes, in big clusters configurations (mail, etc.),\n files could disappear or be moved, suddenly.\n There's nothing to do about it,\n system should be stopped before doing backups.\n Anyway, we log it, and skip it.\n \"\"\"\n self.logger.error(\"error while checking {}: {}\".format(relative_path, e))\n return\n\n if local_path in self.exclude_list:\n self.logger.info(\"Skipping excluded file %s.\", local_path)\n return\n\n # the (absolute) remote address of f.\n remote_path = path_join(self.remote_path, relative_path, f)\n\n # First case: f is a directory\n if S_ISDIR(l_st.st_mode):\n # we check if the folder exists on the remote side\n # it has to be a folder, otherwise it would have already been\n # deleted\n try:\n self.sftp.stat(remote_path)\n except IOError: # it doesn't exist yet on remote side\n self.sftp.mkdir(remote_path)\n\n self._match_modes(remote_path, l_st)\n\n # now, we should traverse f too (recursion magic!)\n self.check_for_upload_create(path_join(relative_path, f))\n\n # Second case: f is a symbolic link\n elif S_ISLNK(l_st.st_mode):\n # read the local link\n local_link = os.readlink(local_path)\n absolute_local_link = os.path.realpath(local_link)\n\n # is it absolute?\n is_absolute = local_link.startswith(\"/\")\n # and does it point inside the shared directory?\n # add trailing slash (security)\n trailing_local_path = path_join(self.local_path, '')\n relpath = os.path.commonprefix(\n [absolute_local_link,\n trailing_local_path]\n ) == trailing_local_path\n\n if relpath:\n relative_link = absolute_local_link[len(trailing_local_path):]\n else:\n relative_link = None\n\n \"\"\"\n # Refactor them all, be efficient!\n\n # Case A: absolute link pointing outside shared directory\n # (we can only update the remote part)\n if is_absolute and not relpath:\n self.create_update_symlink(local_link, remote_path)\n\n # Case B: absolute link pointing inside shared directory\n # (we can leave it as it is or fix the prefix to match the one of the remote server)\n elif is_absolute and relpath:\n if self.fix_symlinks:\n self.create_update_symlink(\n join(\n self.remote_path,\n relative_link,\n ),\n remote_path\n )\n else:\n self.create_update_symlink(local_link, remote_path)\n\n # Case C: relative link pointing outside shared directory\n # (all we can do is try to make the link anyway)\n elif not is_absolute and not relpath:\n self.create_update_symlink(local_link, remote_path)\n\n # Case D: relative link pointing inside shared directory\n # (we preserve the relativity and link it!)\n elif not is_absolute and relpath:\n self.create_update_symlink(local_link, remote_path)\n \"\"\"\n\n if is_absolute and relpath:\n if self.fix_symlinks:\n self.create_update_symlink(\n path_join(\n self.remote_path,\n relative_link,\n ),\n remote_path\n )\n else:\n self.create_update_symlink(local_link, remote_path)\n\n # Third case: regular file\n elif S_ISREG(l_st.st_mode):\n try:\n r_st = self.sftp.lstat(remote_path)\n if self._file_need_upload(l_st, r_st):\n self.file_upload(local_path, remote_path, l_st)\n except IOError as e:\n if e.errno == errno.ENOENT:\n self.file_upload(local_path, remote_path, l_st)\n\n # Anything else.\n else:\n self.logger.warning(\"Skipping unsupported file %s.\", local_path)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef check_for_upload_create(self, relative_path=None):\n for f in os.listdir(\n path_join(\n self.local_path, relative_path) if relative_path else self.local_path\n ):\n self.node_check_for_upload_create(relative_path, f)", "response": "Traverse the relative_path tree and check for files that need to be uploaded or created."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run(self):\n\n # Check if remote path is present\n try:\n self.sftp.stat(self.remote_path)\n except FileNotFoundError as e:\n if self.create_remote_directory:\n self.sftp.mkdir(self.remote_path)\n self.logger.info(\n \"Created missing remote dir: '\" + self.remote_path + \"'\")\n else:\n self.logger.error(\n \"Remote folder does not exists. \"\n \"Add '-r' to create it if missing.\")\n sys.exit(1)\n\n try:\n if self.delete:\n # First check for items to be removed\n self.check_for_deletion()\n\n # Now scan local for items to upload/create\n self.check_for_upload_create()\n except FileNotFoundError:\n # If this happens, probably the remote folder doesn't exist.\n self.logger.error(\n \"Error while opening remote folder. Are you sure it does exist?\")\n sys.exit(1)", "response": "Runs the sync.\n command."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlist files in a directory", "response": "def list_files(start_path):\n \"\"\"tree unix command replacement.\"\"\"\n s = u'\\n'\n for root, dirs, files in os.walk(start_path):\n level = root.replace(start_path, '').count(os.sep)\n indent = ' ' * 4 * level\n s += u'{}{}/\\n'.format(indent, os.path.basename(root))\n sub_indent = ' ' * 4 * (level + 1)\n for f in files:\n s += u'{}{}\\n'.format(sub_indent, f)\n return s"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a nested dictionary that represents the folder structure of start_path.", "response": "def file_tree(start_path):\n \"\"\"\n Create a nested dictionary that represents the folder structure of `start_path`.\n\n Liberally adapted from\n http://code.activestate.com/recipes/577879-create-a-nested-dictionary-from-oswalk/\n \"\"\"\n nested_dirs = {}\n root_dir = start_path.rstrip(os.sep)\n start = root_dir.rfind(os.sep) + 1\n for path, dirs, files in os.walk(root_dir):\n folders = path[start:].split(os.sep)\n subdir = dict.fromkeys(files)\n parent = reduce(dict.get, folders[:-1], nested_dirs)\n parent[folders[-1]] = subdir\n return nested_dirs"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef capture_sys_output():\n capture_out, capture_err = StringIO(), StringIO()\n current_out, current_err = sys.stdout, sys.stderr\n try:\n sys.stdout, sys.stderr = capture_out, capture_err\n yield capture_out, capture_err\n finally:\n sys.stdout, sys.stderr = current_out, current_err", "response": "Capture standard output and error."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef override_env_variables():\n env_vars = (\"LOGNAME\", \"USER\", \"LNAME\", \"USERNAME\")\n old = [os.environ[v] if v in os.environ else None for v in env_vars]\n\n for v in env_vars:\n os.environ[v] = \"test\"\n yield\n\n for i, v in enumerate(env_vars):\n if old[i]:\n os.environ[v] = old[i]", "response": "Override user environmental variables with custom one."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\noverride the SSH_AUTH_SOCK env variable to mock the absence of an SSH agent.", "response": "def override_ssh_auth_env():\n \"\"\"Override the `$SSH_AUTH_SOCK `env variable to mock the absence of an SSH agent.\"\"\"\n ssh_auth_sock = \"SSH_AUTH_SOCK\"\n old_ssh_auth_sock = os.environ.get(ssh_auth_sock)\n\n del os.environ[ssh_auth_sock]\n\n yield\n\n if old_ssh_auth_sock:\n os.environ[ssh_auth_sock] = old_ssh_auth_sock"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the configurations from. tldrrc and return it as a dict.", "response": "def get_config():\n \"\"\"Get the configurations from .tldrrc and return it as a dict.\"\"\"\n config_path = path.join(\n (os.environ.get('TLDR_CONFIG_DIR') or path.expanduser('~')),\n '.tldrrc')\n if not path.exists(config_path):\n sys.exit(\"Can't find config file at: {0}. You may use `tldr init` \"\n \"to init the config file.\".format(config_path))\n\n with io.open(config_path, encoding='utf-8') as f:\n try:\n config = yaml.safe_load(f)\n except yaml.scanner.ScannerError:\n sys.exit(\"The config file is not a valid YAML file.\")\n\n supported_colors = ['black', 'red', 'green', 'yellow', 'blue',\n 'magenta', 'cyan', 'white']\n if not set(config['colors'].values()).issubset(set(supported_colors)):\n sys.exit(\"Unsupported colors in config file: {0}.\".format(\n ', '.join(set(config['colors'].values()) - set(supported_colors))))\n if not path.exists(config['repo_directory']):\n sys.exit(\"Can't find the tldr repo, check the `repo_directory` \"\n \"setting in config file.\")\n\n return config"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing the man page and return the parsed lines.", "response": "def parse_man_page(command, platform):\n \"\"\"Parse the man page and return the parsed lines.\"\"\"\n page_path = find_page_location(command, platform)\n output_lines = parse_page(page_path)\n return output_lines"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_page_location(command, specified_platform):\n repo_directory = get_config()['repo_directory']\n default_platform = get_config()['platform']\n command_platform = (\n specified_platform if specified_platform else default_platform)\n\n with io.open(path.join(repo_directory, 'pages/index.json'),\n encoding='utf-8') as f:\n index = json.load(f)\n command_list = [item['name'] for item in index['commands']]\n if command not in command_list:\n sys.exit(\n (\"Sorry, we don't support command: {0} right now.\\n\"\n \"You can file an issue or send a PR on github:\\n\"\n \" https://github.com/tldr-pages/tldr\").format(command))\n\n supported_platforms = index['commands'][\n command_list.index(command)]['platform']\n if command_platform in supported_platforms:\n platform = command_platform\n elif 'common' in supported_platforms:\n platform = 'common'\n else:\n platform = ''\n if not platform:\n sys.exit(\n (\"Sorry, command {0} is not supported on your platform.\\n\"\n \"You can file an issue or send a PR on github:\\n\"\n \" https://github.com/tldr-pages/tldr\").format(command))\n\n page_path = path.join(path.join(repo_directory, 'pages'),\n path.join(platform, command + '.md'))\n return page_path", "response": "Find the command man page in the pages directory."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfinding the command usage.", "response": "def find(command, on):\n \"\"\"Find the command usage.\"\"\"\n output_lines = parse_man_page(command, on)\n click.echo(''.join(output_lines))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nupdate to the latest pages.", "response": "def update():\n \"\"\"Update to the latest pages.\"\"\"\n repo_directory = get_config()['repo_directory']\n os.chdir(repo_directory)\n click.echo(\"Check for updates...\")\n\n local = subprocess.check_output('git rev-parse master'.split()).strip()\n remote = subprocess.check_output(\n 'git ls-remote https://github.com/tldr-pages/tldr/ HEAD'.split()\n ).split()[0]\n if local != remote:\n click.echo(\"Updating...\")\n subprocess.check_call('git checkout master'.split())\n subprocess.check_call('git pull --rebase'.split())\n build_index()\n click.echo(\"Update to the latest and rebuild the index.\")\n else:\n click.echo(\"No need for updates.\")"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize the config file.", "response": "def init():\n \"\"\"Init config file.\"\"\"\n default_config_path = path.join(\n (os.environ.get('TLDR_CONFIG_DIR') or path.expanduser('~')),\n '.tldrrc')\n if path.exists(default_config_path):\n click.echo(\"There is already a config file exists, \"\n \"skip initializing it.\")\n else:\n repo_path = click.prompt(\"Input the tldr repo path(absolute path)\")\n if not path.exists(repo_path):\n sys.exit(\"Repo path not exist, clone it first.\")\n\n platform = click.prompt(\"Input your platform(linux, osx or sunos)\")\n if platform not in ['linux', 'osx', 'sunos']:\n sys.exit(\"Platform should be in linux, osx or sunos.\")\n\n colors = {\n \"description\": \"blue\",\n \"usage\": \"green\",\n \"command\": \"cyan\"\n }\n\n config = {\n \"repo_directory\": repo_path,\n \"colors\": colors,\n \"platform\": platform\n }\n with open(default_config_path, 'w') as f:\n f.write(yaml.safe_dump(config, default_flow_style=False))\n\n click.echo(\"Initializing the config file at {0}\".format(\n default_config_path))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nlocate the command s man page.", "response": "def locate(command, on):\n \"\"\"Locate the command's man page.\"\"\"\n location = find_page_location(command, on)\n click.echo(location)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef relate(cls, propname, *args, **kwargs):\n class_mapper(cls)._configure_property(propname, relationship(*args, **kwargs))", "response": "Produce a relationship between this mapped table and another. \n This makes usage of SQLAlchemy s SQLAlchemy s relationship function."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef execute(self, stmt, **params):\n return self.session.execute(sql.text(stmt, bind=self.bind), **params)", "response": "Execute a SQL statement."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconfigure a mapping to the given attribute name.", "response": "def map_to(self, attrname, tablename=None, selectable=None, \n schema=None, base=None, mapper_args=util.immutabledict()):\n \"\"\"Configure a mapping to the given attrname.\n\n This is the \"master\" method that can be used to create any \n configuration.\n\n :param attrname: String attribute name which will be\n established as an attribute on this :class:.`.SQLSoup`\n instance.\n :param base: a Python class which will be used as the\n base for the mapped class. If ``None``, the \"base\"\n argument specified by this :class:`.SQLSoup`\n instance's constructor will be used, which defaults to\n ``object``.\n :param mapper_args: Dictionary of arguments which will\n be passed directly to :func:`.orm.mapper`.\n :param tablename: String name of a :class:`.Table` to be\n reflected. If a :class:`.Table` is already available,\n use the ``selectable`` argument. This argument is\n mutually exclusive versus the ``selectable`` argument.\n :param selectable: a :class:`.Table`, :class:`.Join`, or\n :class:`.Select` object which will be mapped. This\n argument is mutually exclusive versus the ``tablename``\n argument.\n :param schema: String schema name to use if the\n ``tablename`` argument is present.\n\n\n \"\"\"\n if attrname in self._cache:\n raise SQLSoupError(\n \"Attribute '%s' is already mapped to '%s'\" % (\n attrname,\n class_mapper(self._cache[attrname]).mapped_table\n ))\n\n if tablename is not None:\n if not isinstance(tablename, basestring):\n raise ArgumentError(\"'tablename' argument must be a string.\"\n )\n if selectable is not None:\n raise ArgumentError(\"'tablename' and 'selectable' \"\n \"arguments are mutually exclusive\")\n\n selectable = Table(tablename, \n self._metadata, \n autoload=True, \n autoload_with=self.bind, \n schema=schema or self.schema)\n elif schema:\n raise ArgumentError(\"'tablename' argument is required when \"\n \"using 'schema'.\")\n elif selectable is not None:\n if not isinstance(selectable, expression.FromClause):\n raise ArgumentError(\"'selectable' argument must be a \"\n \"table, select, join, or other \"\n \"selectable construct.\")\n else:\n raise ArgumentError(\"'tablename' or 'selectable' argument is \"\n \"required.\")\n\n if not selectable.primary_key.columns and not \\\n 'primary_key' in mapper_args:\n if tablename:\n raise SQLSoupError(\n \"table '%s' does not have a primary \"\n \"key defined\" % tablename)\n else:\n raise SQLSoupError(\n \"selectable '%s' does not have a primary \"\n \"key defined\" % selectable)\n\n mapped_cls = _class_for_table(\n self.session,\n self.engine,\n selectable,\n base or self.base,\n mapper_args\n )\n self._cache[attrname] = mapped_cls\n return mapped_cls"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmap a selectable directly.", "response": "def map(self, selectable, base=None, **mapper_args):\n \"\"\"Map a selectable directly.\n\n The class and its mapping are not cached and will\n be discarded once dereferenced (as of 0.6.6).\n\n :param selectable: an :func:`.expression.select` construct.\n :param base: a Python class which will be used as the\n base for the mapped class. If ``None``, the \"base\"\n argument specified by this :class:`.SQLSoup`\n instance's constructor will be used, which defaults to\n ``object``.\n :param mapper_args: Dictionary of arguments which will\n be passed directly to :func:`.orm.mapper`.\n\n \"\"\"\n\n return _class_for_table(\n self.session,\n self.engine,\n selectable,\n base or self.base,\n mapper_args\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nmaps a selectable directly wrapping the selectable in a subquery with labels.", "response": "def with_labels(self, selectable, base=None, **mapper_args):\n \"\"\"Map a selectable directly, wrapping the \n selectable in a subquery with labels.\n\n The class and its mapping are not cached and will\n be discarded once dereferenced (as of 0.6.6).\n\n :param selectable: an :func:`.expression.select` construct.\n :param base: a Python class which will be used as the\n base for the mapped class. If ``None``, the \"base\"\n argument specified by this :class:`.SQLSoup`\n instance's constructor will be used, which defaults to\n ``object``.\n :param mapper_args: Dictionary of arguments which will\n be passed directly to :func:`.orm.mapper`.\n\n \"\"\"\n\n # TODO give meaningful aliases\n return self.map(\n expression._clause_element_as_expr(selectable).\n select(use_labels=True).\n alias('foo'), base=base, **mapper_args)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef join(self, left, right, onclause=None, isouter=False, \n base=None, **mapper_args):\n \"\"\"Create an :func:`.expression.join` and map to it.\n\n The class and its mapping are not cached and will\n be discarded once dereferenced (as of 0.6.6).\n\n :param left: a mapped class or table object.\n :param right: a mapped class or table object.\n :param onclause: optional \"ON\" clause construct..\n :param isouter: if True, the join will be an OUTER join.\n :param base: a Python class which will be used as the\n base for the mapped class. If ``None``, the \"base\"\n argument specified by this :class:`.SQLSoup`\n instance's constructor will be used, which defaults to\n ``object``.\n :param mapper_args: Dictionary of arguments which will\n be passed directly to :func:`.orm.mapper`.\n\n \"\"\"\n\n j = join(left, right, onclause=onclause, isouter=isouter)\n return self.map(j, base=base, **mapper_args)", "response": "Create an SQLAlchemy join expression and map it to the current table."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef entity(self, attr, schema=None):\n try:\n return self._cache[attr]\n except KeyError, ke:\n return self.map_to(attr, tablename=attr, schema=schema)", "response": "Return the named entity from this : class :. SQLSoup schema if not present."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndistances between two features.", "response": "def distance(f1, f2):\n \"\"\"\\\n Distance between 2 features. The integer result is always positive or zero.\n If the features overlap or touch, it is zero.\n >>> from intersecter import Feature, distance\n >>> distance(Feature(1, 2), Feature(12, 13))\n 10\n >>> distance(Feature(1, 2), Feature(2, 3))\n 0\n >>> distance(Feature(1, 100), Feature(20, 30))\n 0\n\n \"\"\"\n if f1.end < f2.start: return f2.start - f1.end\n if f2.end < f1.start: return f1.start - f2.end\n return 0"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find(self, start, end, chrom=None):\n intervals = self.intervals[chrom]\n ilen = len(intervals)\n # NOTE: we only search for starts, since any feature that starts within max_len of\n # the query could overlap, we must subtract max_len from the start to get the needed\n # search space. everything else proceeds like a binary search.\n # (but add distance calc for candidates).\n if not chrom in self.max_len: return []\n ileft = binsearch_left_start(intervals, start - self.max_len[chrom], 0, ilen)\n iright = binsearch_right_end(intervals, end, ileft, ilen)\n query = Feature(start, end)\n # we have to check the distance to make sure we didnt pick up anything \n # that started within max_len, but wasnt as long as max_len\n return [f for f in intervals[ileft:iright] if distance(f, query) == 0]", "response": "Return a list of all stored intervals intersecting between start and end inclusive."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the nearest n features strictly to the left of a Feature f", "response": "def left(self, f, n=1):\n \"\"\"return the nearest n features strictly to the left of a Feature f.\n Overlapping features are not considered as to the left.\n\n f: a Feature object\n n: the number of features to return\n \"\"\"\n intervals = self.intervals[f.chrom]\n if intervals == []: return []\n\n iright = binsearch_left_start(intervals, f.start, 0 , len(intervals)) + 1\n ileft = binsearch_left_start(intervals, f.start - self.max_len[f.chrom] - 1, 0, 0)\n\n results = sorted((distance(other, f), other) for other in intervals[ileft:iright] if other.end < f.start and distance(f, other) != 0)\n if len(results) == n:\n return [r[1] for r in results]\n\n # have to do some extra work here since intervals are sorted\n # by starts, and we dont know which end may be around...\n # in this case, we got some extras, just return as many as\n # needed once we see a gap in distances.\n for i in range(n, len(results)):\n if results[i - 1][0] != results[i][0]:\n return [r[1] for r in results[:i]]\n\n if ileft == 0:\n return [r[1] for r in results]\n\n # here, didn't get enough, so move left and try again. \n 1/0"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef right(self, f, n=1):\n intervals = self.intervals[f.chrom]\n ilen = len(intervals)\n iright = binsearch_right_end(intervals, f.end, 0, ilen)\n results = []\n\n while iright < ilen:\n i = len(results)\n if i > n:\n if distance(f, results[i - 1]) != distance(f, results[i - 2]):\n return results[:i - 1]\n other = intervals[iright]\n iright += 1\n if distance(other, f) == 0: continue\n results.append(other)\n return results", "response": "return the nearest n features strictly to the right of a Feature f"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfind n upstream features where upstream is determined by the strand of the query Feature f", "response": "def upstream(self, f, n=1):\n \"\"\"find n upstream features where upstream is determined by\n the strand of the query Feature f\n Overlapping features are not considered.\n\n f: a Feature object\n n: the number of features to return\n \"\"\"\n if f.strand == -1:\n return self.right(f, n)\n return self.left(f, n)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinds n downstream features where downstream is determined by the strand of the query Feature f", "response": "def downstream(self, f, n=1):\n \"\"\"find n downstream features where downstream is determined by\n the strand of the query Feature f\n Overlapping features are not considered.\n\n f: a Feature object\n n: the number of features to return\n \"\"\"\n if f.strand == -1:\n return self.left(f, n)\n return self.right(f, n)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef knearest(self, f_or_start, end=None, chrom=None, k=1):\n\n\n if end is not None:\n f = Feature(f_or_start, end, chrom=chrom)\n else:\n f = f_or_start\n\n DIST = 2000\n feats = filter_feats(self.find(f.start - DIST, f.end + DIST, chrom=f.chrom), f, k)\n if len(feats) >= k:\n return feats\n\n nfeats = k - len(feats)\n fleft = Feature(f.start - DIST, f.start, chrom=f.chrom)\n feats.extend(self.left(fleft, n=nfeats))\n\n fright = Feature(f.end, f.end + DIST, chrom=f.chrom)\n feats.extend(self.right(fright, n=nfeats))\n return filter_feats(feats, f, k)", "response": "return the n nearest neighbors to the given feature f_or_start end chrom = None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nfinding all elements between start and end", "response": "def find(self, start, end):\n \"\"\"find all elements between (or overlapping) start and end\"\"\"\n if self.intervals and not end < self.intervals[0].start:\n overlapping = [i for i in self.intervals if i.end >= start\n and i.start <= end]\n else:\n overlapping = []\n\n if self.left and start <= self.center:\n overlapping += self.left.find(start, end)\n\n if self.right and end >= self.center:\n overlapping += self.right.find(start, end)\n\n return overlapping"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sequence(db, chrom, start, end):\n url = \"http://genome.ucsc.edu/cgi-bin/das/%s\" % db\n url += \"/dna?segment=%s:%i,%i\"\n xml = U.urlopen(url % (chrom, start, end)).read()\n return _seq_from_xml(xml)", "response": "get the sequence for a region using the UCSC DAS server"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_table(genome, table, table_name, connection_string, metadata):\n table = Table(table_name, genome._metadata, autoload=True,\n autoload_with=genome.bind, extend_existing=True)\n\n #print \"\\t\".join([c.name for c in table.columns])\n # need to prefix the indexes with the table name to avoid collisions\n for i, idx in enumerate(table.indexes):\n idx.name = table_name + \".\" + idx.name + \"_ix\" + str(i)\n\n cols = []\n for i, col in enumerate(table.columns):\n # convert mysql-specific types to varchar\n #print col.name, col.type, isinstance(col.type, ENUM)\n if isinstance(col.type, (LONGBLOB, ENUM)):\n\n if 'sqlite' in connection_string:\n col.type = VARCHAR()\n elif 'postgres' in connection_string:\n if isinstance(col.type, ENUM):\n #print dir(col)\n col.type = PG_ENUM(*col.type.enums, name=col.name,\n create_type=True)\n else:\n col.type = VARCHAR()\n elif str(col.type) == \"VARCHAR\" \\\n and (\"mysql\" in connection_string \\\n or \"postgres\" in connection_string):\n if col.type.length is None:\n col.type.length = 48 if col.name != \"description\" else None\n if not \"mysql\" in connection_string:\n if str(col.type).lower().startswith(\"set(\"):\n col.type = VARCHAR(15)\n cols.append(col)\n\n table = Table(table_name, genome._metadata, *cols,\n autoload_replace=True, extend_existing=True)\n\n return table", "response": "alter the table to work between different dialects\n "} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef dataframe(self, table):\n from pandas import DataFrame\n if isinstance(table, six.string_types):\n table = getattr(self, table)\n try:\n rec = table.first()\n except AttributeError:\n rec = table[0]\n if hasattr(table, \"all\"):\n records = table.all()\n else:\n records = [tuple(t) for t in table]\n cols = [c.name for c in rec._table.columns]\n return DataFrame.from_records(records, columns=cols)", "response": "create a pandas dataframe from a table or query\n "} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload a file into a table", "response": "def load_file(self, fname, table=None, sep=\"\\t\", bins=False, indexes=None):\n \"\"\"\n use some of the machinery in pandas to load a file into a table\n\n Parameters\n ----------\n\n fname : str\n filename or filehandle to load\n\n table : str\n table to load the file to\n\n sep : str\n CSV separator\n\n bins : bool\n add a \"bin\" column for efficient spatial queries.\n\n indexes : list[str]\n list of columns to index\n\n \"\"\"\n convs = {\"#chr\": \"chrom\", \"start\": \"txStart\", \"end\": \"txEnd\", \"chr\":\n \"chrom\", \"pos\": \"start\", \"POS\": \"start\", \"chromStart\": \"txStart\",\n \"chromEnd\": \"txEnd\"}\n if table is None:\n import os.path as op\n table = op.basename(op.splitext(fname)[0]).replace(\".\", \"_\")\n print(\"writing to:\", table, file=sys.stderr)\n\n from pandas.io import sql\n import pandas as pa\n from toolshed import nopen\n\n needs_name = False\n for i, chunk in enumerate(pa.read_csv(nopen(fname), iterator=True,\n chunksize=100000, sep=sep, encoding=\"latin-1\")):\n chunk.columns = [convs.get(k, k) for k in chunk.columns]\n if not \"name\" in chunk.columns:\n needs_name = True\n chunk['name'] = chunk.get('chrom', chunk[chunk.columns[0]])\n if bins:\n chunk['bin'] = 1\n if i == 0 and not table in self.tables:\n flavor = self.url.split(\":\")[0]\n schema = sql.get_schema(chunk, table, flavor)\n print(schema)\n self.engine.execute(schema)\n elif i == 0:\n print >>sys.stderr,\\\n \"\"\"adding to existing table, you may want to drop first\"\"\"\n\n tbl = getattr(self, table)._table\n cols = chunk.columns\n data = list(dict(zip(cols, x)) for x in chunk.values)\n if needs_name:\n for d in data:\n d['name'] = \"%s:%s\" % (d.get(\"chrom\"), d.get(\"txStart\", d.get(\"chromStart\")))\n if bins:\n for d in data:\n d['bin'] = max(Genome.bins(int(d[\"txStart\"]), int(d[\"txEnd\"])))\n self.engine.execute(tbl.insert(), data)\n self.session.commit()\n if i > 0:\n print >>sys.stderr, \"writing row:\", i * 100000\n if \"txStart\" in chunk.columns:\n if \"chrom\" in chunk.columns:\n ssql = \"\"\"CREATE INDEX \"%s.chrom_txStart\" ON \"%s\" (chrom, txStart)\"\"\" % (table, table)\n else:\n ssql = \"\"\"CREATE INDEX \"%s.txStart\" ON \"%s\" (txStart)\"\"\" % (table, table)\n\n self.engine.execute(ssql)\n for index in (indexes or []):\n ssql = \"\"\"CREATE INDEX \"%s.%s\" ON \"%s\" (%s)\"\"\" % (table,\n index, table, index)\n self.engine.execute(ssql)\n\n if bins:\n ssql = \"\"\"CREATE INDEX \"%s.chrom_bin\" ON \"%s\" (chrom, bin)\"\"\" % (table, table)\n self.engine.execute(ssql)\n\n self.session.commit()"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef david_go(refseq_list, annot=('SP_PIR_KEYWORDS', 'GOTERM_BP_FAT',\n 'GOTERM_CC_FAT', 'GOTERM_MF_FAT')):\n\n \"\"\"\n open a web-browser to the DAVID online enrichment tool\n\n Parameters\n ----------\n\n refseq_list : list\n list of refseq names to check for enrichment\n\n annot : list\n iterable of DAVID annotations to check for enrichment\n \"\"\"\n URL = \"http://david.abcc.ncifcrf.gov/api.jsp?type=REFSEQ_MRNA&ids=%s&tool=term2term&annot=\"\n import webbrowser\n webbrowser.open(URL % \",\".join(set(refseq_list)) + \",\".join(annot))", "response": "Open a web - browser to the DAVID online enrichment tool."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef bin_query(self, table, chrom, start, end):\n if isinstance(table, six.string_types):\n table = getattr(self, table)\n\n try:\n tbl = table._table\n except AttributeError:\n tbl = table.column_descriptions[0]['type']._table\n\n q = table.filter(tbl.c.chrom == chrom)\n\n if hasattr(tbl.c, \"bin\"):\n bins = Genome.bins(start, end)\n if len(bins) < 100:\n q = q.filter(tbl.c.bin.in_(bins))\n\n if hasattr(tbl.c, \"txStart\"):\n return q.filter(tbl.c.txStart <= end).filter(tbl.c.txEnd >= start)\n return q.filter(tbl.c.chromStart <= end).filter(tbl.c.chromEnd >= start)", "response": "Perform an efficient spatial query using the bin column if available."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef upstream(self, table, chrom_or_feat, start=None, end=None, k=1):\n res = self.knearest(table, chrom_or_feat, start, end, k, \"up\")\n end = getattr(chrom_or_feat, \"end\", end)\n start = getattr(chrom_or_feat, \"start\", start)\n rev = getattr(chrom_or_feat, \"strand\", \"+\") == \"-\"\n if rev:\n return [x for x in res if x.end > start]\n else:\n return [x for x in res if x.start < end]", "response": "Return k - nearest upstream features for a given feature."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns k - nearest features for a given feature.", "response": "def knearest(self, table, chrom_or_feat, start=None, end=None, k=1,\n _direction=None):\n \"\"\"\n Return k-nearest features\n\n Parameters\n ----------\n\n table : str or table\n table against which to query\n\n chrom_or_feat : str or feat\n either a chromosome, e.g. 'chr3' or a feature with .chrom, .start,\n .end attributes\n\n start : int\n if `chrom_or_feat` is a chrom, then this must be the integer start\n\n end : int\n if `chrom_or_feat` is a chrom, then this must be the integer end\n\n k : int\n number of downstream neighbors to return\n\n _direction : (None, \"up\", \"down\")\n internal (don't use this)\n \"\"\"\n assert _direction in (None, \"up\", \"down\")\n\n # they sent in a feature\n if start is None:\n assert end is None\n chrom, start, end = chrom_or_feat.chrom, chrom_or_feat.start, chrom_or_feat.end\n\n # if the query is directional and the feature as a strand,\n # adjust...\n if _direction in (\"up\", \"down\") and getattr(chrom_or_feat,\n \"strand\", None) == \"-\":\n _direction = \"up\" if _direction == \"down\" else \"up\"\n else:\n chrom = chrom_or_feat\n\n qstart, qend = long(start), long(end)\n res = self.bin_query(table, chrom, qstart, qend)\n\n i, change = 1, 350\n try:\n while res.count() < k:\n if _direction in (None, \"up\"):\n if qstart == 0 and _direction == \"up\": break\n qstart = max(0, qstart - change)\n if _direction in (None, \"down\"):\n qend += change\n i += 1\n change *= (i + 5)\n res = self.bin_query(table, chrom, qstart, qend)\n except BigException:\n return []\n\n def dist(f):\n d = 0\n if start > f.end:\n d = start - f.end\n elif f.start > end:\n d = f.start - end\n # add dist as an attribute to the feature\n return d\n\n dists = sorted([(dist(f), f) for f in res])\n if len(dists) == 0:\n return []\n\n dists, res = zip(*dists)\n\n if len(res) == k:\n return res\n\n if k > len(res): # had to break because of end of chrom\n if k == 0: return []\n k = len(res)\n\n ndist = dists[k - 1]\n # include all features that are the same distance as the nth closest\n # feature (accounts for ties).\n while k < len(res) and dists[k] == ndist:\n k = k + 1\n return res[:k]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef annotate(self, fname, tables, feature_strand=False, in_memory=False,\n header=None, out=sys.stdout, parallel=False):\n \"\"\"\n annotate a file with a number of tables\n\n Parameters\n ----------\n\n fname : str or file\n file name or file-handle\n\n tables : list\n list of tables with which to annotate `fname`\n\n feature_strand : bool\n if this is True, then the up/downstream designations are based on\n the features in `tables` rather than the features in `fname`\n\n in_memoory : bool\n if True, then tables are read into memory. This usually makes the\n annotation much faster if there are more than 500 features in\n `fname` and the number of features in the table is less than 100K.\n\n header : str\n header to print out (if True, use existing header)\n\n out : file\n where to print output\n\n parallel : bool\n if True, use multiprocessing library to execute the annotation of\n each chromosome in parallel. Uses more memory.\n \"\"\"\n from .annotate import annotate\n return annotate(self, fname, tables, feature_strand, in_memory, header=header,\n out=out, parallel=parallel)", "response": "annotate a file with a number of tables"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef bins(start, end):\n if end - start < 536870912:\n offsets = [585, 73, 9, 1]\n else:\n raise BigException\n offsets = [4681, 585, 73, 9, 1]\n binFirstShift = 17\n binNextShift = 3\n\n start = start >> binFirstShift\n end = (end - 1) >> binFirstShift\n\n bins = [1]\n for offset in offsets:\n bins.extend(range(offset + start, offset + end + 1))\n start >>= binNextShift\n end >>= binNextShift\n return frozenset(bins)", "response": "Get all the bin numbers for a particular interval defined by\n "} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsave a bed12 file of the query.", "response": "def save_bed(cls, query, filename=sys.stdout):\n \"\"\"\n write a bed12 file of the query.\n Parameters\n ----------\n\n query : query\n a table or query to save to file\n filename : file\n string or filehandle to write output\n\n \"\"\"\n out = _open(filename, 'w')\n for o in query:\n out.write(o.bed() + '\\n')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a StaticFileNode instance for the given tag.", "response": "def staticfile_node(parser, token, optimize_if_possible=False):\n \"\"\"For example:\n {% staticfile \"/js/foo.js\" %}\n or\n {% staticfile \"/js/foo.js\" as variable_name %}\n Or for multiples:\n {% staticfile \"/foo.js; /bar.js\" %}\n or\n {% staticfile \"/foo.js; /bar.js\" as variable_name %}\n \"\"\"\n args = token.split_contents()\n tag = args[0]\n\n if len(args) == 4 and args[-2] == 'as':\n context_name = args[-1]\n args = args[:-2]\n else:\n context_name = None\n\n filename = parser.compile_filter(args[1])\n\n return StaticFileNode(filename,\n symlink_if_possible=_CAN_SYMLINK,\n optimize_if_possible=optimize_if_possible,\n context_name=context_name)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _mkdir(newdir):\n if os.path.isdir(newdir):\n pass\n elif os.path.isfile(newdir):\n raise OSError(\"a file with the same name as the desired \" \\\n \"dir, '%s', already exists.\" % newdir)\n else:\n head, tail = os.path.split(newdir)\n if head and not os.path.isdir(head):\n _mkdir(head)\n if tail:\n os.mkdir(newdir)", "response": "Make a new directory if it does not exist."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _find_filepath_in_roots(filename):\n for root in settings.DJANGO_STATIC_MEDIA_ROOTS:\n filepath = _filename2filepath(filename, root)\n if os.path.isfile(filepath):\n return filepath, root\n # havent found it in DJANGO_STATIC_MEDIA_ROOTS look for apps' files if we're\n # in DEBUG mode\n if settings.DEBUG:\n try:\n from django.contrib.staticfiles import finders\n absolute_path = finders.find(filename)\n if absolute_path:\n root, filepath = os.path.split(absolute_path)\n return absolute_path, root\n except ImportError:\n pass\n return None, None", "response": "Look for filename in all MEDIA_ROOTS and return the first one found."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef default_combine_filenames_generator(filenames, max_length=40):\n path = None\n names = []\n extension = None\n timestamps = []\n for filename in filenames:\n name = os.path.basename(filename)\n if not extension:\n extension = os.path.splitext(name)[1]\n elif os.path.splitext(name)[1] != extension:\n raise ValueError(\"Can't combine multiple file extensions\")\n\n for each in re.finditer('\\.\\d{10}\\.', name):\n timestamps.append(int(each.group().replace('.','')))\n name = name.replace(each.group(), '.')\n name = os.path.splitext(name)[0]\n names.append(name)\n\n if path is None:\n path = os.path.dirname(filename)\n else:\n if len(os.path.dirname(filename)) < len(path):\n path = os.path.dirname(filename)\n\n\n new_filename = '_'.join(names)\n if timestamps:\n new_filename += \".%s\" % max(timestamps)\n\n new_filename = new_filename[:max_length]\n new_filename += extension\n\n return os.path.join(path, new_filename)", "response": "This function is a helper function that returns a new filename to use as the combined file name for a bunch of files."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninspecting the code and look for files that can be turned into combos.", "response": "def render(self, context):\n \"\"\"inspect the code and look for files that can be turned into combos.\n Basically, the developer could type this:\n {% slimall %}\n \n \n {% endslimall %}\n And it should be reconsidered like this:\n \n which we already have routines for doing.\n \"\"\"\n code = self.nodelist.render(context)\n if not settings.DJANGO_STATIC:\n # Append MEDIA_URL if set\n # quick and dirty\n if settings.DJANGO_STATIC_MEDIA_URL_ALWAYS:\n for match in STYLES_REGEX.finditer(code):\n for filename in match.groups():\n code = (code.replace(filename,\n settings.DJANGO_STATIC_MEDIA_URL + filename))\n\n for match in SCRIPTS_REGEX.finditer(code):\n for filename in match.groups():\n code = (code.replace(filename,\n settings.DJANGO_STATIC_MEDIA_URL + filename))\n\n return code\n\n return code\n\n new_js_filenames = []\n for match in SCRIPTS_REGEX.finditer(code):\n whole_tag = match.group()\n async_defer = ASYNC_DEFER_REGEX.search(whole_tag)\n for filename in match.groups():\n\n optimize_if_possible = self.optimize_if_possible\n if optimize_if_possible and \\\n (filename.endswith('.min.js') or filename.endswith('.minified.js')):\n # Override! Because we simply don't want to run slimmer\n # on files that have the file extension .min.js\n optimize_if_possible = False\n\n new_js_filenames.append(filename)\n code = code.replace(whole_tag, '')\n\n # Now, we need to combine these files into one\n if new_js_filenames:\n new_js_filename = _static_file(new_js_filenames,\n optimize_if_possible=optimize_if_possible,\n symlink_if_possible=self.symlink_if_possible)\n else:\n new_js_filename = None\n\n new_image_filenames = []\n def image_replacer(match):\n tag = match.group()\n for filename in match.groups():\n new_filename = _static_file(filename,\n symlink_if_possible=self.symlink_if_possible)\n if new_filename != filename:\n tag = tag.replace(filename, new_filename)\n return tag\n\n code = IMG_REGEX.sub(image_replacer, code)\n\n new_css_filenames = defaultdict(list)\n\n # It's less trivial with CSS because we can't combine those that are\n # of different media\n media_regex = re.compile('media=[\"\\']([^\"\\']+)[\"\\']')\n for match in STYLES_REGEX.finditer(code):\n whole_tag = match.group()\n try:\n media_type = media_regex.findall(whole_tag)[0]\n except IndexError:\n media_type = ''\n\n for filename in match.groups():\n new_css_filenames[media_type].append(filename)\n code = code.replace(whole_tag, '')\n\n # Now, we need to combine these files into one\n new_css_filenames_combined = {}\n if new_css_filenames:\n for media_type, filenames in new_css_filenames.items():\n r = _static_file(filenames,\n optimize_if_possible=self.optimize_if_possible,\n symlink_if_possible=self.symlink_if_possible)\n new_css_filenames_combined[media_type] = r\n\n\n if new_js_filename:\n # Now is the time to apply the name prefix if there is one\n if async_defer:\n new_tag = ('' %\n (async_defer.group(0), new_js_filename))\n else:\n new_tag = '' % new_js_filename\n code = \"%s%s\" % (new_tag, code)\n\n for media_type, new_css_filename in new_css_filenames_combined.items():\n extra_params = ''\n if media_type:\n extra_params += ' media=\"%s\"' % media_type\n new_tag = '' % \\\n (extra_params, new_css_filename)\n code = \"%s%s\" % (new_tag, code)\n\n return code"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking for overlap with the other interval", "response": "def overlaps(self, other):\n \"\"\"\n check for overlap with the other interval\n \"\"\"\n if self.chrom != other.chrom: return False\n if self.start >= other.end: return False\n if other.start >= self.end: return False\n return True"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef is_upstream_of(self, other):\n if self.chrom != other.chrom: return None\n if getattr(other, \"strand\", None) == \"+\":\n return self.end <= other.start\n # other feature is on - strand, so this must have higher start\n return self.start >= other.end", "response": "check if this interval is upstream of the other interval taking the strand of the other interval into account\n "} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef distance(self, other_or_start=None, end=None, features=False):\n if end is None:\n assert other_or_start.chrom == self.chrom\n\n other_start, other_end = get_start_end(other_or_start, end)\n\n if other_start > self.end:\n return other_start - self.end\n if self.start > other_end:\n return self.start - other_end\n return 0", "response": "check the distance between this interval and another interval"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of exons for this object", "response": "def exons(self):\n \"\"\"\n return a list of exons [(start, stop)] for this object if appropriate\n \"\"\"\n # drop the trailing comma\n if not self.is_gene_pred: return []\n if hasattr(self, \"exonStarts\"):\n try:\n starts = (long(s) for s in self.exonStarts[:-1].split(\",\"))\n ends = (long(s) for s in self.exonEnds[:-1].split(\",\"))\n except TypeError:\n starts = (long(s) for s in self.exonStarts[:-1].decode().split(\",\"))\n ends = (long(s) for s in self.exonEnds[:-1].decode().split(\",\"))\n else: # it is bed12\n starts = [self.start + long(s) for s in\n self.chromStarts[:-1].decode().split(\",\")]\n ends = [starts[i] + long(size) for i, size \\\n in enumerate(self.blockSizes[:-1].decode().split(\",\"))]\n\n\n return zip(starts, ends)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef gene_features(self):\n nm, strand = self.gene_name, self.strand\n feats = [(self.chrom, self.start, self.end, nm, strand, 'gene')]\n for feat in ('introns', 'exons', 'utr5', 'utr3', 'cdss'):\n fname = feat[:-1] if feat[-1] == 's' else feat\n res = getattr(self, feat)\n if res is None or all(r is None for r in res): continue\n if not isinstance(res, list): res = [res]\n feats.extend((self.chrom, s, e, nm, strand, fname) for s, e in res)\n\n tss = self.tss(down=1)\n if tss is not None:\n feats.append((self.chrom, tss[0], tss[1], nm, strand, 'tss'))\n prom = self.promoter()\n feats.append((self.chrom, prom[0], prom[1], nm, strand, 'promoter'))\n\n return sorted(feats, key=itemgetter(1))", "response": "return a list of features for the gene features of this object."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef tss(self, up=0, down=0):\n if not self.is_gene_pred: return None\n tss = self.txEnd if self.strand == '-' else self.txStart\n start, end = tss, tss\n if self.strand == '+':\n start -= up\n end += down\n else:\n start += up\n end -= down\n start, end = end, start\n return max(0, start), max(end, start, 0)", "response": "Return a start end tuple of positions around the transcription - start\n site."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef promoter(self, up=2000, down=0):\n if not self.is_gene_pred: return None\n return self.tss(up=up, down=down)", "response": "Return a start end tuple of positions for the promoter region of this gene."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef coding_exons(self):\n # drop the trailing comma\n starts = (long(s) for s in self.exonStarts[:-1].split(\",\"))\n ends = (long(s) for s in self.exonEnds[:-1].split(\",\"))\n return [(s, e) for s, e in zip(starts, ends)\n if e > self.cdsStart and\n s < self.cdsEnd]", "response": "Returns a list of coding exon names."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cds(self):\n ces = self.coding_exons\n if len(ces) < 1: return ces\n ces[0] = (self.cdsStart, ces[0][1])\n ces[-1] = (ces[-1][0], self.cdsEnd)\n assert all((s < e for s, e in ces))\n return ces", "response": "just the parts of the exons that are translated"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef is_downstream_of(self, other):\n if self.chrom != other.chrom: return None\n if getattr(other, \"strand\", None) == \"-\":\n # other feature is on - strand, so this must have higher start\n return self.end <= other.start\n return self.start >= other.end", "response": "return a boolean indicating whether this feature is downstream of another"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a list of all features that overlap other_start and other_end", "response": "def features(self, other_start, other_end):\n \"\"\"\n return e.g. \"intron;exon\" if the other_start, end overlap introns and\n exons\n \"\"\"\n # completely encases gene.\n if other_start <= self.start and other_end >= self.end:\n return ['gene' if self.cdsStart != self.cdsEnd else 'nc_gene']\n other = Interval(other_start, other_end)\n ovls = []\n tx = 'txEnd' if self.strand == \"-\" else 'txStart'\n if hasattr(self, tx) and other_start <= getattr(self, tx) <= other_end \\\n and self.cdsStart != self.cdsEnd:\n ovls = [\"TSS\"]\n for ftype in ('introns', 'exons', 'utr5', 'utr3', 'cdss'):\n feats = getattr(self, ftype)\n if not isinstance(feats, list): feats = [feats]\n if any(Interval(f[0], f[1]).overlaps(other) for f in feats):\n ovls.append(ftype[:-1] if ftype[-1] == 's' else ftype)\n if 'cds' in ovls:\n ovls = [ft for ft in ovls if ft != 'exon']\n if self.cdsStart == self.cdsEnd:\n ovls = ['nc_' + ft for ft in ovls]\n return ovls"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the start and end of the upstream gene that is at least distance", "response": "def upstream(self, distance):\n \"\"\"\n return the (start, end) of the region before the geneStart\n \"\"\"\n if getattr(self, \"strand\", None) == \"+\":\n e = self.start\n s = e - distance\n else:\n s = self.end\n e = s + distance\n return self._xstream(s, e)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the 5 UTR if appropriate", "response": "def utr5(self):\n \"\"\"\n return the 5' UTR if appropriate\n \"\"\"\n if not self.is_coding or len(self.exons) < 2: return (None, None)\n if self.strand == \"+\":\n s, e = (self.txStart, self.cdsStart)\n else:\n s, e = (self.cdsEnd, self.txEnd)\n if s == e: return (None, None)\n return s, e"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sequence(self, per_exon=False):\n db = self.db\n if not per_exon:\n start = self.txStart + 1\n return _sequence(db, self.chrom, start, self.txEnd)\n else:\n # TODO: use same strategy as cds_sequence to reduce # of requests.\n seqs = []\n for start, end in self.exons:\n seqs.append(_sequence(db, self.chrom, start + 1, end))\n return seqs", "response": "Return the sequence for this feature."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nperforming an NCBI blast against the sequence of this feature.", "response": "def ncbi_blast(self, db=\"nr\", megablast=True, sequence=None):\n \"\"\"\n perform an NCBI blast against the sequence of this feature\n \"\"\"\n import requests\n requests.defaults.max_retries = 4\n assert sequence in (None, \"cds\", \"mrna\")\n seq = self.sequence() if sequence is None else (\"\".join(self.cds_sequence if sequence == \"cds\" else self.mrna_sequence))\n r = requests.post('http://blast.ncbi.nlm.nih.gov/Blast.cgi',\n timeout=20,\n data=dict(\n PROGRAM=\"blastn\",\n #EXPECT=2,\n DESCRIPTIONS=100,\n ALIGNMENTS=0,\n FILTER=\"L\", # low complexity\n CMD=\"Put\",\n MEGABLAST=True,\n DATABASE=db,\n QUERY=\">%s\\n%s\" % (self.name, seq)\n )\n )\n\n if not (\"RID =\" in r.text and \"RTOE\" in r.text):\n print(\"no results\", file=sys.stderr)\n raise StopIteration\n rid = r.text.split(\"RID = \")[1].split(\"\\n\")[0]\n\n import time\n time.sleep(4)\n print(\"checking...\", file=sys.stderr)\n r = requests.post('http://blast.ncbi.nlm.nih.gov/Blast.cgi',\n data=dict(RID=rid, format=\"Text\",\n DESCRIPTIONS=100,\n DATABASE=db,\n CMD=\"Get\", ))\n while \"Status=WAITING\" in r.text:\n print(\"checking...\", file=sys.stderr)\n time.sleep(10)\n r = requests.post('http://blast.ncbi.nlm.nih.gov/Blast.cgi',\n data=dict(RID=rid, format=\"Text\",\n CMD=\"Get\", ))\n for rec in _ncbi_parse(r.text):\n yield rec"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmaking a request to the genome - browsers BLAT interface returning a list of features that are hits to this sequence.", "response": "def blat(self, db=None, sequence=None, seq_type=\"DNA\"):\n \"\"\"\n make a request to the genome-browsers BLAT interface\n sequence is one of None, \"mrna\", \"cds\"\n returns a list of features that are hits to this sequence.\n \"\"\"\n from . blat_blast import blat, blat_all\n assert sequence in (None, \"cds\", \"mrna\")\n seq = self.sequence() if sequence is None else (\"\".join(self.cds_sequence if sequence == \"cds\" else self.mrna_sequence))\n if isinstance(db, (tuple, list)):\n return blat_all(seq, self.gene_name, db, seq_type)\n else:\n return blat(seq, self.gene_name, db or self.db, seq_type)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a bed formatted string of this feature", "response": "def bed(self, *attrs, **kwargs):\n \"\"\"\n return a bed formatted string of this feature\n \"\"\"\n exclude = (\"chrom\", \"start\", \"end\", \"txStart\", \"txEnd\", \"chromStart\",\n \"chromEnd\")\n if self.is_gene_pred:\n return self.bed12(**kwargs)\n return \"\\t\".join(map(str, (\n [self.chrom, self.start, self.end] +\n [getattr(self, attr) for attr in attrs if not attr in exclude]\n )))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a bed12 representation of this interval", "response": "def bed12(self, score=\"0\", rgb=\".\"):\n \"\"\"\n return a bed12 (http://genome.ucsc.edu/FAQ/FAQformat.html#format1)\n representation of this interval\n \"\"\"\n if not self.is_gene_pred:\n raise CruzException(\"can't create bed12 from non genepred feature\")\n exons = list(self.exons)\n # go from global start, stop, to relative start, length...\n sizes = \",\".join([str(e[1] - e[0]) for e in exons]) + \",\"\n starts = \",\".join([str(e[0] - self.txStart) for e in exons]) + \",\"\n name = self.name2 + \",\" + self.name if hasattr(self, \"name2\") \\\n else self.name\n return \"\\t\".join(map(str, (\n self.chrom, self.txStart, self.txEnd, name,\n score, self.strand, self.cdsStart, self.cdsEnd, rgb,\n len(exons), sizes, starts)))"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks the distance between this interval and another interval", "response": "def distance(self, other_or_start=None, end=None, features=\"unused\",\n shore_dist=3000):\n \"\"\"\n check the distance between this an another interval\n Parameters\n ----------\n\n other_or_start : Interval or int\n either an integer or an Interval with a start attribute indicating\n the start of the interval\n\n end : int\n if `other_or_start` is an integer, this must be an integer\n indicating the end of the interval\n\n features : bool\n if True, the features, such as CDS, intron, etc. that this feature\n overlaps are returned.\n \"\"\"\n # leave features kwarg to match signature from Feature.distance\n if end is None:\n assert other_or_start.chrom == self.chrom\n other_start, other_end = get_start_end(other_or_start, end)\n\n dist = 0\n if other_start > self.end:\n dist = other_start - self.end\n elif self.start > other_end:\n dist = self.start - other_end\n assert dist >= 0\n\n if dist > 0: dist = (dist, \"shore\" if abs(dist) <= shore_dist else \"\")\n else: dist = (0, \"island\")\n return dist"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef annotate(g, fname, tables, feature_strand=False, in_memory=False,\n header=None, out=sys.stdout, _chrom=None, parallel=False):\n \"\"\"\n annotate bed file in fname with tables.\n distances are integers for distance. and intron/exon/utr5 etc for gene-pred\n tables. if the annotation features have a strand, the distance reported is\n negative if the annotation feature is upstream of the feature in question\n if feature_strand is True, then the distance is negative if t\n \"\"\"\n close = False\n if isinstance(out, basestring):\n out = nopen(out, \"w\")\n close = True\n\n\n if parallel:\n import multiprocessing\n import signal\n p = multiprocessing.Pool(initializer=lambda:\n signal.signal(signal.SIGINT, signal.SIG_IGN))\n chroms = _split_chroms(fname)\n\n def write_result(fanno, written=[False]):\n for i, d in enumerate(reader(fanno, header=\"ordered\")):\n if i == 0 and written[0] == False:\n print >>out, \"\\t\".join(d.keys())\n written[0] = True\n print >>out, \"\\t\".join(x if x else \"NA\" for x in d.values())\n os.unlink(fanno)\n os.unlink(fanno.replace(\".anno\", \"\"))\n\n for fchrom, (fout, fanno) in chroms:\n p.apply_async(annotate, args=(g.db, fout.name, tables, feature_strand, True,\n header, fanno, fchrom),\n callback=write_result)\n p.close()\n p.join()\n return out.name\n\n if isinstance(g, basestring):\n from . import Genome\n g = Genome(g)\n if in_memory:\n from . intersecter import Intersecter\n intersecters = [] # 1 per table.\n for t in tables:\n q = getattr(g, t) if isinstance(t, basestring) else t\n if _chrom is not None:\n q = q.filter_by(chrom=_chrom)\n table_iter = q #page_query(q, g.session)\n intersecters.append(Intersecter(table_iter))\n\n elif isinstance(fname, basestring) and os.path.exists(fname) \\\n and sum(1 for _ in nopen(fname)) > 25000:\n print >>sys.stderr, \"annotating many intervals, may be faster using in_memory=True\"\n if header is None:\n header = []\n extra_header = []\n for j, toks in enumerate(reader(fname, header=False)):\n if j == 0 and not header:\n if not (toks[1] + toks[2]).isdigit():\n header = toks\n if j == 0:\n for t in tables:\n annos = (getattr(g, t) if isinstance(t, basestring) else t).first().anno_cols\n h = t if isinstance(t, basestring) else t._table.name if hasattr(t, \"_table\") else t.first()._table.name\n extra_header += [\"%s_%s\" % (h, a) for a in annos]\n\n if 0 != len(header):\n if not header[0].startswith(\"#\"):\n header[0] = \"#\" + header[0]\n print >>out, \"\\t\".join(header + extra_header)\n if header == toks: continue\n\n if not isinstance(toks, ABase):\n f = Feature()\n f.chrom = toks[0]\n f.txStart = int(toks[1])\n f.txEnd = int(toks[2])\n try:\n f.strand = toks[header.index('strand')]\n except ValueError:\n pass\n else:\n f = toks\n # for now, use the objects str to get the columns\n # might want to use getattr on the original cols\n\n toks = f.bed(*header).split(\"\\t\")\n sep = \"^*^\"\n for ti, tbl in enumerate(tables):\n if in_memory:\n objs = intersecters[ti].knearest(int(toks[1]), int(toks[2]), chrom=toks[0], k = 1)\n else:\n objs = g.knearest(tbl, toks[0], int(toks[1]), int(toks[2]), k=1)\n if len(objs) == 0:\n print >>out, \"\\t\".join(toks + [\"\", \"\", \"\"])\n continue\n\n gp = hasattr(objs[0], \"exonStarts\")\n names = [o.gene_name for o in objs]\n if feature_strand:\n strands = [-1 if f.is_upstream_of(o) else 1 for o in objs]\n else:\n strands = [-1 if o.is_upstream_of(f) else 1 for o in objs]\n\n # dists can be a list of tuples where the 2nd item is something\n # like 'island' or 'shore'\n dists = [o.distance(f, features=gp) for o in objs]\n pure_dists = [d[0] if isinstance(d, (tuple, list)) else d for d in dists]\n\n # convert to negative if the feature is upstream of the query\n for i, s in enumerate(strands):\n if s == 1: continue\n if isinstance(pure_dists[i], basestring): continue\n pure_dists[i] *= -1\n\n for i, (pd, d) in enumerate(zip(pure_dists, dists)):\n if isinstance(d, tuple):\n if len(d) > 1:\n dists[i] = \"%s%s%s\" % (pd, sep, sep.join(d[1:]))\n else:\n dists[i] = pd\n # keep uniqe name, dist combinations (occurs because of\n # transcripts)\n name_dists = set([\"%s%s%s\" % (n, sep, d) \\\n for (n, d) in zip(names, dists)])\n name_dists = [nd.split(sep) for nd in name_dists]\n\n # just take the first gene name if they are all the same\n if len(set(nd[0] for nd in name_dists)) == 1:\n toks.append(name_dists[0][0])\n else:\n toks.append(\";\".join(nd[0] for nd in name_dists))\n\n # iterate over the feat type, dist cols\n for i in range(1, len(name_dists[0])):\n\n toks.append(\";\".join(nd[i] for nd in name_dists))\n print >>out, \"\\t\".join(toks)\n\n if close:\n out.close()\n return out.name", "response": "annotate bed file in fname with tables."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef entry_point():\n try:\n main(\"omego\", items=[\n (InstallCommand.NAME, InstallCommand),\n (UpgradeCommand.NAME, UpgradeCommand),\n (ConvertCommand.NAME, ConvertCommand),\n (DownloadCommand.NAME, DownloadCommand),\n (DbCommand.NAME, DbCommand),\n (Version.NAME, Version)])\n except Stop, stop:\n if stop.rc != 0:\n print \"ERROR:\", stop\n else:\n print stop\n sys.exit(stop.rc)", "response": "Entry point which calls main and\n "} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open_url(url, httpuser=None, httppassword=None, method=None):\n if os.getenv('OMEGO_SSL_NO_VERIFY') == '1':\n # This needs to come first to override the default HTTPS handler\n log.debug('OMEGO_SSL_NO_VERIFY=1')\n try:\n sslctx = ssl.create_default_context()\n except Exception as e:\n log.error('Failed to create Default SSL context: %s' % e)\n raise Stop(\n 'Failed to create Default SSL context, OMEGO_SSL_NO_VERIFY '\n 'is not supported on older versions of Python')\n sslctx.check_hostname = False\n sslctx.verify_mode = ssl.CERT_NONE\n opener = urllib2.build_opener(urllib2.HTTPSHandler(context=sslctx))\n else:\n opener = urllib2.build_opener()\n\n if 'USER_AGENT' in os.environ:\n opener.addheaders = [('User-agent', os.environ.get('USER_AGENT'))]\n log.debug('Setting user-agent: %s', os.environ.get('USER_AGENT'))\n\n if httpuser and httppassword:\n mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()\n mgr.add_password(None, url, httpuser, httppassword)\n log.debug('Enabling HTTP authentication')\n opener.add_handler(urllib2.HTTPBasicAuthHandler(mgr))\n opener.add_handler(urllib2.HTTPDigestAuthHandler(mgr))\n elif httpuser or httppassword:\n raise FileException(\n 'httpuser and httppassword must be used together', url)\n\n # Override method http://stackoverflow.com/a/4421485\n req = urllib2.Request(url)\n if method:\n req.get_method = lambda: method\n\n return opener.open(req)", "response": "Open a URL using an opener that will simulate a browser user - agent url httpuser httppassword and method."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dereference_url(url):\n res = open_url(url, method='HEAD')\n res.close()\n return res.url", "response": "Dereferences a URL into a final destination."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef read(url, **kwargs):\n response = open_url(url, **kwargs)\n try:\n return response.read()\n finally:\n response.close()", "response": "Read the contents of a URL into memory."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef download(url, filename=None, print_progress=0, delete_fail=True,\n **kwargs):\n \"\"\"\n Download a file, optionally printing a simple progress bar\n url: The URL to download\n filename: The filename to save to, default is to use the URL basename\n print_progress: The length of the progress bar, use 0 to disable\n delete_fail: If True delete the file if the download was not successful,\n default is to keep the temporary file\n return: The downloaded filename\n \"\"\"\n blocksize = 1024 * 1024\n downloaded = 0\n progress = None\n\n log.info('Downloading %s', url)\n response = open_url(url, **kwargs)\n\n if not filename:\n filename = os.path.basename(url)\n\n output = None\n try:\n total = int(response.headers['Content-Length'])\n\n if print_progress:\n progress = ProgressBar(print_progress, total)\n\n with tempfile.NamedTemporaryFile(\n prefix=filename + '.', dir='.', delete=False) as output:\n while downloaded < total:\n block = response.read(blocksize)\n output.write(block)\n downloaded += len(block)\n if progress:\n progress.update(downloaded)\n os.rename(output.name, filename)\n output = None\n return filename\n finally:\n response.close()\n if delete_fail and output:\n os.unlink(output.name)", "response": "Download a file from the specified URL and save it to the specified filename."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef rename_backup(name, suffix='.bak'):\n newname = '%s%s' % (name, suffix)\n n = 0\n while os.path.exists(newname):\n n += 1\n newname = '%s%s.%d' % (name, suffix, n)\n log.info('Renaming %s to %s', name, newname)\n os.rename(name, newname)\n return newname", "response": "Rename a backup file or directory."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a string of the form [ basename - TIMESTAMP. ext ] where TIMESTAMP is of the form YYYYMMDD - HHMMSS - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC - MILSEC.", "response": "def timestamp_filename(basename, ext=None):\n \"\"\"\n Return a string of the form [basename-TIMESTAMP.ext]\n where TIMESTAMP is of the form YYYYMMDD-HHMMSS-MILSEC\n \"\"\"\n dt = datetime.now().strftime('%Y%m%d-%H%M%S-%f')\n if ext:\n return '%s-%s.%s' % (basename, dt, ext)\n return '%s-%s' % (basename, dt)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef check_extracted_paths(namelist, subdir=None):\n def relpath(p):\n # relpath strips a trailing sep\n # Windows paths may also use unix sep\n q = os.path.relpath(p)\n if p.endswith(os.path.sep) or p.endswith('/'):\n q += os.path.sep\n return q\n\n parent = os.path.abspath('.')\n if subdir:\n if os.path.isabs(subdir):\n raise FileException('subdir must be a relative path', subdir)\n subdir = relpath(subdir + os.path.sep)\n\n for name in namelist:\n if os.path.commonprefix([parent, os.path.abspath(name)]) != parent:\n raise FileException('Insecure path in zipfile', name)\n\n if subdir and os.path.commonprefix(\n [subdir, relpath(name)]) != subdir:\n raise FileException(\n 'Path in zipfile is not in required subdir', name)", "response": "Checks whether the paths in a list of files are all relative and optionally in a subdirectory of a subdirectory."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef unzip(filename, match_dir=False, destdir=None):\n if not destdir:\n destdir = '.'\n\n z = zipfile.ZipFile(filename)\n unzipped = '.'\n\n if match_dir:\n if not filename.endswith('.zip'):\n raise FileException('Expected .zip file extension', filename)\n unzipped = os.path.basename(filename)[:-4]\n check_extracted_paths(z.namelist(), unzipped)\n else:\n check_extracted_paths(z.namelist())\n\n # File permissions, see\n # http://stackoverflow.com/a/6297838\n # http://stackoverflow.com/a/3015466\n for info in z.infolist():\n log.debug('Extracting %s to %s', info.filename, destdir)\n z.extract(info, destdir)\n perms = info.external_attr >> 16 & 4095\n if perms > 0:\n os.chmod(os.path.join(destdir, info.filename), perms)\n\n return os.path.join(destdir, unzipped)", "response": "Unzips a zip file into a subdirectory of the current directory."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a new zip archive containing files and directories", "response": "def zip(filename, paths, strip_prefix=''):\n \"\"\"\n Create a new zip archive containing files\n filename: The name of the zip file to be created\n paths: A list of files or directories\n strip_dir: Remove this prefix from all file-paths before adding to zip\n \"\"\"\n if isinstance(paths, basestring):\n paths = [paths]\n\n filelist = set()\n for p in paths:\n if os.path.isfile(p):\n filelist.add(p)\n else:\n for root, dirs, files in os.walk(p):\n for f in files:\n filelist.add(os.path.join(root, f))\n\n z = zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED)\n for f in sorted(filelist):\n arcname = f\n if arcname.startswith(strip_prefix):\n arcname = arcname[len(strip_prefix):]\n if arcname.startswith(os.path.sep):\n arcname = arcname[1:]\n log.debug('Adding %s to %s[%s]', f, filename, arcname)\n z.write(f, arcname)\n\n z.close()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_as_local_path(path, overwrite, progress=0,\n httpuser=None, httppassword=None):\n \"\"\"\n Automatically handle local and remote URLs, files and directories\n\n path: Either a local directory, file or remote URL. If a URL is given\n it will be fetched. If this is a zip it will be automatically\n expanded by default.\n overwrite: Whether to overwrite an existing file:\n 'error': Raise an exception\n 'backup: Renamed the old file and use the new one\n 'keep': Keep the old file, don't overwrite or raise an exception\n progress: Number of progress dots, default 0 (don't print)\n httpuser, httppass: Credentials for HTTP authentication\n return: A tuple (type, localpath)\n type:\n 'file': localpath is the path to a local file\n 'directory': localpath is the path to a local directory\n 'unzipped': localpath is the path to a local unzipped directory\n \"\"\"\n m = re.match('([A-Za-z]+)://', path)\n if m:\n # url_open handles multiple protocols so don't bother validating\n log.debug('Detected URL protocol: %s', m.group(1))\n\n # URL should use / as the pathsep\n localpath = path.split('/')[-1]\n if not localpath:\n raise FileException(\n 'Remote path appears to be a directory', path)\n\n if os.path.exists(localpath):\n if overwrite == 'error':\n raise FileException('File already exists', localpath)\n elif overwrite == 'keep':\n log.info('Keeping existing %s', localpath)\n elif overwrite == 'backup':\n rename_backup(localpath)\n download(path, localpath, progress, httpuser=httpuser,\n httppassword=httppassword)\n else:\n raise Exception('Invalid overwrite flag: %s' % overwrite)\n else:\n download(path, localpath, progress, httpuser=httpuser,\n httppassword=httppassword)\n else:\n localpath = path\n log.debug(\"Local path: %s\", localpath)\n\n if os.path.isdir(localpath):\n return 'directory', localpath\n if os.path.exists(localpath):\n return 'file', localpath\n\n # Somethings gone very wrong\n raise Exception('Local path does not exist: %s' % localpath)", "response": "Get a file from the specified path as a local path."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create(fs, channels, application):\n\n result_code = ctypes.c_int()\n\n result = _create(fs, channels, application, ctypes.byref(result_code))\n if result_code.value is not constants.OK:\n raise OpusError(result_code.value)\n\n return result", "response": "Allocates and initializes an encoder state."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nencodes an Opus frame Returns string output", "response": "def encode(encoder, pcm, frame_size, max_data_bytes):\n \"\"\"Encodes an Opus frame\n\n Returns string output payload\n \"\"\"\n\n pcm = ctypes.cast(pcm, c_int16_pointer)\n data = (ctypes.c_char * max_data_bytes)()\n\n result = _encode(encoder, pcm, frame_size, data, max_data_bytes)\n if result < 0:\n raise OpusError(result)\n\n return array.array('c', data[:result]).tostring()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nencoding an Opus frame from floating point input", "response": "def encode_float(encoder, pcm, frame_size, max_data_bytes):\n \"\"\"Encodes an Opus frame from floating point input\"\"\"\n\n pcm = ctypes.cast(pcm, c_float_pointer)\n data = (ctypes.c_char * max_data_bytes)()\n\n result = _encode_float(encoder, pcm, frame_size, data, max_data_bytes)\n if result < 0:\n raise OpusError(result)\n\n return array.array('c', data[:result]).tostring()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nbuild and returns the MeCab function for parsing Unicode text.", "response": "def __parse_tostr(self, text, **kwargs):\n '''Builds and returns the MeCab function for parsing Unicode text.\n\n Args:\n fn_name: MeCab function name that determines the function\n behavior, either 'mecab_sparse_tostr' or\n 'mecab_nbest_sparse_tostr'.\n\n Returns:\n A function definition, tailored to parsing Unicode text and\n returning the result as a string suitable for display on stdout,\n using either the default or N-best behavior.\n '''\n n = self.options.get('nbest', 1)\n\n if self._KW_BOUNDARY in kwargs:\n patt = kwargs.get(self._KW_BOUNDARY, '.')\n tokens = list(self.__split_pattern(text, patt))\n text = ''.join([t[0] for t in tokens])\n\n btext = self.__str2bytes(text)\n self.__mecab.mecab_lattice_set_sentence(self.lattice, btext)\n\n bpos = 0\n self.__mecab.mecab_lattice_set_boundary_constraint(\n self.lattice, bpos, self.MECAB_TOKEN_BOUNDARY)\n\n for (token, match) in tokens:\n bpos += 1\n if match:\n mark = self.MECAB_INSIDE_TOKEN\n else:\n mark = self.MECAB_ANY_BOUNDARY\n\n for _ in range(1, len(self.__str2bytes(token))):\n self.__mecab.mecab_lattice_set_boundary_constraint(\n self.lattice, bpos, mark)\n bpos += 1\n self.__mecab.mecab_lattice_set_boundary_constraint(\n self.lattice, bpos, self.MECAB_TOKEN_BOUNDARY)\n elif self._KW_FEATURE in kwargs:\n features = kwargs.get(self._KW_FEATURE, ())\n fd = {morph: self.__str2bytes(feat) for morph, feat in features}\n\n tokens = self.__split_features(text, [e[0] for e in features])\n text = ''.join([t[0] for t in tokens])\n\n btext = self.__str2bytes(text)\n self.__mecab.mecab_lattice_set_sentence(self.lattice, btext)\n\n bpos = 0\n for chunk, match in tokens:\n c = len(self.__str2bytes(chunk))\n if match == True:\n self.__mecab.mecab_lattice_set_feature_constraint(\n self.lattice, bpos, bpos+c, fd[chunk])\n bpos += c\n else:\n btext = self.__str2bytes(text)\n self.__mecab.mecab_lattice_set_sentence(self.lattice, btext)\n\n self.__mecab.mecab_parse_lattice(self.tagger, self.lattice)\n\n if n > 1:\n res = self.__mecab.mecab_lattice_nbest_tostr(self.lattice, n)\n else:\n res = self.__mecab.mecab_lattice_tostr(self.lattice)\n\n if res != self.__ffi.NULL:\n raw = self.__ffi.string(res)\n return self.__bytes2str(raw).strip()\n else:\n err = self.__mecab.mecab_lattice_strerror(self.lattice)\n logger.error(self.__bytes2str(self.__ffi.string(err)))\n raise MeCabError(self.__bytes2str(self.__ffi.string(err)))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbuild and returns the MeCab function for parsing to nodes using the morpheme boundary constraints and parsing as features.", "response": "def __parse_tonodes(self, text, **kwargs):\n '''Builds and returns the MeCab function for parsing to nodes using\n morpheme boundary constraints.\n\n Args:\n format_feature: flag indicating whether or not to format the feature\n value for each node yielded.\n\n Returns:\n A function which returns a Generator, tailored to using boundary\n constraints and parsing as nodes, using either the default or\n N-best behavior.\n '''\n n = self.options.get('nbest', 1)\n\n try:\n if self._KW_BOUNDARY in kwargs:\n patt = kwargs.get(self._KW_BOUNDARY, '.')\n tokens = list(self.__split_pattern(text, patt))\n text = ''.join([t[0] for t in tokens])\n\n btext = self.__str2bytes(text)\n self.__mecab.mecab_lattice_set_sentence(self.lattice, btext)\n\n bpos = 0\n self.__mecab.mecab_lattice_set_boundary_constraint(\n self.lattice, bpos, self.MECAB_TOKEN_BOUNDARY)\n\n for (token, match) in tokens:\n bpos += 1\n if match:\n mark = self.MECAB_INSIDE_TOKEN\n else:\n mark = self.MECAB_ANY_BOUNDARY\n\n for _ in range(1, len(self.__str2bytes(token))):\n self.__mecab.mecab_lattice_set_boundary_constraint(\n self.lattice, bpos, mark)\n bpos += 1\n self.__mecab.mecab_lattice_set_boundary_constraint(\n self.lattice, bpos, self.MECAB_TOKEN_BOUNDARY)\n elif self._KW_FEATURE in kwargs:\n features = kwargs.get(self._KW_FEATURE, ())\n fd = {morph: self.__str2bytes(feat) for morph, feat in features}\n\n tokens = self.__split_features(text, [e[0] for e in features])\n text = ''.join([t[0] for t in tokens])\n\n btext = self.__str2bytes(text)\n self.__mecab.mecab_lattice_set_sentence(self.lattice, btext)\n\n bpos = 0\n for chunk, match in tokens:\n c = len(self.__str2bytes(chunk))\n if match:\n self.__mecab.mecab_lattice_set_feature_constraint(\n self.lattice, bpos, bpos+c, fd[chunk])\n bpos += c\n else:\n btext = self.__str2bytes(text)\n self.__mecab.mecab_lattice_set_sentence(self.lattice, btext)\n\n self.__mecab.mecab_parse_lattice(self.tagger, self.lattice)\n\n for _ in range(n):\n check = self.__mecab.mecab_lattice_next(self.lattice)\n if n == 1 or check:\n nptr = self.__mecab.mecab_lattice_get_bos_node(self.lattice)\n while nptr != self.__ffi.NULL:\n # skip over any BOS nodes, since mecab does\n if nptr.stat != MeCabNode.BOS_NODE:\n raws = self.__ffi.string(\n nptr.surface[0:nptr.length])\n surf = self.__bytes2str(raws).strip()\n\n if 'output_format_type' in self.options or \\\n 'node_format' in self.options:\n sp = self.__mecab.mecab_format_node(\n self.tagger, nptr)\n if sp != self.__ffi.NULL:\n rawf = self.__ffi.string(sp)\n else:\n err = self.__mecab.mecab_strerror(\n self.tagger)\n err = self.__bytes2str(\n self.__ffi.string(err))\n msg = self._ERROR_NODEFORMAT.format(\n surf, err)\n raise MeCabError(msg)\n else:\n rawf = self.__ffi.string(nptr.feature)\n feat = self.__bytes2str(rawf).strip()\n\n mnode = MeCabNode(nptr, surf, feat)\n yield mnode\n nptr = getattr(nptr, 'next')\n except GeneratorExit:\n logger.debug('close invoked on generator')\n except MeCabError:\n raise\n except:\n err = self.__mecab.mecab_lattice_strerror(self.lattice)\n logger.error(self.__bytes2str(self.__ffi.string(err)))\n raise MeCabError(self.__bytes2str(self.__ffi.string(err)))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef parse(self, text, **kwargs):\n '''Parse the given text and return result from MeCab.\n\n :param text: the text to parse.\n :type text: str\n :param as_nodes: return generator of MeCabNodes if True;\n or string if False.\n :type as_nodes: bool, defaults to False\n :param boundary_constraints: regular expression for morpheme boundary\n splitting; if non-None and feature_constraints is None, then\n boundary constraint parsing will be used.\n :type boundary_constraints: str or re\n :param feature_constraints: tuple containing tuple instances of\n target morpheme and corresponding feature string in order\n of precedence; if non-None and boundary_constraints is None,\n then feature constraint parsing will be used.\n :type feature_constraints: tuple\n :return: A single string containing the entire MeCab output;\n or a Generator yielding the MeCabNode instances.\n :raises: MeCabError\n '''\n if text is None:\n logger.error(self._ERROR_EMPTY_STR)\n raise MeCabError(self._ERROR_EMPTY_STR)\n elif not isinstance(text, str):\n logger.error(self._ERROR_NOTSTR)\n raise MeCabError(self._ERROR_NOTSTR)\n elif 'partial' in self.options and not text.endswith(\"\\n\"):\n logger.error(self._ERROR_MISSING_NL)\n raise MeCabError(self._ERROR_MISSING_NL)\n\n if self._KW_BOUNDARY in kwargs:\n val = kwargs[self._KW_BOUNDARY]\n if not isinstance(val, self._REGEXTYPE) and not isinstance(val, str):\n logger.error(self._ERROR_BOUNDARY)\n raise MeCabError(self._ERROR_BOUNDARY)\n elif self._KW_FEATURE in kwargs:\n val = kwargs[self._KW_FEATURE]\n if not isinstance(val, tuple):\n logger.error(self._ERROR_FEATURE)\n raise MeCabError(self._ERROR_FEATURE)\n\n as_nodes = kwargs.get(self._KW_ASNODES, False)\n\n if as_nodes:\n return self.__parse_tonodes(text, **kwargs)\n else:\n return self.__parse_tostr(text, **kwargs)", "response": "Parse the given text and return the result from MeCab."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses a GO term file into a tree of Term objects.", "response": "def parse(filename, MAX_TERM_COUNT=1000):\n \"\"\"\n MAX_TERM_COUNT = 10000 # There are 39,000 terms in the GO!\n \"\"\"\n with open(filename, \"r\") as f:\n\n termId = None\n name = None\n desc = None\n parents = []\n\n termCount = 0\n for l in f.readlines():\n if l.startswith(\"id:\"):\n termId = l.strip()[4:]\n if l.startswith(\"name:\"):\n name = l.strip()[6:]\n elif l.startswith(\"def:\"):\n desc = l.strip()[5:]\n elif l.startswith(\"is_a:\"):\n pid = l.strip()[6:].split(\" \", 1)[0]\n parents.append(pid)\n if len(l) == 1: # newline\n # save\n if termId is not None and name is not None:\n terms[termId] = {'name': name, 'desc': desc,\n 'parents': parents[:], 'children': []}\n termId = None\n name = None\n parents = []\n termCount += 1\n if MAX_TERM_COUNT is not None and \\\n termCount > MAX_TERM_COUNT:\n break\n\n count = 0\n for tid, tdict in terms.items():\n count += 1 # purely for display\n for p in tdict['parents']:\n if p in terms.keys():\n terms[p]['children'].append(tid)\n\n # Get unique term IDs for Tag Groups.\n tagGroups = set()\n for tid, tdict in terms.items():\n # Only create Tags for GO:terms that are 'leafs' of the tree\n if len(tdict['children']) == 0:\n for p in tdict['parents']:\n tagGroups.add(p)\n\n return tagGroups, terms"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef generate(tagGroups, terms):\n\n rv = []\n for pid in tagGroups:\n # In testing we may not have complete set\n if pid not in terms.keys():\n continue\n\n groupData = terms[pid]\n groupName = \"[%s] %s\" % (pid, groupData['name'])\n groupDesc = groupData['desc']\n children = []\n group = dict(name=groupName, desc=groupDesc, set=children)\n rv.append(group)\n\n for cid in groupData['children']:\n cData = terms[cid]\n cName = \"[%s] %s\" % (cid, cData['name'])\n cDesc = cData['desc']\n child = dict(name=cName, desc=cDesc)\n children.append(child)\n\n return json.dumps(rv, indent=2)", "response": "Generate a list of Tag Groups and Child Tags using data from terms dict\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _handle_args(self, cmd, args):\n if cmd == 'install':\n if args.upgrade:\n # Current behaviour: install or upgrade\n if args.initdb or args.upgradedb:\n raise Stop(10, (\n 'Deprecated --initdb --upgradedb flags '\n 'are incompatible with --upgrade'))\n newinstall = None\n else:\n # Current behaviour: Server must not exist\n newinstall = True\n\n if args.managedb:\n # Current behaviour\n if args.initdb or args.upgradedb:\n raise Stop(10, (\n 'Deprecated --initdb --upgradedb flags '\n 'are incompatible with --managedb'))\n args.initdb = True\n args.upgradedb = True\n else:\n if args.initdb or args.upgradedb:\n log.warn('--initdb and --upgradedb are deprecated, '\n 'use --managedb')\n\n elif cmd == 'upgrade':\n # Deprecated behaviour\n log.warn(\n '\"omero upgrade\" is deprecated, use \"omego install --upgrade\"')\n cmd = 'install'\n args.upgrade = True\n # Deprecated behaviour: Server must exist\n newinstall = False\n\n else:\n raise Exception('Unexpected command: %s' % cmd)\n\n return args, newinstall", "response": "Handle command line arguments."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_server_dir(self):\n if not self.args.server:\n if self.args.skipunzip:\n raise Stop(0, 'Unzip disabled, exiting')\n\n log.info('Downloading server')\n\n # The downloader automatically symlinks the server, however if\n # we are upgrading we want to delay the symlink swap, so this\n # overrides args.sym\n # TODO: Find a nicer way to do this?\n artifact_args = copy.copy(self.args)\n artifact_args.sym = ''\n artifacts = Artifacts(artifact_args)\n server = artifacts.download('server')\n else:\n progress = 0\n if self.args.verbose:\n progress = 20\n ptype, server = fileutils.get_as_local_path(\n self.args.server, self.args.overwrite, progress=progress,\n httpuser=self.args.httpuser,\n httppassword=self.args.httppassword)\n if ptype == 'file':\n if self.args.skipunzip:\n raise Stop(0, 'Unzip disabled, exiting')\n log.info('Unzipping %s', server)\n server = fileutils.unzip(\n server, match_dir=True, destdir=self.args.unzipdir)\n\n log.debug('Server directory: %s', server)\n return server", "response": "Download the server if necessary and unzips the server if necessary"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nhandle database initialization and upgrade", "response": "def handle_database(self):\n \"\"\"\n Handle database initialisation and upgrade, taking into account\n command line arguments\n \"\"\"\n # TODO: When initdb and upgradedb are dropped we can just test\n # managedb, but for backwards compatibility we need to support\n # initdb without upgradedb and vice-versa\n\n if self.args.initdb or self.args.upgradedb:\n db = DbAdmin(self.dir, None, self.args, self.external)\n status = db.check()\n log.debug('OMERO database upgrade status: %s', status)\n else:\n log.warn('OMERO database check disabled')\n return DB_INIT_NEEDED\n\n if status == DB_INIT_NEEDED:\n if self.args.initdb:\n log.debug('Initialising OMERO database')\n db.init()\n else:\n log.error('OMERO database not found')\n raise Stop(DB_INIT_NEEDED,\n 'Install/Upgrade failed: OMERO database not found')\n\n elif status == DB_UPGRADE_NEEDED:\n log.warn('OMERO database exists but is out of date')\n if self.args.upgradedb:\n log.debug('Upgrading OMERO database')\n db.upgrade()\n else:\n raise Stop(\n DB_UPGRADE_NEEDED,\n 'Pass --managedb or upgrade your OMERO database manually')\n\n else:\n assert status == DB_UPTODATE\n\n return status"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run(self, command):\n if isinstance(command, basestring):\n command = command.split()\n else:\n command = list(command)\n self.external.omero_cli(command)", "response": "Runs a command as if from the command - line\n without the need for using popen or subprocess."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning the omero command - line client with an array of arguments using the old environment", "response": "def bin(self, command):\n \"\"\"\n Runs the omero command-line client with an array of arguments using the\n old environment\n \"\"\"\n if isinstance(command, basestring):\n command = command.split()\n self.external.omero_bin(command)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking if the default symlink was changed from OMERO - CURRENT to OMERO. server.", "response": "def symlink_check_and_set(self):\n \"\"\"\n The default symlink was changed from OMERO-CURRENT to OMERO.server.\n If `--sym` was not specified and OMERO-CURRENT exists in the current\n directory stop and warn.\n \"\"\"\n if self.args.sym == '':\n if os.path.exists('OMERO-CURRENT'):\n log.error('Deprecated OMERO-CURRENT found but --sym not set')\n raise Stop(\n 30, 'The default for --sym has changed to OMERO.server '\n 'but the current directory contains OMERO-CURRENT. '\n 'Either remove OMERO-CURRENT or explicity pass --sym.')\n if self.args.sym in ('', 'auto'):\n self.args.sym = 'OMERO.server'"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nquerying encoder or decoder with a request value", "response": "def query(request):\n \"\"\"Query encoder/decoder with a request value\"\"\"\n\n def inner(func, obj):\n result_code = func(obj, request)\n\n if result_code is not constants.OK:\n raise OpusError(result_code)\n\n return result_code\n\n return inner"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get(request, result_type):\n\n def inner(func, obj):\n result = result_type()\n result_code = func(obj, request, ctypes.byref(result))\n\n if result_code is not constants.OK:\n raise OpusError(result_code)\n\n return result.value\n\n return inner", "response": "Get CTL value from a encoder or decoder."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets new CTL value to a encoder or decoder", "response": "def set(request):\n \"\"\"Set new CTL value to a encoder/decoder\"\"\"\n\n def inner(func, obj, value):\n result_code = func(obj, request, value)\n if result_code is not constants.OK:\n raise OpusError(result_code)\n\n return inner"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsorts a list of SQL schemas in order", "response": "def sort_schemas(schemas):\n \"\"\"Sort a list of SQL schemas in order\"\"\"\n def keyfun(v):\n x = SQL_SCHEMA_REGEXP.match(v).groups()\n # x3: 'DEV' should come before ''\n return (int(x[0]), x[1], int(x[2]) if x[2] else None,\n x[3] if x[3] else 'zzz', int(x[4]))\n\n return sorted(schemas, key=keyfun)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nparse a list of SQL files and return a dictionary of valid schema files where each key is a valid schema file and the corresponding value is a tuple containing the source and target schema.", "response": "def parse_schema_files(files):\n \"\"\"\n Parse a list of SQL files and return a dictionary of valid schema\n files where each key is a valid schema file and the corresponding value is\n a tuple containing the source and the target schema.\n \"\"\"\n f_dict = {}\n for f in files:\n root, ext = os.path.splitext(f)\n if ext != \".sql\":\n continue\n vto, vfrom = os.path.split(root)\n vto = os.path.split(vto)[1]\n if is_schema(vto) and is_schema(vfrom):\n f_dict[f] = (vfrom, vto)\n return f_dict"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndump the database using the postgres custom format", "response": "def dump(self):\n \"\"\"\n Dump the database using the postgres custom format\n \"\"\"\n dumpfile = self.args.dumpfile\n if not dumpfile:\n db, env = self.get_db_args_env()\n dumpfile = fileutils.timestamp_filename(\n 'omero-database-%s' % db['name'], 'pgdump')\n\n log.info('Dumping database to %s', dumpfile)\n if not self.args.dry_run:\n self.pgdump('-Fc', '-f', dumpfile)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_db_args_env(self):\n db = {\n 'name': self.args.dbname,\n 'host': self.args.dbhost,\n 'user': self.args.dbuser,\n 'pass': self.args.dbpass\n }\n\n if not self.args.no_db_config:\n try:\n c = self.external.get_config(force=True)\n except Exception as e:\n log.warn('config.xml not found: %s', e)\n c = {}\n\n for k in db:\n try:\n db[k] = c['omero.db.%s' % k]\n except KeyError:\n log.info(\n 'Failed to lookup parameter omero.db.%s, using %s',\n k, db[k])\n\n if not db['name']:\n raise Exception('Database name required')\n\n env = os.environ.copy()\n env['PGPASSWORD'] = db['pass']\n return db, env", "response": "Get a dictionary of database connection parameters and create an environment dictionary for running postgres commands."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef psql(self, *psqlargs):\n db, env = self.get_db_args_env()\n\n args = [\n '-v', 'ON_ERROR_STOP=on',\n '-d', db['name'],\n '-h', db['host'],\n '-U', db['user'],\n '-w', '-A', '-t'\n ] + list(psqlargs)\n stdout, stderr = External.run('psql', args, capturestd=True, env=env)\n if stderr:\n log.warn('stderr: %s', stderr)\n log.debug('stdout: %s', stdout)\n return stdout", "response": "Run a psql command and return the stdout and stderr"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pgdump(self, *pgdumpargs):\n db, env = self.get_db_args_env()\n\n args = ['-d', db['name'], '-h', db['host'], '-U', db['user'], '-w'\n ] + list(pgdumpargs)\n stdout, stderr = External.run(\n 'pg_dump', args, capturestd=True, env=env)\n if stderr:\n log.warn('stderr: %s', stderr)\n log.debug('stdout: %s', stdout)\n return stdout", "response": "Run a pg_dump command"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_server_dir(self, dir):\n self.dir = os.path.abspath(dir)\n config = os.path.join(self.dir, 'etc', 'grid', 'config.xml')\n self.configured = os.path.exists(config)", "response": "Set the directory of the server to be controlled by this object"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_config(self, force=False):\n if not force and not self.has_config():\n raise Exception('No config file')\n\n configxml = os.path.join(self.dir, 'etc', 'grid', 'config.xml')\n if not os.path.exists(configxml):\n raise Exception('No config file')\n\n try:\n # Attempt to open config.xml read-only, though this flag is not\n # present in early versions of OMERO 5.0\n c = self._omero.config.ConfigXml(\n configxml, exclusive=False, read_only=True)\n except TypeError:\n c = self._omero.config.ConfigXml(configxml, exclusive=False)\n\n try:\n return c.as_map()\n finally:\n c.close()", "response": "Returns a dictionary of all config. xml properties and omero module."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setup_omero_cli(self):\n if not self.dir:\n raise Exception('No server directory set')\n\n if 'omero.cli' in sys.modules:\n raise Exception('omero.cli can only be imported once')\n\n log.debug(\"Setting up omero CLI\")\n lib = os.path.join(self.dir, \"lib\", \"python\")\n if not os.path.exists(lib):\n raise Exception(\"%s does not exist!\" % lib)\n sys.path.insert(0, lib)\n\n import omero\n import omero.cli\n\n log.debug(\"Using omero CLI from %s\", omero.cli.__file__)\n\n self.cli = omero.cli.CLI()\n self.cli.loadplugins()\n self._omero = omero", "response": "Imports the omero. cli module so that commands can be run directly."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef setup_previous_omero_env(self, olddir, savevarsfile):\n env = self.get_environment(savevarsfile)\n\n def addpath(varname, p):\n if not os.path.exists(p):\n raise Exception(\"%s does not exist!\" % p)\n current = env.get(varname)\n if current:\n env[varname] = p + os.pathsep + current\n else:\n env[varname] = p\n\n olddir = os.path.abspath(olddir)\n lib = os.path.join(olddir, \"lib\", \"python\")\n addpath(\"PYTHONPATH\", lib)\n bin = os.path.join(olddir, \"bin\")\n addpath(\"PATH\", bin)\n self.old_env = env", "response": "Setup the environment for interacting with the current OMERO server installation"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef omero_cli(self, command):\n assert isinstance(command, list)\n if not self.cli:\n raise Exception('omero.cli not initialised')\n log.info(\"Invoking CLI [current environment]: %s\", \" \".join(command))\n self.cli.invoke(command, strict=True)", "response": "Runs a command from the OMERO command - line without the need\n flag."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning the omero command - line client with an array of arguments using the old environment", "response": "def omero_bin(self, command):\n \"\"\"\n Runs the omero command-line client with an array of arguments using the\n old environment\n \"\"\"\n assert isinstance(command, list)\n if not self.old_env:\n raise Exception('Old environment not initialised')\n log.info(\"Running [old environment]: %s\", \" \".join(command))\n self.run('omero', command, capturestd=True, env=self.old_env)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef run(exe, args, capturestd=False, env=None):\n command = [exe] + args\n if env:\n log.info(\"Executing [custom environment]: %s\", \" \".join(command))\n else:\n log.info(\"Executing : %s\", \" \".join(command))\n start = time.time()\n\n # Temp files will be automatically deleted on close()\n # If run() throws the garbage collector should call close(), so don't\n # bother with try-finally\n outfile = None\n errfile = None\n if capturestd:\n outfile = tempfile.TemporaryFile()\n errfile = tempfile.TemporaryFile()\n\n # Use call instead of Popen so that stdin is connected to the console,\n # in case user input is required\n # On Windows shell=True is needed otherwise the modified environment\n # PATH variable is ignored. On Unix this breaks things.\n r = subprocess.call(\n command, env=env, stdout=outfile, stderr=errfile, shell=WINDOWS)\n\n stdout = None\n stderr = None\n if capturestd:\n outfile.seek(0)\n stdout = outfile.read()\n outfile.close()\n errfile.seek(0)\n stderr = errfile.read()\n errfile.close()\n\n end = time.time()\n if r != 0:\n log.error(\"Failed [%.3f s]\", end - start)\n raise RunException(\n \"Non-zero return code\", exe, args, r, stdout, stderr)\n log.info(\"Completed [%.3f s]\", end - start)\n return stdout, stderr", "response": "Runs an executable with an array of arguments optionally in the specified environment. Returns stdout and stderr."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating byte - to - string and string - to - byte conversion functions for internal use.", "response": "def string_support(py3enc):\n '''Create byte-to-string and string-to-byte conversion functions for\n internal use.\n\n :param py3enc: Encoding used by Python 3 environment.\n :type py3enc: str\n '''\n if sys.version < '3':\n def bytes2str(b):\n '''Identity, returns the argument string (bytes).'''\n return b\n def str2bytes(s):\n '''Identity, returns the argument string (bytes).'''\n return s\n else:\n def bytes2str(b):\n '''Transforms bytes into string (Unicode).'''\n return b.decode(py3enc)\n def str2bytes(u):\n '''Transforms Unicode into string (bytes).'''\n return u.encode(py3enc)\n return (bytes2str, str2bytes)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef splitter_support(py2enc):\n '''Create tokenizer for use in boundary constraint parsing.\n\n :param py2enc: Encoding used by Python 2 environment.\n :type py2enc: str\n '''\n if sys.version < '3':\n def _fn_sentence(pattern, sentence):\n if REGEXTYPE == type(pattern):\n if pattern.flags & re.UNICODE:\n return sentence.decode(py2enc)\n else:\n return sentence\n else:\n return sentence\n def _fn_token2str(pattern):\n if REGEXTYPE == type(pattern):\n if pattern.flags & re.UNICODE:\n def _fn(token):\n return token.encode(py2enc)\n else:\n def _fn(token):\n return token\n else:\n def _fn(token):\n return token\n return _fn\n else:\n def _fn_sentence(pattern, sentence):\n return sentence\n def _fn_token2str(pattern):\n def _fn(token):\n return token\n return _fn\n\n def _fn_tokenize_pattern(text, pattern):\n pos = 0\n sentence = _fn_sentence(pattern, text)\n postprocess = _fn_token2str(pattern)\n for m in re.finditer(pattern, sentence):\n if pos < m.start():\n token = postprocess(sentence[pos:m.start()])\n yield (token.strip(), False)\n pos = m.start()\n token = postprocess(sentence[pos:m.end()])\n yield (token.strip(), True)\n pos = m.end()\n if pos < len(sentence):\n token = postprocess(sentence[pos:])\n yield (token.strip(), False)\n\n def _fn_tokenize_features(text, features):\n acc = []\n acc.append((text.strip(), False))\n\n for feat in features:\n for i,e in enumerate(acc):\n if e[1]==False:\n tmp = list(_fn_tokenize_pattern(e[0], feat))\n if len(tmp) > 0:\n acc.pop(i)\n acc[i:i] = tmp\n return acc\n \n return _fn_tokenize_pattern, _fn_tokenize_features", "response": "Create tokenizer for use in boundary constraint parsing."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating the document with the given id with the given update spec.", "response": "def update(self, document_id, update_spec, namespace, timestamp):\n \"\"\"Apply updates given in update_spec to the document whose id\n matches that of doc.\n \"\"\"\n\n index, doc_type = self._index_and_mapping(namespace)\n with self.lock:\n # Check if document source is stored in local buffer\n document = self.BulkBuffer.get_from_sources(index,\n doc_type,\n u(document_id))\n if document:\n # Document source collected from local buffer\n # Perform apply_update on it and then it will be\n # ready for commiting to Elasticsearch\n updated = self.apply_update(document, update_spec)\n # _id is immutable in MongoDB, so won't have changed in update\n updated['_id'] = document_id\n self.upsert(updated, namespace, timestamp)\n else:\n # Document source needs to be retrieved from Elasticsearch\n # before performing update. Pass update_spec to upsert function\n updated = {\"_id\": document_id}\n self.upsert(updated, namespace, timestamp, update_spec)\n # upsert() strips metadata, so only _id + fields in _source still here\n return updated"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef upsert(self, doc, namespace, timestamp, update_spec=None):\n index, doc_type = self._index_and_mapping(namespace)\n # No need to duplicate '_id' in source document\n doc_id = u(doc.pop(\"_id\"))\n metadata = {\n 'ns': namespace,\n '_ts': timestamp\n }\n\n # Index the source document, using lowercase namespace as index name.\n action = {\n '_op_type': 'index',\n '_index': index,\n '_type': doc_type,\n '_id': doc_id,\n '_source': self._formatter.format_document(doc)\n }\n # Index document metadata with original namespace (mixed upper/lower).\n meta_action = {\n '_op_type': 'index',\n '_index': self.meta_index_name,\n '_type': self.meta_type,\n '_id': doc_id,\n '_source': bson.json_util.dumps(metadata)\n }\n\n self.index(action, meta_action, doc, update_spec)\n\n # Leave _id, since it's part of the original document\n doc['_id'] = doc_id", "response": "Insert a document into Elasticsearch."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninsert multiple documents into Elasticsearch.", "response": "def bulk_upsert(self, docs, namespace, timestamp):\n \"\"\"Insert multiple documents into Elasticsearch.\"\"\"\n def docs_to_upsert():\n doc = None\n for doc in docs:\n # Remove metadata and redundant _id\n index, doc_type = self._index_and_mapping(namespace)\n doc_id = u(doc.pop(\"_id\"))\n document_action = {\n '_index': index,\n '_type': doc_type,\n '_id': doc_id,\n '_source': self._formatter.format_document(doc)\n }\n document_meta = {\n '_index': self.meta_index_name,\n '_type': self.meta_type,\n '_id': doc_id,\n '_source': {\n 'ns': namespace,\n '_ts': timestamp\n }\n }\n yield document_action\n yield document_meta\n if doc is None:\n raise errors.EmptyDocsError(\n \"Cannot upsert an empty sequence of \"\n \"documents into Elastic Search\")\n try:\n kw = {}\n if self.chunk_size > 0:\n kw['chunk_size'] = self.chunk_size\n\n responses = streaming_bulk(client=self.elastic,\n actions=docs_to_upsert(),\n **kw)\n\n for ok, resp in responses:\n if not ok:\n LOG.error(\n \"Could not bulk-upsert document \"\n \"into ElasticSearch: %r\" % resp)\n if self.auto_commit_interval == 0:\n self.commit()\n except errors.EmptyDocsError:\n # This can happen when mongo-connector starts up, there is no\n # config file, but nothing to dump\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef remove(self, document_id, namespace, timestamp):\n index, doc_type = self._index_and_mapping(namespace)\n\n action = {\n '_op_type': 'delete',\n '_index': index,\n '_type': doc_type,\n '_id': u(document_id)\n }\n\n meta_action = {\n '_op_type': 'delete',\n '_index': self.meta_index_name,\n '_type': self.meta_type,\n '_id': u(document_id)\n }\n\n self.index(action, meta_action)", "response": "Remove a document from Elasticsearch."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef send_buffered_operations(self):\n with self.lock:\n try:\n action_buffer = self.BulkBuffer.get_buffer()\n if action_buffer:\n successes, errors = bulk(self.elastic, action_buffer)\n LOG.debug(\"Bulk request finished, successfully sent %d \"\n \"operations\", successes)\n if errors:\n LOG.error(\n \"Bulk request finished with errors: %r\", errors)\n except es_exceptions.ElasticsearchException:\n LOG.exception(\"Bulk request failed with exception\")", "response": "Send buffered operations to Elasticsearch."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the most recently modified document from Elasticsearch.", "response": "def get_last_doc(self):\n \"\"\"Get the most recently modified document from Elasticsearch.\n\n This method is used to help define a time window within which documents\n may be in conflict after a MongoDB rollback.\n \"\"\"\n try:\n result = self.elastic.search(\n index=self.meta_index_name,\n body={\n \"query\": {\"match_all\": {}},\n \"sort\": [{\"_ts\": \"desc\"}],\n },\n size=1\n )[\"hits\"][\"hits\"]\n for r in result:\n r['_source']['_id'] = r['_id']\n return r['_source']\n except es_exceptions.RequestError:\n # no documents so ES returns 400 because of undefined _ts mapping\n return None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsplitting a list of parameters and types by commas and brackets.", "response": "def split_sig(params):\n \"\"\"\n Split a list of parameters/types by commas,\n whilst respecting brackets.\n\n For example:\n String arg0, int arg2 = 1, List arg3 = [1, 2, 3]\n => ['String arg0', 'int arg2 = 1', 'List arg3 = [1, 2, 3]']\n \"\"\"\n result = []\n current = ''\n level = 0\n for char in params:\n if char in ('<', '{', '['):\n level += 1\n elif char in ('>', '}', ']'):\n level -= 1\n if char != ',' or level > 0:\n current += char\n elif char == ',' and level == 0:\n result.append(current)\n current = ''\n if current.strip() != '':\n result.append(current)\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_method_signature(sig):\n match = METH_SIG_RE.match(sig.strip())\n if not match:\n raise RuntimeError('Method signature invalid: ' + sig)\n modifiers, return_type, name, generic_types, params = match.groups()\n if params.strip() != '':\n params = split_sig(params)\n params = [parse_param_signature(x) for x in params]\n else:\n params = []\n return (modifiers.split(), return_type, name, generic_types, params)", "response": "Parse a method signature into a tuple of modifiers return_type name generic_types params."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_property_signature(sig):\n match = PROP_SIG_RE.match(sig.strip())\n if not match:\n raise RuntimeError('Property signature invalid: ' + sig)\n groups = match.groups()\n if groups[0] is not None:\n modifiers = [x.strip() for x in groups[:-4]]\n groups = groups[-4:]\n else:\n modifiers = []\n groups = groups[1:]\n typ, name, getter, setter = groups\n return (modifiers, typ, name, getter is not None, setter is not None)", "response": "Parse a property signature into a tuple of modifiers type name getter and setter."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses a indexer signature of the form modifier type params getter and setter.", "response": "def parse_indexer_signature(sig):\n \"\"\" Parse a indexer signature of the form:\n modifier* type this[params] { (get;)? (set;)? } \"\"\"\n match = IDXR_SIG_RE.match(sig.strip())\n if not match:\n raise RuntimeError('Indexer signature invalid: ' + sig)\n modifiers, return_type, params, getter, setter = match.groups()\n params = split_sig(params)\n params = [parse_param_signature(x) for x in params]\n return (modifiers.split(), return_type, params,\n getter is not None, setter is not None)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef parse_param_signature(sig):\n match = PARAM_SIG_RE.match(sig.strip())\n if not match:\n raise RuntimeError('Parameter signature invalid, got ' + sig)\n groups = match.groups()\n modifiers = groups[0].split()\n typ, name, _, default = groups[-4:]\n return ParamTuple(name=name, typ=typ,\n default=default, modifiers=modifiers)", "response": "Parse a parameter signature into a tuple."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_type_signature(sig):\n match = TYPE_SIG_RE.match(sig.strip())\n if not match:\n raise RuntimeError('Type signature invalid, got ' + sig)\n groups = match.groups()\n typ = groups[0]\n generic_types = groups[1]\n if not generic_types:\n generic_types = []\n else:\n generic_types = split_sig(generic_types[1:-1])\n is_array = (groups[2] is not None)\n return typ, generic_types, is_array", "response": "Parse a type signature into its components."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_attr_signature(sig):\n match = ATTR_SIG_RE.match(sig.strip())\n if not match:\n raise RuntimeError('Attribute signature invalid, got ' + sig)\n name, _, params = match.groups()\n if params is not None and params.strip() != '':\n params = split_sig(params)\n params = [parse_param_signature(x) for x in params]\n else:\n params = []\n return (name, params)", "response": "Parse an attribute signature."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ntries and create a reference to a type on MSDN", "response": "def get_msdn_ref(name):\n \"\"\" Try and create a reference to a type on MSDN \"\"\"\n in_msdn = False\n if name in MSDN_VALUE_TYPES:\n name = MSDN_VALUE_TYPES[name]\n in_msdn = True\n if name.startswith('System.'):\n in_msdn = True\n if in_msdn:\n link = name.split('<')[0]\n if link in MSDN_LINK_MAP:\n link = MSDN_LINK_MAP[link]\n else:\n link = link.lower()\n url = 'https://msdn.microsoft.com/en-us/library/'+link+'.aspx'\n node = nodes.reference(name, shorten_type(name))\n node['refuri'] = url\n node['reftitle'] = name\n return node\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nshortens a type. Eg. drops System.", "response": "def shorten_type(typ):\n \"\"\" Shorten a type. E.g. drops 'System.' \"\"\"\n offset = 0\n for prefix in SHORTEN_TYPE_PREFIXES:\n if typ.startswith(prefix):\n if len(prefix) > offset:\n offset = len(prefix)\n return typ[offset:]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef parse_mecab_options(self, options):\r\n '''Parses the MeCab options, returning them in a dictionary.\r\n\r\n Lattice-level option has been deprecated; please use marginal or nbest\r\n instead.\r\n\r\n :options string or dictionary of options to use when instantiating\r\n the MeCab instance. May be in short- or long-form, or in a\r\n Python dictionary.\r\n\r\n Returns:\r\n A dictionary of the specified MeCab options, where the keys are\r\n snake-cased names of the long-form of the option names.\r\n\r\n Raises:\r\n MeCabError: An invalid value for N-best was passed in.\r\n '''\r\n class MeCabArgumentParser(argparse.ArgumentParser):\r\n '''MeCab option parser for natto-py.'''\r\n\r\n def error(self, message):\r\n '''error(message: string)\r\n\r\n Raises ValueError.\r\n '''\r\n raise ValueError(message)\r\n\r\n options = options or {}\r\n dopts = {}\r\n\r\n if type(options) is dict:\r\n for name in iter(list(self._SUPPORTED_OPTS.values())):\r\n if name in options:\r\n if options[name] or options[name] is '':\r\n val = options[name]\r\n if isinstance(val, bytes):\r\n val = self.__bytes2str(options[name])\r\n dopts[name] = val\r\n else:\r\n p = MeCabArgumentParser()\r\n p.add_argument('-r', '--rcfile',\r\n help='use FILE as a resource file',\r\n action='store', dest='rcfile')\r\n p.add_argument('-d', '--dicdir',\r\n help='set DIR as a system dicdir',\r\n action='store', dest='dicdir')\r\n p.add_argument('-u', '--userdic',\r\n help='use FILE as a user dictionary',\r\n action='store', dest='userdic')\r\n p.add_argument('-l', '--lattice-level',\r\n help='lattice information level (DEPRECATED)',\r\n action='store', dest='lattice_level', type=int)\r\n p.add_argument('-O', '--output-format-type',\r\n help='set output format type (wakati, none,...)',\r\n action='store', dest='output_format_type')\r\n p.add_argument('-a', '--all-morphs',\r\n help='output all morphs (default false)',\r\n action='store_true', default=False)\r\n p.add_argument('-N', '--nbest',\r\n help='output N best results (default 1)',\r\n action='store', dest='nbest', type=int)\r\n p.add_argument('-p', '--partial',\r\n help='partial parsing mode (default false)',\r\n action='store_true', default=False)\r\n p.add_argument('-m', '--marginal',\r\n help='output marginal probability (default false)',\r\n action='store_true', default=False)\r\n p.add_argument('-M', '--max-grouping-size',\r\n help=('maximum grouping size for unknown words '\r\n '(default 24)'),\r\n action='store', dest='max_grouping_size', type=int)\r\n p.add_argument('-F', '--node-format',\r\n help='use STR as the user-defined node format',\r\n action='store', dest='node_format')\r\n p.add_argument('-U', '--unk-format',\r\n help=('use STR as the user-defined unknown '\r\n 'node format'),\r\n action='store', dest='unk_format')\r\n p.add_argument('-B', '--bos-format',\r\n help=('use STR as the user-defined '\r\n 'beginning-of-sentence format'),\r\n action='store', dest='bos_format')\r\n p.add_argument('-E', '--eos-format',\r\n help=('use STR as the user-defined '\r\n 'end-of-sentence format'),\r\n action='store', dest='eos_format')\r\n p.add_argument('-S', '--eon-format',\r\n help=('use STR as the user-defined end-of-NBest '\r\n 'format'),\r\n action='store', dest='eon_format')\r\n p.add_argument('-x', '--unk-feature',\r\n help='use STR as the feature for unknown word',\r\n action='store', dest='unk_feature')\r\n p.add_argument('-b', '--input-buffer-size',\r\n help='set input buffer size (default 8192)',\r\n action='store', dest='input_buffer_size', type=int)\r\n p.add_argument('-C', '--allocate-sentence',\r\n help='allocate new memory for input sentence',\r\n action='store_true', dest='allocate_sentence',\r\n default=False)\r\n p.add_argument('-t', '--theta',\r\n help=('set temperature parameter theta '\r\n '(default 0.75)'),\r\n action='store', dest='theta', type=float)\r\n p.add_argument('-c', '--cost-factor',\r\n help='set cost factor (default 700)',\r\n action='store', dest='cost_factor', type=int)\r\n\r\n opts = p.parse_args([o.replace('\\\"', '').replace('\\'', '') for o in options.split()])\r\n\r\n for name in iter(list(self._SUPPORTED_OPTS.values())):\r\n if hasattr(opts, name):\r\n v = getattr(opts, name)\r\n if v or v is '':\r\n dopts[name] = v\r\n\r\n # final checks\r\n if 'nbest' in dopts \\\r\n and (dopts['nbest'] < 1 or dopts['nbest'] > self._NBEST_MAX):\r\n logger.error(self._ERROR_NVALUE)\r\n raise ValueError(self._ERROR_NVALUE)\r\n\r\n # warning for lattice-level deprecation\r\n if 'lattice_level' in dopts:\r\n logger.warn('WARNING: {}\\n'.format(self._WARN_LATTICE_LEVEL))\r\n\r\n return dopts", "response": "Parses the specified MeCab options and returns them in a dictionary."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nbuilding a string concatenation of the options used when instantiating the instance in long - form.", "response": "def build_options_str(self, options):\r\n '''Returns a string concatenation of the MeCab options.\r\n\r\n Args:\r\n options: dictionary of options to use when instantiating the MeCab\r\n instance.\r\n\r\n Returns:\r\n A string concatenation of the options used when instantiating the\r\n MeCab instance, in long-form.\r\n '''\r\n opts = []\r\n for name in iter(list(self._SUPPORTED_OPTS.values())):\r\n if name in options:\r\n key = name.replace('_', '-')\r\n if key in self._BOOLEAN_OPTIONS:\r\n if options[name]:\r\n opts.append('--{}'.format(key))\r\n else:\r\n opts.append('--{}={}'.format(key, options[name]))\r\n\r\n return self.__str2bytes(' '.join(opts))"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nallocates and initializes a decoder state", "response": "def create(fs, channels):\n \"\"\"Allocates and initializes a decoder state\"\"\"\n\n result_code = ctypes.c_int()\n\n result = _create(fs, channels, ctypes.byref(result_code))\n if result_code.value is not 0:\n raise OpusError(result_code.value)\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the bandwidth of an Opus packet.", "response": "def packet_get_bandwidth(data):\n \"\"\"Gets the bandwidth of an Opus packet.\"\"\"\n\n data_pointer = ctypes.c_char_p(data)\n\n result = _packet_get_bandwidth(data_pointer)\n if result < 0:\n raise OpusError(result)\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the number of channels from an Opus packet", "response": "def packet_get_nb_channels(data):\n \"\"\"Gets the number of channels from an Opus packet\"\"\"\n\n data_pointer = ctypes.c_char_p(data)\n\n result = _packet_get_nb_channels(data_pointer)\n if result < 0:\n raise OpusError(result)\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the number of frames in an Opus packet", "response": "def packet_get_nb_frames(data, length=None):\n \"\"\"Gets the number of frames in an Opus packet\"\"\"\n\n data_pointer = ctypes.c_char_p(data)\n if length is None:\n length = len(data)\n\n result = _packet_get_nb_frames(data_pointer, ctypes.c_int(length))\n if result < 0:\n raise OpusError(result)\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef packet_get_samples_per_frame(data, fs):\n\n data_pointer = ctypes.c_char_p(data)\n\n result = _packet_get_nb_frames(data_pointer, ctypes.c_int(fs))\n if result < 0:\n raise OpusError(result)\n\n return result", "response": "Gets the number of samples per frame from an Opus packet"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndecode an Opus frame", "response": "def decode(decoder, data, length, frame_size, decode_fec, channels=2):\n \"\"\"Decode an Opus frame\n\n Unlike the `opus_decode` function , this function takes an additional parameter `channels`,\n which indicates the number of channels in the frame\n \"\"\"\n\n pcm_size = frame_size * channels * ctypes.sizeof(ctypes.c_int16)\n pcm = (ctypes.c_int16 * pcm_size)()\n pcm_pointer = ctypes.cast(pcm, c_int16_pointer)\n\n # Converting from a boolean to int\n decode_fec = int(bool(decode_fec))\n\n result = _decode(decoder, data, length, pcm_pointer, frame_size, decode_fec)\n if result < 0:\n raise OpusError(result)\n\n return array.array('h', pcm).tostring()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse a comma separated list of tag = value pairs from a string", "response": "def label_list_parser(self, url):\n \"\"\"\n Extracts comma separate tag=value pairs from a string\n Assumes all characters other than / and , are valid\n \"\"\"\n labels = re.findall('([^/,]+=[^/,]+)', url)\n slabels = set(labels)\n if '' in slabels:\n slabels.remove('')\n return slabels"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef init(app, register_blueprint=True, url_prefix='/fm', access_control_function=None,\n custom_config_json_path=None, custom_init_js_path=None):\n \"\"\"\n :param app: The Flask app\n :param register_blueprint: Override to False to stop the blueprint from automatically being registered to the\n app\n :param url_prefix: The URL prefix for the blueprint, defaults to /fm\n :param access_control_function: Pass in a function here to implement access control. The function will\n be called any time someone tries to access the filemanager, and a 404\n will be returned if this function returns False\n :param custom_config_json_path: Set this to the full path of you filemanager.config.json file if you want\n to use a custom config. Example:\n os.path.join(app.root_path, 'static/filemanager.config.json')\n :param custom_init_js_path: Set this to the full path of you filemanager.init.js file if you want\n to use a custom init.js. Example:\n os.path.join(app.root_path, 'static/filemanager.init.js')\n \"\"\"\n\n global _initialised, _FILE_PATH, _URL_PREFIX\n\n if _initialised:\n raise Exception('Flask Filemanager can only be registered once!')\n\n _initialised = True\n\n _FILE_PATH = app.config.get('FLASKFILEMANAGER_FILE_PATH')\n if not _FILE_PATH:\n raise Exception('No FLASKFILEMANAGER_FILE_PATH value in config')\n \n log.info('File Manager Using file path: {}'.format(_FILE_PATH))\n util.ensure_dir(_FILE_PATH)\n \n if access_control_function:\n set_access_control_function(access_control_function)\n\n if custom_config_json_path:\n set_custom_config_json_path(custom_config_json_path)\n log.info('File Manager using custom config.json path: {}'.format(custom_config_json_path))\n\n if custom_init_js_path:\n set_custom_init_js_path(custom_init_js_path)\n log.info('File Manager using custom init.js path: {}'.format(custom_init_js_path))\n\n if register_blueprint:\n log.info('Registering filemanager blueprint to {}'.format(url_prefix))\n app.register_blueprint(filemanager_blueprint, url_prefix=url_prefix)", "response": ":param app: The Flask app\n :param register_blueprint: Override to False to stop the blueprint from automatically being registered to the\n app\n :param url_prefix: The URL prefix for the blueprint, defaults to /fm\n :param access_control_function: Pass in a function here to implement access control. The function will\n be called any time someone tries to access the filemanager, and a 404\n will be returned if this function returns False\n :param custom_config_json_path: Set this to the full path of you filemanager.config.json file if you want\n to use a custom config. Example:\n os.path.join(app.root_path, 'static/filemanager.config.json')\n :param custom_init_js_path: Set this to the full path of you filemanager.init.js file if you want\n to use a custom init.js. Example:\n os.path.join(app.root_path, 'static/filemanager.init.js')"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets file from path or content.", "response": "def get_file(path=None, content=None):\n \"\"\"\n :param path: relative path, or None to get from request\n :param content: file content, output in data. Used for editfile\n \"\"\"\n if path is None:\n path = request.args.get('path')\n\n if path is None:\n return error('No path in request')\n \n filename = os.path.split(path.rstrip('/'))[-1]\n extension = filename.rsplit('.', 1)[-1]\n os_file_path = web_path_to_os_path(path)\n\n if os.path.isdir(os_file_path):\n file_type = 'folder'\n # Ensure trailing slash\n if path[-1] != '/':\n path += '/'\n else:\n file_type = 'file'\n\n ctime = int(os.path.getctime(os_file_path))\n mtime = int(os.path.getmtime(os_file_path))\n\n height = 0\n width = 0\n if extension in ['gif', 'jpg', 'jpeg', 'png']:\n try:\n im = PIL.Image.open(os_file_path)\n height, width = im.size\n except OSError:\n log.exception('Error loading image \"{}\" to get width and height'.format(os_file_path))\n \n attributes = {\n 'name': filename,\n 'path': get_url_path(path),\n 'readable': 1 if os.access(os_file_path, os.R_OK) else 0,\n 'writeable': 1 if os.access(os_file_path, os.W_OK) else 0,\n 'created': datetime.datetime.fromtimestamp(ctime).ctime(),\n 'modified': datetime.datetime.fromtimestamp(mtime).ctime(),\n 'timestamp': mtime,\n 'width': width,\n 'height': height,\n 'size': os.path.getsize(os_file_path)\n }\n\n if content:\n attributes['content'] = content\n\n return {\n 'id': path,\n 'type': file_type,\n 'attributes': attributes\n }"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __get_charset(self):\r\n '''Return the character encoding (charset) used internally by MeCab.\r\n\r\n Charset is that of the system dictionary used by MeCab. Will defer to\r\n the user-specified MECAB_CHARSET environment variable, if set.\r\n\r\n Defaults to shift-jis on Windows.\r\n Defaults to utf-8 on Mac OS.\r\n Defaults to euc-jp, as per MeCab documentation, when all else fails.\r\n\r\n Returns:\r\n Character encoding (charset) used by MeCab.\r\n '''\r\n cset = os.getenv(self.MECAB_CHARSET)\r\n if cset:\r\n logger.debug(self._DEBUG_CSET_DEFAULT.format(cset))\r\n return cset\r\n else:\r\n try:\r\n res = Popen(['mecab', '-D'], stdout=PIPE).communicate()\r\n lines = res[0].decode()\r\n if not lines.startswith('unrecognized'):\r\n dicinfo = lines.split(os.linesep)\r\n t = [t for t in dicinfo if t.startswith('charset')]\r\n if len(t) > 0:\r\n cset = t[0].split()[1].lower()\r\n logger.debug(self._DEBUG_CSET_DEFAULT.format(cset))\r\n return cset\r\n else:\r\n logger.error('{}\\n'.format(self._ERROR_NODIC))\r\n raise EnvironmentError(self._ERROR_NODIC)\r\n else:\r\n logger.error('{}\\n'.format(self._ERROR_NOCMD))\r\n raise EnvironmentError(self._ERROR_NOCMD)\r\n except OSError:\r\n cset = 'euc-jp'\r\n if sys.platform == 'win32':\r\n cset = 'shift-jis'\r\n elif sys.platform == 'darwin':\r\n cset = 'utf8'\r\n logger.debug(self._DEBUG_CSET_DEFAULT.format(cset))\r\n return cset", "response": "Returns the character encoding used internally by MeCab."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __get_libpath(self):\r\n '''Return the absolute path to the MeCab library.\r\n\r\n On Windows, the path to the system dictionary is used to deduce the\r\n path to libmecab.dll.\r\n\r\n Otherwise, mecab-config is used find the libmecab shared object or\r\n dynamic library (*NIX or Mac OS, respectively).\r\n\r\n Will defer to the user-specified MECAB_PATH environment variable, if\r\n set.\r\n\r\n Returns:\r\n The absolute path to the MeCab library.\r\n\r\n Raises:\r\n EnvironmentError: A problem was encountered in trying to locate the\r\n MeCab library.\r\n '''\r\n libp = os.getenv(self.MECAB_PATH)\r\n if libp:\r\n return os.path.abspath(libp)\r\n else:\r\n plat = sys.platform\r\n if plat == 'win32':\r\n lib = self._LIBMECAB.format(self._WINLIB_EXT)\r\n\r\n try:\r\n v = self.__regkey_value(self._WINHKEY, self._WINVALUE)\r\n ldir = v.split('etc')[0]\r\n libp = os.path.join(ldir, 'bin', lib)\r\n except EnvironmentError as err:\r\n logger.error('{}\\n'.format(err))\r\n logger.error('{}\\n'.format(sys.exc_info()[0]))\r\n raise EnvironmentError(\r\n self._ERROR_WINREG.format(self._WINVALUE,\r\n self._WINHKEY))\r\n else:\r\n # UNIX-y OS?\r\n if plat == 'darwin':\r\n lib = self._LIBMECAB.format(self._MACLIB_EXT)\r\n else:\r\n lib = self._LIBMECAB.format(self._UNIXLIB_EXT)\r\n\r\n try:\r\n cmd = ['mecab-config', '--libs-only-L']\r\n res = Popen(cmd, stdout=PIPE).communicate()\r\n lines = res[0].decode()\r\n if not lines.startswith('unrecognized'):\r\n linfo = lines.strip()\r\n libp = os.path.join(linfo, lib)\r\n else:\r\n raise EnvironmentError(\r\n self._ERROR_MECABCONFIG.format(lib))\r\n except EnvironmentError as err:\r\n logger.error('{}\\n'.format(err))\r\n logger.error('{}\\n'.format(sys.exc_info()[0]))\r\n raise EnvironmentError(self._ERROR_NOLIB.format(lib))\r\n\r\n if libp and os.path.exists(libp):\r\n libp = os.path.abspath(libp)\r\n os.environ[self.MECAB_PATH] = libp\r\n return libp\r\n else:\r\n raise EnvironmentError(self._ERROR_NOLIB.format(libp))", "response": "Return the absolute path to the MeCab library."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __regkey_value(self, path, name='', start_key=None):\r\n r'''Return the data of value mecabrc at MeCab HKEY node.\r\n\r\n On Windows, the path to the mecabrc as set in the Windows Registry is\r\n used to deduce the path to libmecab.dll.\r\n\r\n Returns:\r\n The full path to the mecabrc on Windows.\r\n\r\n Raises:\r\n WindowsError: A problem was encountered in trying to locate the\r\n value mecabrc at HKEY_CURRENT_USER\\Software\\MeCab.\r\n '''\r\n if sys.version < '3':\r\n import _winreg as reg\r\n else:\r\n import winreg as reg\r\n\r\n def _fn(path, name='', start_key=None):\r\n if isinstance(path, str):\r\n path = path.split('\\\\')\r\n if start_key is None:\r\n start_key = getattr(reg, path[0])\r\n return _fn(path[1:], name, start_key)\r\n else:\r\n subkey = path.pop(0)\r\n with reg.OpenKey(start_key, subkey) as handle:\r\n if path:\r\n return _fn(path, name, handle)\r\n else:\r\n desc, i = None, 0\r\n while not desc or desc[0] != name:\r\n desc = reg.EnumValue(handle, i)\r\n i += 1\r\n return desc[1]\r\n return _fn(path, name, start_key)", "response": "r Returns the data of value mecabrc at the given path at the given name on Windows"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef diff(old_html, new_html, cutoff=0.0, plaintext=False, pretty=False):\n if plaintext:\n old_dom = parse_text(old_html)\n new_dom = parse_text(new_html)\n else:\n old_dom = parse_minidom(old_html)\n new_dom = parse_minidom(new_html)\n\n # If the two documents are not similar enough, don't show the changes.\n if not check_text_similarity(old_dom, new_dom, cutoff):\n return '

The differences from the previous version are too large to show concisely.

'\n\n dom = dom_diff(old_dom, new_dom)\n\n # HTML-specific cleanup.\n if not plaintext:\n fix_lists(dom)\n fix_tables(dom)\n\n # Only return html for the document body contents.\n body_elements = dom.getElementsByTagName('body')\n if len(body_elements) == 1:\n dom = body_elements[0]\n\n return minidom_tostring(dom, pretty=pretty)", "response": "Show the differences between two html documents as html."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef adjusted_ops(opcodes):\n while opcodes:\n op = opcodes.pop(0)\n tag, i1, i2, j1, j2 = op\n shift = 0\n if tag == 'equal':\n continue\n if tag == 'replace':\n # change the single replace op into a delete then insert\n # pay careful attention to the variables here, there's no typo\n opcodes = [\n ('delete', i1, i2, j1, j1),\n ('insert', i2, i2, j1, j2),\n ] + opcodes\n continue\n yield op\n if tag == 'delete':\n shift = -(i2 - i1)\n elif tag == 'insert':\n shift = +(j2 - j1)\n new_opcodes = []\n for tag, i1, i2, j1, j2 in opcodes:\n new_opcodes.append((\n tag,\n i1 + shift,\n i2 + shift,\n j1,\n j2,\n ))\n opcodes = new_opcodes", "response": "Iterate through opcodes and yield all the adjusted operations."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nyield index tuples for each place in the match.", "response": "def match_indices(match):\n \"\"\"Yield index tuples (old_index, new_index) for each place in the match.\"\"\"\n a, b, size = match\n for i in range(size):\n yield a + i, b + i"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nuses difflib to get the opcodes for a set of matching blocks.", "response": "def get_opcodes(matching_blocks):\n \"\"\"Use difflib to get the opcodes for a set of matching blocks.\"\"\"\n sm = difflib.SequenceMatcher(a=[], b=[])\n sm.matching_blocks = matching_blocks\n return sm.get_opcodes()"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef match_blocks(hash_func, old_children, new_children):\n sm = difflib.SequenceMatcher(\n _is_junk,\n a=[hash_func(c) for c in old_children],\n b=[hash_func(c) for c in new_children],\n )\n return sm", "response": "Use difflib to find matching blocks."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_nonmatching_blocks(matching_blocks):\n i = j = 0\n for match in matching_blocks:\n a, b, size = match\n yield (i, a, j, b)\n i = a + size\n j = b + size", "response": "Given a list of matching blocks output the gaps between them."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef merge_blocks(a_blocks, b_blocks):\n # Check sentinels for sequence length.\n assert a_blocks[-1][2] == b_blocks[-1][2] == 0 # sentinel size is 0\n assert a_blocks[-1] == b_blocks[-1]\n combined_blocks = sorted(list(set(a_blocks + b_blocks)))\n # Check for overlaps.\n i = j = 0\n for a, b, size in combined_blocks:\n assert i <= a\n assert j <= b\n i = a + size\n j = b + size\n return combined_blocks", "response": "Given two lists of blocks combine them in the proper order."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nremoves comments from the xml string.", "response": "def remove_comments(xml):\n \"\"\"\n Remove comments, as they can break the xml parser.\n\n See html5lib issue #122 ( http://code.google.com/p/html5lib/issues/detail?id=122 ).\n\n >>> remove_comments('')\n ''\n >>> remove_comments('')\n ''\n >>> remove_comments('

stuffstuff

')\n '

stuffstuff

'\n \"\"\"\n regex = re.compile(r'', re.DOTALL)\n return regex.sub('', xml)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nremove text nodes that are not inside the dom.", "response": "def remove_insignificant_text_nodes(dom):\n \"\"\"\n For html elements that should not have text nodes inside them, remove all\n whitespace. For elements that may have text, collapse multiple spaces to a\n single space.\n \"\"\"\n nodes_to_remove = []\n for node in walk_dom(dom):\n if is_text(node):\n text = node.nodeValue\n if node.parentNode.tagName in _non_text_node_tags:\n nodes_to_remove.append(node)\n else:\n node.nodeValue = re.sub(r'\\s+', ' ', text)\n for node in nodes_to_remove:\n remove_node(node)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the child at the given index.", "response": "def get_child(parent, child_index):\n \"\"\"\n Get the child at the given index, or return None if it doesn't exist.\n \"\"\"\n if child_index < 0 or child_index >= len(parent.childNodes):\n return None\n return parent.childNodes[child_index]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_location(dom, location):\n node = dom.documentElement\n for i in location:\n node = get_child(node, i)\n if not node:\n raise ValueError('Node at location %s does not exist.' % location) #TODO: line not covered\n return node", "response": "Get the node at the specified location in the specified dom."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef check_text_similarity(a_dom, b_dom, cutoff):\n a_words = list(tree_words(a_dom))\n b_words = list(tree_words(b_dom))\n\n sm = WordMatcher(a=a_words, b=b_words)\n if sm.text_ratio() >= cutoff:\n return True\n return False", "response": "Check whether two dom trees have similar text or not."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning all the significant text below the given node as a list of words.", "response": "def tree_words(node):\n \"\"\"Return all the significant text below the given node as a list of words.\n >>> list(tree_words(parse_minidom('

one

two
threefour
')))\n ['one', 'two', 'three', 'four']\n \"\"\"\n for word in split_text(tree_text(node)):\n word = word.strip()\n if word:\n yield word"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef tree_text(node):\n text = []\n for descendant in walk_dom(node):\n if is_text(descendant):\n text.append(descendant.nodeValue)\n return ' '.join(text)", "response": "Return the text of a node in a tree."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninserts the node before next_sibling.", "response": "def insert_or_append(parent, node, next_sibling):\n \"\"\"\n Insert the node before next_sibling. If next_sibling is None, append the node last instead.\n \"\"\"\n # simple insert\n if next_sibling:\n parent.insertBefore(node, next_sibling)\n else:\n parent.appendChild(node)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef wrap(node, tag):\n wrap_node = node.ownerDocument.createElement(tag)\n parent = node.parentNode\n if parent:\n parent.replaceChild(wrap_node, node)\n wrap_node.appendChild(node)\n return wrap_node", "response": "Wrap the given tag around a node."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwraps the given tag around the contents of a node.", "response": "def wrap_inner(node, tag):\n \"\"\"Wrap the given tag around the contents of a node.\"\"\"\n children = list(node.childNodes)\n wrap_node = node.ownerDocument.createElement(tag)\n for c in children:\n wrap_node.appendChild(c)\n node.appendChild(wrap_node)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef unwrap(node):\n for child in list(node.childNodes):\n node.parentNode.insertBefore(child, node)\n remove_node(node)", "response": "Removes a node replacing it with its children."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsplit the text by the regex and keep all parts.", "response": "def full_split(text, regex):\n \"\"\"\n Split the text by the regex, keeping all parts.\n The parts should re-join back into the original text.\n\n >>> list(full_split('word', re.compile('&.*?')))\n ['word']\n \"\"\"\n while text:\n m = regex.search(text)\n if not m:\n yield text\n break\n left = text[:m.start()]\n middle = text[m.start():m.end()]\n right = text[m.end():]\n if left:\n yield left\n if middle:\n yield middle\n text = right"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsplits the given text into pieces by the given regexes.", "response": "def multi_split(text, regexes):\n \"\"\"\n Split the text by the given regexes, in priority order.\n\n Make sure that the regex is parenthesized so that matches are returned in\n re.split().\n\n Splitting on a single regex works like normal split.\n >>> '|'.join(multi_split('one two three', [r'\\w+']))\n 'one| |two| |three'\n\n Splitting on digits first separates the digits from their word\n >>> '|'.join(multi_split('one234five 678', [r'\\d+', r'\\w+']))\n 'one|234|five| |678'\n\n Splitting on words first keeps the word with digits intact.\n >>> '|'.join(multi_split('one234five 678', [r'\\w+', r'\\d+']))\n 'one234five| |678'\n \"\"\"\n def make_regex(s):\n return re.compile(s) if isinstance(s, basestring) else s\n regexes = [make_regex(r) for r in regexes]\n\n # Run the list of pieces through the regex split, splitting it into more\n # pieces. Once a piece has been matched, add it to finished_pieces and\n # don't split it again. The pieces should always join back together to form\n # the original text.\n piece_list = [text]\n finished_pieces = set()\n def apply_re(regex, piece_list):\n for piece in piece_list:\n if piece in finished_pieces:\n yield piece\n continue\n for s in full_split(piece, regex):\n if regex.match(s):\n finished_pieces.add(s)\n if s:\n yield s\n\n for regex in regexes:\n piece_list = list(apply_re(regex, piece_list))\n assert ''.join(piece_list) == text\n return piece_list"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef text_ratio(self):\n return _calculate_ratio(\n self.match_length(),\n self._text_length(self.a) + self._text_length(self.b),\n )", "response": "Return a measure of the sequences s word similarity."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef match_length(self):\n length = 0\n for match in self.get_matching_blocks():\n a, b, size = match\n length += self._text_length(self.a[a:a+size])\n return length", "response": "Find the total length of all words that match between the two sequences."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun an xml edit script and return the new html produced.", "response": "def run_edit_script(self):\n \"\"\"\n Run an xml edit script, and return the new html produced.\n \"\"\"\n for action, location, properties in self.edit_script:\n if action == 'delete':\n node = get_location(self.dom, location)\n self.action_delete(node)\n elif action == 'insert':\n parent = get_location(self.dom, location[:-1])\n child_index = location[-1]\n self.action_insert(parent, child_index, **properties)\n return self.dom"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_changes_markup(dom, ins_nodes, del_nodes):\n # add markup for inserted and deleted sections\n for node in reversed(del_nodes):\n # diff algorithm deletes nodes in reverse order, so un-reverse the\n # order for this iteration\n insert_or_append(node.orig_parent, node, node.orig_next_sibling)\n wrap(node, 'del')\n for node in ins_nodes:\n wrap(node, 'ins')\n # Perform post-processing and cleanup.\n remove_nesting(dom, 'del')\n remove_nesting(dom, 'ins')\n sort_del_before_ins(dom)\n merge_adjacent(dom, 'del')\n merge_adjacent(dom, 'ins')", "response": "Add markup for inserted and deleted sections to the dom to show changes."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef remove_nesting(dom, tag_name):\n for node in dom.getElementsByTagName(tag_name):\n for ancestor in ancestors(node):\n if ancestor is node:\n continue\n if ancestor is dom.documentElement:\n break\n if ancestor.tagName == tag_name:\n unwrap(node)\n break", "response": "Unwrap items in the node list that have no ancestors with the same tag."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsort the nodes of the dom in - place based on a comparison function.", "response": "def sort_nodes(dom, cmp_func):\n \"\"\"\n Sort the nodes of the dom in-place, based on a comparison function.\n \"\"\"\n dom.normalize()\n for node in list(walk_dom(dom, elements_only=True)):\n prev_sib = node.previousSibling\n while prev_sib and cmp_func(prev_sib, node) == 1:\n node.parentNode.insertBefore(node, prev_sib)\n prev_sib = node.previousSibling"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nmerges all adjacent tags with the specified tag name.", "response": "def merge_adjacent(dom, tag_name):\n \"\"\"\n Merge all adjacent tags with the specified tag name.\n Return the number of merges performed.\n \"\"\"\n for node in dom.getElementsByTagName(tag_name):\n prev_sib = node.previousSibling\n if prev_sib and prev_sib.nodeName == node.tagName:\n for child in list(node.childNodes):\n prev_sib.appendChild(child)\n remove_node(node)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndistributes the given element to its children.", "response": "def distribute(node):\n \"\"\"\n Wrap a copy of the given element around the contents of each of its\n children, removing the node in the process.\n \"\"\"\n children = list(c for c in node.childNodes if is_element(c))\n unwrap(node)\n tag_name = node.tagName\n for c in children:\n wrap_inner(c, tag_name)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save(self, *args, **kwargs):\n self.__class__.objects.exclude(id=self.id).delete()\n super(SingletonModel, self).save(*args, **kwargs)", "response": "Save the object to the database."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the mappings from MAGICC to OpenSCM regions.", "response": "def get_magicc_region_to_openscm_region_mapping(inverse=False):\n \"\"\"Get the mappings from MAGICC to OpenSCM regions.\n\n This is not a pure inverse of the other way around. For example, we never provide\n \"GLOBAL\" as a MAGICC return value because it's unnecesarily confusing when we also\n have \"World\". Fortunately MAGICC doesn't ever read the name \"GLOBAL\" so this\n shouldn't matter.\n\n Parameters\n ----------\n inverse : bool\n If True, return the inverse mappings i.e. MAGICC to OpenSCM mappings\n Returns\n -------\n dict\n Dictionary of mappings\n \"\"\"\n\n def get_openscm_replacement(in_region):\n world = \"World\"\n if in_region in (\"WORLD\", \"GLOBAL\"):\n return world\n if in_region in (\"BUNKERS\"):\n return DATA_HIERARCHY_SEPARATOR.join([world, \"Bunkers\"])\n elif in_region.startswith((\"NH\", \"SH\")):\n in_region = in_region.replace(\"-\", \"\")\n hem = \"Northern Hemisphere\" if \"NH\" in in_region else \"Southern Hemisphere\"\n if in_region in (\"NH\", \"SH\"):\n return DATA_HIERARCHY_SEPARATOR.join([world, hem])\n\n land_ocean = \"Land\" if \"LAND\" in in_region else \"Ocean\"\n return DATA_HIERARCHY_SEPARATOR.join([world, hem, land_ocean])\n else:\n return DATA_HIERARCHY_SEPARATOR.join([world, in_region])\n\n # we generate the mapping dynamically, the first name in the list\n # is the one which will be used for inverse mappings\n _magicc_regions = [\n \"WORLD\",\n \"GLOBAL\",\n \"OECD90\",\n \"ALM\",\n \"REF\",\n \"ASIA\",\n \"R5ASIA\",\n \"R5OECD\",\n \"R5REF\",\n \"R5MAF\",\n \"R5LAM\",\n \"R6OECD90\",\n \"R6REF\",\n \"R6LAM\",\n \"R6MAF\",\n \"R6ASIA\",\n \"NHOCEAN\",\n \"SHOCEAN\",\n \"NHLAND\",\n \"SHLAND\",\n \"NH-OCEAN\",\n \"SH-OCEAN\",\n \"NH-LAND\",\n \"SH-LAND\",\n \"SH\",\n \"NH\",\n \"BUNKERS\",\n ]\n\n replacements = {}\n for magicc_region in _magicc_regions:\n openscm_region = get_openscm_replacement(magicc_region)\n # i.e. if we've already got a value for the inverse, we don't want to overwrite\n if (openscm_region in replacements.values()) and inverse:\n continue\n replacements[magicc_region] = openscm_region\n\n if inverse:\n return {v: k for k, v in replacements.items()}\n else:\n return replacements"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert MAGICC regions to OpenSCM regions", "response": "def convert_magicc_to_openscm_regions(regions, inverse=False):\n \"\"\"\n Convert MAGICC regions to OpenSCM regions\n\n Parameters\n ----------\n regions : list_like, str\n Regions to convert\n\n inverse : bool\n If True, convert the other way i.e. convert OpenSCM regions to MAGICC7\n regions\n\n Returns\n -------\n ``type(regions)``\n Set of converted regions\n \"\"\"\n if isinstance(regions, (list, pd.Index)):\n return [_apply_convert_magicc_to_openscm_regions(r, inverse) for r in regions]\n else:\n return _apply_convert_magicc_to_openscm_regions(regions, inverse)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_magicc7_to_openscm_variable_mapping(inverse=False):\n\n def get_openscm_replacement(in_var):\n if in_var.endswith(\"_INVERSE_EMIS\"):\n prefix = \"Inverse Emissions\"\n elif in_var.endswith(\"_EMIS\"):\n prefix = \"Emissions\"\n elif in_var.endswith(\"_CONC\"):\n prefix = \"Atmospheric Concentrations\"\n elif in_var.endswith(\"_RF\"):\n prefix = \"Radiative Forcing\"\n elif in_var.endswith(\"_OT\"):\n prefix = \"Optical Thickness\"\n else:\n raise ValueError(\"This shouldn't happen\")\n\n variable = in_var.split(\"_\")[0]\n # I hate edge cases\n if variable.endswith(\"EQ\"):\n variable = variable.replace(\"EQ\", \" Equivalent\")\n\n if \"GHG\" in variable:\n variable = variable.replace(\"GHG\", \"Greenhouse Gases\")\n\n if \"BIOMASSAER\" in variable:\n variable = variable.replace(\"BIOMASSAER\", \"Aerosols|MAGICC AFOLU\")\n\n if \"CO2CH4N2O\" in variable:\n variable = variable.replace(\"CO2CH4N2O\", \"CO2, CH4 and N2O\")\n\n aggregate_indicators = {\n \"KYOTO\": \"Kyoto Gases\",\n \"FGASSUM\": \"F Gases\",\n \"MHALOSUM\": \"Montreal Protocol Halogen Gases\",\n }\n for agg_indicator, long_name in aggregate_indicators.items():\n if variable.startswith(agg_indicator):\n stripped_var = variable.replace(agg_indicator, \"\")\n if stripped_var:\n variable = DATA_HIERARCHY_SEPARATOR.join([stripped_var, long_name])\n else:\n variable = long_name\n\n edge_case_B = variable.upper() in (\"HCFC141B\", \"HCFC142B\")\n if variable.endswith(\"I\"):\n variable = DATA_HIERARCHY_SEPARATOR.join(\n [variable[:-1], \"MAGICC Fossil and Industrial\"]\n )\n elif variable.endswith(\"B\") and not edge_case_B:\n variable = DATA_HIERARCHY_SEPARATOR.join([variable[:-1], \"MAGICC AFOLU\"])\n\n case_adjustments = {\n \"SOX\": \"SOx\",\n \"NOX\": \"NOx\",\n \"HFC134A\": \"HFC134a\",\n \"HFC143A\": \"HFC143a\",\n \"HFC152A\": \"HFC152a\",\n \"HFC227EA\": \"HFC227ea\",\n \"HFC236FA\": \"HFC236fa\",\n \"HFC245FA\": \"HFC245fa\",\n \"HFC365MFC\": \"HFC365mfc\",\n \"HCFC141B\": \"HCFC141b\",\n \"HCFC142B\": \"HCFC142b\",\n \"CH3CCL3\": \"CH3CCl3\",\n \"CCL4\": \"CCl4\",\n \"CH3CL\": \"CH3Cl\",\n \"CH2CL2\": \"CH2Cl2\",\n \"CHCL3\": \"CHCl3\",\n \"CH3BR\": \"CH3Br\",\n \"HALON1211\": \"Halon1211\",\n \"HALON1301\": \"Halon1301\",\n \"HALON2402\": \"Halon2402\",\n \"HALON1202\": \"Halon1202\",\n \"SOLAR\": \"Solar\",\n \"VOLCANIC\": \"Volcanic\",\n \"EXTRA\": \"Extra\",\n }\n variable = apply_string_substitutions(variable, case_adjustments)\n\n return DATA_HIERARCHY_SEPARATOR.join([prefix, variable])\n\n magicc7_suffixes = [\"_EMIS\", \"_CONC\", \"_RF\", \"_OT\", \"_INVERSE_EMIS\"]\n magicc7_base_vars = MAGICC7_EMISSIONS_UNITS.magicc_variable.tolist() + [\n \"SOLAR\",\n \"VOLCANIC\",\n \"CO2EQ\",\n \"KYOTOCO2EQ\",\n \"FGASSUMHFC134AEQ\",\n \"MHALOSUMCFC12EQ\",\n \"GHG\",\n \"KYOTOGHG\",\n \"FGASSUM\",\n \"MHALOSUM\",\n \"BIOMASSAER\",\n \"CO2CH4N2O\",\n \"EXTRA\",\n ]\n magicc7_vars = [\n base_var + suffix\n for base_var in magicc7_base_vars\n for suffix in magicc7_suffixes\n ]\n\n replacements = {m7v: get_openscm_replacement(m7v) for m7v in magicc7_vars}\n\n replacements.update(\n {\n \"SURFACE_TEMP\": \"Surface Temperature\",\n \"TOTAL_INCLVOLCANIC_RF\": \"Radiative Forcing\",\n \"VOLCANIC_ANNUAL_RF\": \"Radiative Forcing|Volcanic\",\n \"TOTAL_ANTHRO_RF\": \"Radiative Forcing|Anthropogenic\",\n \"TOTAER_DIR_RF\": \"Radiative Forcing|Aerosols|Direct Effect\",\n \"CLOUD_TOT_RF\": \"Radiative Forcing|Aerosols|Indirect Effect\",\n \"MINERALDUST_RF\": \"Radiative Forcing|Mineral Dust\",\n \"STRATOZ_RF\": \"Radiative Forcing|Stratospheric Ozone\",\n \"TROPOZ_RF\": \"Radiative Forcing|Tropospheric Ozone\",\n \"CH4OXSTRATH2O_RF\": \"Radiative Forcing|CH4 Oxidation Stratospheric H2O\", # what is this\n \"LANDUSE_RF\": \"Radiative Forcing|Land-use Change\",\n \"BCSNOW_RF\": \"Radiative Forcing|Black Carbon on Snow\",\n \"CO2PF_EMIS\": \"Land to Air Flux|CO2|MAGICC Permafrost\",\n # \"CH4PF_EMIS\": \"Land to Air Flux|CH4|MAGICC Permafrost\", # TODO: test and then add when needed\n }\n )\n\n agg_ocean_heat_top = \"Aggregated Ocean Heat Content\"\n heat_content_aggreg_depths = {\n \"HEATCONTENT_AGGREG_DEPTH{}\".format(i): \"{}{}Depth {}\".format(\n agg_ocean_heat_top, DATA_HIERARCHY_SEPARATOR, i\n )\n for i in range(1, 4)\n }\n replacements.update(heat_content_aggreg_depths)\n replacements.update({\"HEATCONTENT_AGGREG_TOTAL\": agg_ocean_heat_top})\n\n ocean_temp_layer = {\n \"OCEAN_TEMP_LAYER_{0:03d}\".format(i): \"Ocean Temperature{}Layer {}\".format(\n DATA_HIERARCHY_SEPARATOR, i\n )\n for i in range(1, 999)\n }\n replacements.update(ocean_temp_layer)\n\n if inverse:\n return {v: k for k, v in replacements.items()}\n else:\n return replacements", "response": "Get the mappings from MAGICC7 to OpenSCM variables."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef convert_magicc7_to_openscm_variables(variables, inverse=False):\n if isinstance(variables, (list, pd.Index)):\n return [\n _apply_convert_magicc7_to_openscm_variables(v, inverse) for v in variables\n ]\n else:\n return _apply_convert_magicc7_to_openscm_variables(variables, inverse)", "response": "Convert MAGICC7 variables to OpenSCM variables."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_magicc6_to_magicc7_variable_mapping(inverse=False):\n # we generate the mapping dynamically, the first name in the list\n # is the one which will be used for inverse mappings\n magicc6_simple_mapping_vars = [\n \"KYOTO-CO2EQ\",\n \"CO2I\",\n \"CO2B\",\n \"CH4\",\n \"N2O\",\n \"BC\",\n \"OC\",\n \"SOx\",\n \"NOx\",\n \"NMVOC\",\n \"CO\",\n \"SF6\",\n \"NH3\",\n \"CF4\",\n \"C2F6\",\n \"HFC4310\",\n \"HFC43-10\",\n \"HFC-43-10\",\n \"HFC4310\",\n \"HFC134a\",\n \"HFC143a\",\n \"HFC227ea\",\n \"CCl4\",\n \"CH3CCl3\",\n \"HFC245fa\",\n \"Halon 1211\",\n \"Halon 1202\",\n \"Halon 1301\",\n \"Halon 2402\",\n \"Halon1211\",\n \"Halon1202\",\n \"Halon1301\",\n \"Halon2402\",\n \"CH3Br\",\n \"CH3Cl\",\n \"C6F14\",\n ]\n\n magicc6_sometimes_hyphen_vars = [\n \"CFC-11\",\n \"CFC-12\",\n \"CFC-113\",\n \"CFC-114\",\n \"CFC-115\",\n \"HCFC-22\",\n \"HFC-23\",\n \"HFC-32\",\n \"HFC-125\",\n \"HFC-134a\",\n \"HFC-143a\",\n \"HCFC-141b\",\n \"HCFC-142b\",\n \"HFC-227ea\",\n \"HFC-245fa\",\n ]\n magicc6_sometimes_hyphen_vars = [\n v.replace(\"-\", \"\") for v in magicc6_sometimes_hyphen_vars\n ] + magicc6_sometimes_hyphen_vars\n\n magicc6_sometimes_underscore_vars = [\n \"HFC43_10\",\n \"CFC_11\",\n \"CFC_12\",\n \"CFC_113\",\n \"CFC_114\",\n \"CFC_115\",\n \"HCFC_22\",\n \"HCFC_141b\",\n \"HCFC_142b\",\n ]\n magicc6_sometimes_underscore_replacements = {\n v: v.replace(\"_\", \"\") for v in magicc6_sometimes_underscore_vars\n }\n\n special_case_replacements = {\n \"FossilCO2\": \"CO2I\",\n \"OtherCO2\": \"CO2B\",\n \"MCF\": \"CH3CCL3\",\n \"CARB_TET\": \"CCL4\",\n \"MHALOSUMCFC12EQ\": \"MHALOSUMCFC12EQ\", # special case to avoid confusion with MCF\n }\n\n one_way_replacements = {\"HFC-245ca\": \"HFC245FA\", \"HFC245ca\": \"HFC245FA\"}\n\n all_possible_magicc6_vars = (\n magicc6_simple_mapping_vars\n + magicc6_sometimes_hyphen_vars\n + magicc6_sometimes_underscore_vars\n + list(special_case_replacements.keys())\n + list(one_way_replacements.keys())\n )\n replacements = {}\n for m6v in all_possible_magicc6_vars:\n if m6v in special_case_replacements:\n replacements[m6v] = special_case_replacements[m6v]\n elif (\n m6v in magicc6_sometimes_underscore_vars and not inverse\n ): # underscores one way\n replacements[m6v] = magicc6_sometimes_underscore_replacements[m6v]\n elif (m6v in one_way_replacements) and not inverse:\n replacements[m6v] = one_way_replacements[m6v]\n else:\n m7v = m6v.replace(\"-\", \"\").replace(\" \", \"\").upper()\n # i.e. if we've already got a value for the inverse, we don't\n # want to overwrite it\n if (m7v in replacements.values()) and inverse:\n continue\n replacements[m6v] = m7v\n\n if inverse:\n return {v: k for k, v in replacements.items()}\n else:\n return replacements", "response": "Get the mappings from MAGICC6 to MAGICC7 variables."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef convert_magicc6_to_magicc7_variables(variables, inverse=False):\n if isinstance(variables, (list, pd.Index)):\n return [\n _apply_convert_magicc6_to_magicc7_variables(v, inverse) for v in variables\n ]\n else:\n return _apply_convert_magicc6_to_magicc7_variables(variables, inverse)", "response": "Convert MAGICC6 variables to MAGICC7 variables."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a dictionary mapping from Pint to Fortran safe units.", "response": "def get_pint_to_fortran_safe_units_mapping(inverse=False):\n \"\"\"Get the mappings from Pint to Fortran safe units.\n\n Fortran can't handle special characters like \"^\" or \"/\" in names, but we need\n these in Pint. Conversely, Pint stores variables with spaces by default e.g. \"Mt\n CO2 / yr\" but we don't want these in the input files as Fortran is likely to think\n the whitespace is a delimiter.\n\n Parameters\n ----------\n inverse : bool\n If True, return the inverse mappings i.e. Fortran safe to Pint mappings\n\n Returns\n -------\n dict\n Dictionary of mappings\n \"\"\"\n replacements = {\"^\": \"super\", \"/\": \"per\", \" \": \"\"}\n if inverse:\n replacements = {v: k for k, v in replacements.items()}\n # mapping nothing to something is obviously not going to work in the inverse\n # hence remove\n replacements.pop(\"\")\n\n return replacements"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts Pint units to Fortran safe units.", "response": "def convert_pint_to_fortran_safe_units(units, inverse=False):\n \"\"\"\n Convert Pint units to Fortran safe units\n\n Parameters\n ----------\n units : list_like, str\n Units to convert\n\n inverse : bool\n If True, convert the other way i.e. convert Fortran safe units to Pint units\n\n Returns\n -------\n ``type(units)``\n Set of converted units\n \"\"\"\n if inverse:\n return apply_string_substitutions(units, FORTRAN_SAFE_TO_PINT_UNITS_MAPPING)\n else:\n return apply_string_substitutions(units, PINT_TO_FORTRAN_SAFE_UNITS_MAPPING)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run_evaluate(self) -> None:\n result = None\n self.eval_error = False\n if self._needs_evaluation:\n result = self._schema.value.evaluate(self._evaluation_context)\n\n self.eval_error = result is None\n if self.eval_error:\n return\n\n # Only set the value if it conforms to the field type\n if not self._schema.is_type_of(result):\n try:\n result = self._schema.type_object(result)\n except Exception as err:\n logging.debug('{} in casting {} to {} for field {}. Error: {}'.format(\n type(err).__name__, result, self._schema.type,\n self._schema.fully_qualified_name, err))\n self.eval_error = True\n return\n\n try:\n result = self._schema.sanitize_object(result)\n except Exception as err:\n logging.debug('{} in sanitizing {} of type {} for field {}. Error: {}'.format(\n type(err).__name__, result, self._schema.type, self._schema.fully_qualified_name,\n err))\n self.eval_error = True\n return\n\n self.value = result", "response": "Runs the base evaluation of the value and the schema."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set(self, key: Any, value: Any) -> None:\n if key is not None:\n self[key] = value", "response": "Sets the value of a key to a supplied value."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef increment(self, key: Any, by: int = 1) -> None:\n if key is not None:\n self[key] = self.get(key, 0) + by", "response": "Increments the value of a key."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef insert(self, index: int, obj: Any) -> None:\n if obj is not None:\n super().insert(index, obj)", "response": "Inserts an item into the list as long as it is not None."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the THISFILE_DATTYPE and THISFILE_REGIONMODE flags for a given list of regions.", "response": "def get_dattype_regionmode(regions, scen7=False):\n \"\"\"\n Get the THISFILE_DATTYPE and THISFILE_REGIONMODE flags for a given region set.\n\n In all MAGICC input files, there are two flags: THISFILE_DATTYPE and\n THISFILE_REGIONMODE. These tell MAGICC how to read in a given input file. This\n function maps the regions which are in a given file to the value of these flags\n expected by MAGICC.\n\n Parameters\n ----------\n regions : list_like\n The regions to get THISFILE_DATTYPE and THISFILE_REGIONMODE flags for.\n\n scen7 : bool, optional\n Whether the file we are getting the flags for is a SCEN7 file or not.\n\n Returns\n -------\n dict\n Dictionary where the flags are the keys and the values are the value they\n should be set to for the given inputs.\n \"\"\"\n dattype_flag = \"THISFILE_DATTYPE\"\n regionmode_flag = \"THISFILE_REGIONMODE\"\n region_dattype_row = _get_dattype_regionmode_regions_row(regions, scen7=scen7)\n\n dattype = DATTYPE_REGIONMODE_REGIONS[dattype_flag.lower()][region_dattype_row].iloc[\n 0\n ]\n regionmode = DATTYPE_REGIONMODE_REGIONS[regionmode_flag.lower()][\n region_dattype_row\n ].iloc[0]\n\n return {dattype_flag: dattype, regionmode_flag: regionmode}"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the region order expected by MAGICC.", "response": "def get_region_order(regions, scen7=False):\n \"\"\"\n Get the region order expected by MAGICC.\n\n Parameters\n ----------\n regions : list_like\n The regions to get THISFILE_DATTYPE and THISFILE_REGIONMODE flags for.\n\n scen7 : bool, optional\n Whether the file we are getting the flags for is a SCEN7 file or not.\n\n Returns\n -------\n list\n Region order expected by MAGICC for the given region set.\n \"\"\"\n region_dattype_row = _get_dattype_regionmode_regions_row(regions, scen7=scen7)\n region_order = DATTYPE_REGIONMODE_REGIONS[\"regions\"][region_dattype_row].iloc[0]\n\n return region_order"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_special_scen_code(regions, emissions):\n if sorted(set(PART_OF_SCENFILE_WITH_EMISSIONS_CODE_0)) == sorted(set(emissions)):\n scenfile_emissions_code = 0\n elif sorted(set(PART_OF_SCENFILE_WITH_EMISSIONS_CODE_1)) == sorted(set(emissions)):\n scenfile_emissions_code = 1\n else:\n msg = \"Could not determine scen special code for emissions {}\".format(emissions)\n raise ValueError(msg)\n\n if set(regions) == set([\"WORLD\"]):\n scenfile_region_code = 1\n elif set(regions) == set([\"WORLD\", \"OECD90\", \"REF\", \"ASIA\", \"ALM\"]):\n scenfile_region_code = 2\n elif set(regions) == set([\"WORLD\", \"R5OECD\", \"R5REF\", \"R5ASIA\", \"R5MAF\", \"R5LAM\"]):\n scenfile_region_code = 3\n elif set(regions) == set(\n [\"WORLD\", \"R5OECD\", \"R5REF\", \"R5ASIA\", \"R5MAF\", \"R5LAM\", \"BUNKERS\"]\n ):\n scenfile_region_code = 4\n try:\n return scenfile_region_code * 10 + scenfile_emissions_code\n except NameError:\n msg = \"Could not determine scen special code for regions {}\".format(regions)\n raise ValueError(msg)", "response": "Get the special code for the given set of regions and emissions."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef determine_tool(filepath, tool_to_get):\n file_regexp_reader_writer = {\n \"SCEN\": {\"regexp\": r\"^.*\\.SCEN$\", \"reader\": _ScenReader, \"writer\": _ScenWriter},\n \"SCEN7\": {\n \"regexp\": r\"^.*\\.SCEN7$\",\n \"reader\": _Scen7Reader,\n \"writer\": _Scen7Writer,\n },\n \"prn\": {\"regexp\": r\"^.*\\.prn$\", \"reader\": _PrnReader, \"writer\": _PrnWriter},\n # \"Sector\": {\"regexp\": r\".*\\.SECTOR$\", \"reader\": _Scen7Reader, \"writer\": _Scen7Writer},\n \"EmisIn\": {\n \"regexp\": r\"^.*\\_EMIS.*\\.IN$\",\n \"reader\": _HistEmisInReader,\n \"writer\": _HistEmisInWriter,\n },\n \"ConcIn\": {\n \"regexp\": r\"^.*\\_CONC.*\\.IN$\",\n \"reader\": _ConcInReader,\n \"writer\": _ConcInWriter,\n },\n \"OpticalThicknessIn\": {\n \"regexp\": r\"^.*\\_OT\\.IN$\",\n \"reader\": _OpticalThicknessInReader,\n \"writer\": _OpticalThicknessInWriter,\n },\n \"RadiativeForcingIn\": {\n \"regexp\": r\"^.*\\_RF\\.(IN|MON)$\",\n \"reader\": _RadiativeForcingInReader,\n \"writer\": _RadiativeForcingInWriter,\n },\n \"SurfaceTemperatureIn\": {\n \"regexp\": r\"^.*SURFACE\\_TEMP\\.(IN|MON)$\",\n \"reader\": _SurfaceTemperatureInReader,\n \"writer\": _SurfaceTemperatureInWriter,\n },\n \"Out\": {\n \"regexp\": r\"^DAT\\_.*(?>> cfg = pull_cfg_from_parameters_out(magicc.metadata[\"parameters\"]) >>> cfg.write(\"/somewhere/else/ANOTHERNAME.cfg\")", "response": "def pull_cfg_from_parameters_out(parameters_out, namelist_to_read=\"nml_allcfgs\"):\n \"\"\"Pull out a single config set from a parameters_out namelist.\n\n This function returns a single file with the config that needs to be passed to\n MAGICC in order to do the same run as is represented by the values in\n ``parameters_out``.\n\n Parameters\n ----------\n parameters_out : dict, f90nml.Namelist\n The parameters to dump\n\n namelist_to_read : str\n The namelist to read from the file.\n\n Returns\n -------\n :obj:`f90nml.Namelist`\n An f90nml object with the cleaned, read out config.\n\n Examples\n --------\n >>> cfg = pull_cfg_from_parameters_out(magicc.metadata[\"parameters\"])\n >>> cfg.write(\"/somewhere/else/ANOTHERNAME.cfg\")\n \"\"\"\n single_cfg = Namelist({namelist_to_read: {}})\n for key, value in parameters_out[namelist_to_read].items():\n if \"file_tuning\" in key:\n single_cfg[namelist_to_read][key] = \"\"\n else:\n try:\n if isinstance(value, str):\n single_cfg[namelist_to_read][key] = value.strip(\" \\t\\n\\r\").replace(\n \"\\x00\", \"\"\n )\n elif isinstance(value, list):\n clean_list = [v.strip(\" \\t\\n\\r\").replace(\"\\x00\", \"\") for v in value]\n single_cfg[namelist_to_read][key] = [v for v in clean_list if v]\n else:\n assert isinstance(value, Number)\n single_cfg[namelist_to_read][key] = value\n except AttributeError:\n if isinstance(value, list):\n assert all([isinstance(v, Number) for v in value])\n single_cfg[namelist_to_read][key] = value\n else:\n raise AssertionError(\n \"Unexpected cause in out parameters conversion\"\n )\n\n return single_cfg"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\npulling out a single config set from a MAGICC PARAMETERS. OUT file.", "response": "def pull_cfg_from_parameters_out_file(\n parameters_out_file, namelist_to_read=\"nml_allcfgs\"\n):\n \"\"\"Pull out a single config set from a MAGICC ``PARAMETERS.OUT`` file.\n\n This function reads in the ``PARAMETERS.OUT`` file and returns a single file with\n the config that needs to be passed to MAGICC in order to do the same run as is\n represented by the values in ``PARAMETERS.OUT``.\n\n Parameters\n ----------\n parameters_out_file : str\n The ``PARAMETERS.OUT`` file to read\n\n namelist_to_read : str\n The namelist to read from the file.\n\n Returns\n -------\n :obj:`f90nml.Namelist`\n An f90nml object with the cleaned, read out config.\n\n Examples\n --------\n >>> cfg = pull_cfg_from_parameters_out_file(\"PARAMETERS.OUT\")\n >>> cfg.write(\"/somewhere/else/ANOTHERNAME.cfg\")\n \"\"\"\n parameters_out = read_cfg_file(parameters_out_file)\n return pull_cfg_from_parameters_out(\n parameters_out, namelist_to_read=namelist_to_read\n )"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_generic_rcp_name(inname):\n # TODO: move into OpenSCM\n mapping = {\n \"rcp26\": \"rcp26\",\n \"rcp3pd\": \"rcp26\",\n \"rcp45\": \"rcp45\",\n \"rcp6\": \"rcp60\",\n \"rcp60\": \"rcp60\",\n \"rcp85\": \"rcp85\",\n }\n try:\n return mapping[inname.lower()]\n except KeyError:\n error_msg = \"No generic name for input: {}\".format(inname)\n raise ValueError(error_msg)", "response": "Convert an RCP name into the generic Pymagicc RCP name."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\njoining two sets of time series into a single time series.", "response": "def join_timeseries(base, overwrite, join_linear=None):\n \"\"\"Join two sets of timeseries\n\n Parameters\n ----------\n base : :obj:`MAGICCData`, :obj:`pd.DataFrame`, filepath\n Base timeseries to use. If a filepath, the data will first be loaded from disk.\n\n overwrite : :obj:`MAGICCData`, :obj:`pd.DataFrame`, filepath\n Timeseries to join onto base. Any points which are in both `base` and\n `overwrite` will be taken from `overwrite`. If a filepath, the data will first\n be loaded from disk.\n\n join_linear : tuple of len(2)\n A list/array which specifies the period over which the two timeseries should\n be joined. The first element is the start time of the join period, the second\n element is the end time of the join period. In the join period (excluding the\n start and end times), output data will be a linear interpolation between (the\n annually interpolated) `base` and `overwrite` data. If None, no linear join\n will be done and any points in (the annually interpolated) `overwrite` data\n will simply overwrite any points in `base`.\n\n Returns\n -------\n :obj:`MAGICCData`\n The joint timeseries. The resulting data is linearly interpolated onto annual steps\n \"\"\"\n if join_linear is not None:\n if len(join_linear) != 2:\n raise ValueError(\"join_linear must have a length of 2\")\n\n if isinstance(base, str):\n base = MAGICCData(base)\n elif isinstance(base, MAGICCData):\n base = deepcopy(base)\n\n if isinstance(overwrite, str):\n overwrite = MAGICCData(overwrite)\n elif isinstance(overwrite, MAGICCData):\n overwrite = deepcopy(overwrite)\n\n result = _join_timeseries_mdata(base, overwrite, join_linear)\n\n return MAGICCData(result)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread a MAGICC. SCEN file and return the data and metadata.", "response": "def read_scen_file(\n filepath,\n columns={\n \"model\": [\"unspecified\"],\n \"scenario\": [\"unspecified\"],\n \"climate_model\": [\"unspecified\"],\n },\n **kwargs\n):\n \"\"\"\n Read a MAGICC .SCEN file.\n\n Parameters\n ----------\n filepath : str\n Filepath of the .SCEN file to read\n\n columns : dict\n Passed to ``__init__`` method of MAGICCData. See\n ``MAGICCData.__init__`` for details.\n\n kwargs\n Passed to ``__init__`` method of MAGICCData. See\n ``MAGICCData.__init__`` for details.\n\n Returns\n -------\n :obj:`pymagicc.io.MAGICCData`\n ``MAGICCData`` object containing the data and metadata.\n \"\"\"\n mdata = MAGICCData(filepath, columns=columns, **kwargs)\n\n return mdata"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_openscm_var_from_filepath(filepath):\n reader = determine_tool(filepath, \"reader\")(filepath)\n openscm_var = convert_magicc7_to_openscm_variables(\n convert_magicc6_to_magicc7_variables(reader._get_variable_from_filepath())\n )\n\n return openscm_var", "response": "Returns the OpenSCM variable implied by a filepath."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nfinds the start and end index of the embedded namelist.", "response": "def _find_nml(self):\n \"\"\"\n Find the start and end of the embedded namelist.\n\n Returns\n -------\n (int, int)\n start and end index for the namelist\n \"\"\"\n nml_start = None\n nml_end = None\n for i in range(len(self.lines)):\n if self.lines[i].strip().startswith(\"&\"):\n nml_start = i\n\n if self.lines[i].strip().startswith(\"/\"):\n nml_end = i\n assert (\n nml_start is not None and nml_end is not None\n ), \"Could not find namelist within {}\".format(self.filepath)\n return nml_end, nml_start"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nextracts the tabulated data from the input file and return a pandas. DataFrame and the metadata.", "response": "def process_data(self, stream, metadata):\n \"\"\"\n Extract the tabulated data from the input file.\n\n Parameters\n ----------\n stream : Streamlike object\n A Streamlike object (nominally StringIO)\n containing the table to be extracted\n metadata : dict\n Metadata read in from the header and the namelist\n\n Returns\n -------\n (pandas.DataFrame, dict)\n The first element contains the data, processed to the standard\n MAGICCData format.\n The second element is th updated metadata based on the processing performed.\n \"\"\"\n ch, metadata = self._get_column_headers_and_update_metadata(stream, metadata)\n df = self._convert_data_block_and_headers_to_df(stream)\n\n return df, metadata, ch"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert the data block and headers to a Pandas DataFrame.", "response": "def _convert_data_block_and_headers_to_df(self, stream):\n \"\"\"\n stream : Streamlike object\n A Streamlike object (nominally StringIO) containing the data to be\n extracted\n\n ch : dict\n Column headers to use for the output pd.DataFrame\n\n Returns\n -------\n :obj:`pd.DataFrame`\n Dataframe with processed datablock\n \"\"\"\n df = pd.read_csv(\n stream,\n skip_blank_lines=True,\n delim_whitespace=True,\n header=None,\n index_col=0,\n )\n\n if isinstance(df.index, pd.core.indexes.numeric.Float64Index):\n df.index = df.index.to_series().round(3)\n\n # reset the columns to be 0..n instead of starting at 1\n df.columns = list(range(len(df.columns)))\n\n return df"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndetermines the file variable from the filepath.", "response": "def _get_variable_from_filepath(self):\n \"\"\"\n Determine the file variable from the filepath.\n\n Returns\n -------\n str\n Best guess of variable name from the filepath\n \"\"\"\n try:\n return self.regexp_capture_variable.search(self.filepath).group(1)\n except AttributeError:\n self._raise_cannot_determine_variable_from_filepath_error()"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse the header for additional metadata.", "response": "def process_header(self, header):\n \"\"\"\n Parse the header for additional metadata.\n\n Parameters\n ----------\n header : str\n All the lines in the header.\n\n Returns\n -------\n dict\n The metadata in the header.\n \"\"\"\n metadata = {}\n for line in header.split(\"\\n\"):\n line = line.strip()\n for tag in self.header_tags:\n tag_text = \"{}:\".format(tag)\n if line.lower().startswith(tag_text):\n metadata[tag] = line[len(tag_text) + 1 :].strip()\n\n return metadata"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _read_data_header_line(self, stream, expected_header):\n pos = stream.tell()\n expected_header = (\n [expected_header] if isinstance(expected_header, str) else expected_header\n )\n for exp_hd in expected_header:\n tokens = stream.readline().split()\n try:\n assert tokens[0] == exp_hd\n return tokens[1:]\n except AssertionError:\n stream.seek(pos)\n continue\n\n assertion_msg = \"Expected a header token of {}, got {}\".format(\n expected_header, tokens[0]\n )\n raise AssertionError(assertion_msg)", "response": "Read a data header line and ensure that it starts with the expected header."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef read_chunk(self, t):\n size = self.data[self.pos : self.pos + 4].cast(\"i\")[0]\n d = self.data[self.pos + 4 : self.pos + 4 + size]\n\n assert (\n self.data[self.pos + 4 + size : self.pos + 4 + size + 4].cast(\"i\")[0]\n == size\n )\n self.pos = self.pos + 4 + size + 4\n\n res = np.array(d.cast(t))\n\n # Return as a scalar or a numpy array if it is an array\n if res.size == 1:\n return res[0]\n return res", "response": "Read out the next chunk of memory from the fortran binary stream."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef process_data(self, stream, metadata):\n index = np.arange(metadata[\"firstyear\"], metadata[\"lastyear\"] + 1)\n\n # The first variable is the global values\n globe = stream.read_chunk(\"d\")\n\n assert len(globe) == len(index)\n\n regions = stream.read_chunk(\"d\")\n num_regions = int(len(regions) / len(index))\n regions = regions.reshape((-1, num_regions), order=\"F\")\n\n data = np.concatenate((globe[:, np.newaxis], regions), axis=1)\n\n df = pd.DataFrame(data, index=index)\n\n if isinstance(df.index, pd.core.indexes.numeric.Float64Index):\n df.index = df.index.to_series().round(3)\n\n df.index.name = \"time\"\n\n regions = [\n \"World\",\n \"World|Northern Hemisphere|Ocean\",\n \"World|Northern Hemisphere|Land\",\n \"World|Southern Hemisphere|Ocean\",\n \"World|Southern Hemisphere|Land\",\n ]\n\n variable = convert_magicc6_to_magicc7_variables(\n self._get_variable_from_filepath()\n )\n variable = convert_magicc7_to_openscm_variables(variable)\n column_headers = {\n \"variable\": [variable] * (num_regions + 1),\n \"region\": regions,\n \"unit\": [\"unknown\"] * len(regions),\n \"todo\": [\"SET\"] * len(regions),\n }\n\n return df, metadata, self._set_column_defaults(column_headers)", "response": "Extracts the tabulated data from the input file and returns a Pandas DataFrame containing the data extracted to the standard MAGICCData format."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef process_header(self, data):\n metadata = {\n \"datacolumns\": data.read_chunk(\"I\"),\n \"firstyear\": data.read_chunk(\"I\"),\n \"lastyear\": data.read_chunk(\"I\"),\n \"annualsteps\": data.read_chunk(\"I\"),\n }\n if metadata[\"annualsteps\"] != 1:\n raise InvalidTemporalResError(\n \"{}: Only annual files can currently be processed\".format(self.filepath)\n )\n\n return metadata", "response": "Reads the first part of the file to get some essential metadata"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting a MAGICC input file from df and metadata", "response": "def write(self, magicc_input, filepath):\n \"\"\"\n Write a MAGICC input file from df and metadata\n\n Parameters\n ----------\n magicc_input : :obj:`pymagicc.io.MAGICCData`\n MAGICCData object which holds the data to write\n\n filepath : str\n Filepath of the file to write to.\n \"\"\"\n self._filepath = filepath\n # TODO: make copy attribute for MAGICCData\n self.minput = deepcopy(magicc_input)\n self.data_block = self._get_data_block()\n\n output = StringIO()\n\n output = self._write_header(output)\n output = self._write_namelist(output)\n output = self._write_datablock(output)\n\n with open(\n filepath, \"w\", encoding=\"utf-8\", newline=self._newline_char\n ) as output_file:\n output.seek(0)\n copyfileobj(output, output_file)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nappending any input which can be converted to MAGICCData to self.", "response": "def append(self, other, inplace=False, **kwargs):\n \"\"\"\n Append any input which can be converted to MAGICCData to self.\n\n Parameters\n ----------\n other : MAGICCData, pd.DataFrame, pd.Series, str\n Source of data to append.\n\n inplace : bool\n If True, append ``other`` inplace, otherwise return a new ``MAGICCData``\n instance.\n\n **kwargs\n Passed to ``MAGICCData`` constructor (only used if ``MAGICCData`` is not a\n ``MAGICCData`` instance).\n \"\"\"\n if not isinstance(other, MAGICCData):\n other = MAGICCData(other, **kwargs)\n\n if inplace:\n super().append(other, inplace=inplace)\n self.metadata.update(other.metadata)\n else:\n res = super().append(other, inplace=inplace)\n res.metadata = deepcopy(self.metadata)\n res.metadata.update(other.metadata)\n\n return res"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite an input file to disk.", "response": "def write(self, filepath, magicc_version):\n \"\"\"\n Write an input file to disk.\n\n Parameters\n ----------\n filepath : str\n Filepath of the file to write.\n\n magicc_version : int\n The MAGICC version for which we want to write files. MAGICC7 and MAGICC6\n namelists are incompatible hence we need to know which one we're writing\n for.\n \"\"\"\n writer = determine_tool(filepath, \"writer\")(magicc_version=magicc_version)\n writer.write(self, filepath)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef validate_python_identifier_attributes(fully_qualified_name: str, spec: Dict[str, Any],\n *attributes: str) -> List[InvalidIdentifierError]:\n \"\"\" Validates a set of attributes as identifiers in a spec \"\"\"\n errors: List[InvalidIdentifierError] = []\n\n checks: List[Tuple[Callable, InvalidIdentifierError.Reason]] = [\n (lambda x: x.startswith('_'), InvalidIdentifierError.Reason.STARTS_WITH_UNDERSCORE),\n (lambda x: x.startswith('run_'), InvalidIdentifierError.Reason.STARTS_WITH_RUN),\n (lambda x: not x.isidentifier(), InvalidIdentifierError.Reason.INVALID_PYTHON_IDENTIFIER),\n ]\n\n for attribute in attributes:\n if attribute not in spec or spec.get(ATTRIBUTE_INTERNAL, False):\n continue\n\n for check in checks:\n if check[0](spec[attribute]):\n errors.append(\n InvalidIdentifierError(fully_qualified_name, spec, attribute, check[1]))\n break\n\n return errors", "response": "Validates a set of attributes as identifiers in a resource tree."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nvalidating that a set of attributes are present in the resource tree.", "response": "def validate_required_attributes(fully_qualified_name: str, spec: Dict[str, Any],\n *attributes: str) -> List[RequiredAttributeError]:\n \"\"\" Validates to ensure that a set of attributes are present in spec \"\"\"\n return [\n RequiredAttributeError(fully_qualified_name, spec, attribute)\n for attribute in attributes\n if attribute not in spec\n ]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef validate_empty_attributes(fully_qualified_name: str, spec: Dict[str, Any],\n *attributes: str) -> List[EmptyAttributeError]:\n \"\"\" Validates to ensure that a set of attributes do not contain empty values \"\"\"\n return [\n EmptyAttributeError(fully_qualified_name, spec, attribute)\n for attribute in attributes\n if not spec.get(attribute, None)\n ]", "response": "Validates that a set of attributes do not contain empty values."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nvalidate that the value of the specified attribute is a number of the specified type and lies with the specified range.", "response": "def validate_number_attribute(\n fully_qualified_name: str,\n spec: Dict[str, Any],\n attribute: str,\n value_type: Union[Type[int], Type[float]] = int,\n minimum: Optional[Union[int, float]] = None,\n maximum: Optional[Union[int, float]] = None) -> Optional[InvalidNumberError]:\n \"\"\" Validates to ensure that the value is a number of the specified type, and lies with the specified range \"\"\"\n\n if attribute not in spec:\n return\n\n try:\n value = value_type(spec[attribute])\n if (minimum is not None and value < minimum) or (maximum is not None and value > maximum):\n raise None\n except:\n return InvalidNumberError(fully_qualified_name, spec, attribute, value_type, minimum,\n maximum)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nvalidating that the value of an attribute lies within a set of candidates.", "response": "def validate_enum_attribute(fully_qualified_name: str, spec: Dict[str, Any], attribute: str,\n candidates: Set[Union[str, int, float]]) -> Optional[InvalidValueError]:\n \"\"\" Validates to ensure that the value of an attribute lies within an allowed set of candidates \"\"\"\n\n if attribute not in spec:\n return\n\n if spec[attribute] not in candidates:\n return InvalidValueError(fully_qualified_name, spec, attribute, candidates)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse(key_string: str) -> 'Key':\n parts = key_string.split(Key.PARTITION)\n key_type = KeyType.DIMENSION\n if parts[3]:\n key_type = KeyType.TIMESTAMP\n return Key(key_type, parts[0], parts[1], parts[2].split(Key.DIMENSION_PARTITION)\n if parts[2] else [],\n parser.parse(parts[3]) if parts[3] else None)", "response": "Parses a flat key string and returns a Key object"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses a flat key string and returns a Key object", "response": "def parse_sort_key(identity: str, sort_key_string: str) -> 'Key':\n \"\"\" Parses a flat key string and returns a key \"\"\"\n parts = sort_key_string.split(Key.PARTITION)\n key_type = KeyType.DIMENSION\n if parts[2]:\n key_type = KeyType.TIMESTAMP\n return Key(key_type, identity, parts[0], parts[1].split(Key.DIMENSION_PARTITION)\n if parts[1] else [],\n parser.parse(parts[2]) if parts[2] else None)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking if this key starts with the other key. Returns False if this key starts with other key.", "response": "def starts_with(self, other: 'Key') -> bool:\n \"\"\"\n Checks if this key starts with the other key provided. Returns False if key_type, identity\n or group are different.\n For `KeyType.TIMESTAMP` returns True.\n For `KeyType.DIMENSION` does prefix match between the two dimensions property.\n \"\"\"\n if (self.key_type, self.identity, self.group) != (other.key_type, other.identity,\n other.group):\n return False\n if self.key_type == KeyType.TIMESTAMP:\n return True\n if self.key_type == KeyType.DIMENSION:\n if len(self.dimensions) < len(other.dimensions):\n return False\n return self.dimensions[0:len(other.dimensions)] == other.dimensions"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _evaluate_dimension_fields(self) -> bool:\n for _, item in self._dimension_fields.items():\n item.run_evaluate()\n if item.eval_error:\n return False\n return True", "response": "Evaluates the dimension fields. Returns True if any of the fields could not be evaluated."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _compare_dimensions_to_fields(self) -> bool:\n for name, item in self._dimension_fields.items():\n if item.value != self._nested_items[name].value:\n return False\n return True", "response": "Compares the dimension field values to the value in regular fields."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating the Key object based on dimension fields.", "response": "def _key(self):\n \"\"\" Generates the Key object based on dimension fields. \"\"\"\n return Key(self._schema.key_type, self._identity, self._name,\n [str(item.value) for item in self._dimension_fields.values()])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef run(scenario, magicc_version=6, **kwargs):\n if magicc_version == 6:\n magicc_cls = MAGICC6\n elif magicc_version == 7:\n magicc_cls = MAGICC7\n else:\n raise ValueError(\"MAGICC version {} is not available\".format(magicc_version))\n\n with magicc_cls() as magicc:\n results = magicc.run(scenario=scenario, **kwargs)\n\n return results", "response": "Run a MAGICC scenario and return the output data and config parameters."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef run_evaluate(self, block: TimeAggregate) -> bool:\n if self._anchor.evaluate_anchor(block, self._evaluation_context):\n\n try:\n self.run_reset()\n self._evaluation_context.global_add('anchor', block)\n self._evaluate()\n self._anchor.add_condition_met()\n return True\n finally:\n self._evaluation_context.global_remove('anchor')\n\n return False", "response": "Evaluates the anchor condition against the specified block."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef extend_schema_spec(self) -> None:\n super().extend_schema_spec()\n\n if self.ATTRIBUTE_FIELDS in self._spec:\n # Add new fields to the schema spec. Since `_identity` is added by the super, new elements are added after\n predefined_field = self._build_time_fields_spec(self._spec[self.ATTRIBUTE_NAME])\n self._spec[self.ATTRIBUTE_FIELDS][1:1] = predefined_field\n\n # Add new field schema to the schema loader\n for field_schema in predefined_field:\n self.schema_loader.add_schema_spec(field_schema, self.fully_qualified_name)", "response": "Extends the schema spec with the block start and end times."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconstruct the spec for predefined fields that are to be included in the master spec prior to schema load", "response": "def _build_time_fields_spec(name_in_context: str) -> List[Dict[str, Any]]:\n \"\"\"\n Constructs the spec for predefined fields that are to be included in the master spec prior to schema load\n :param name_in_context: Name of the current object in the context\n :return:\n \"\"\"\n return [\n {\n 'Name': '_start_time',\n 'Type': Type.DATETIME,\n 'Value': ('time if {aggregate}._start_time is None else time '\n 'if time < {aggregate}._start_time else {aggregate}._start_time'\n ).format(aggregate=name_in_context),\n ATTRIBUTE_INTERNAL: True\n },\n {\n 'Name': '_end_time',\n 'Type': Type.DATETIME,\n 'Value': ('time if {aggregate}._end_time is None else time '\n 'if time > {aggregate}._end_time else {aggregate}._end_time'\n ).format(aggregate=name_in_context),\n ATTRIBUTE_INTERNAL: True\n },\n ]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef apply_string_substitutions(\n inputs,\n substitutions,\n inverse=False,\n case_insensitive=False,\n unused_substitutions=\"ignore\",\n):\n \"\"\"Apply a number of substitutions to a string(s).\n\n The substitutions are applied effectively all at once. This means that conflicting\n substitutions don't interact. Where substitutions are conflicting, the one which\n is longer takes precedance. This is confusing so we recommend that you look at\n the examples.\n\n Parameters\n ----------\n inputs : str, list of str\n The string(s) to which we want to apply the substitutions.\n\n substitutions : dict\n The substitutions we wish to make. The keys are the strings we wish to\n substitute, the values are the strings which we want to appear in the output\n strings.\n\n inverse : bool\n If True, do the substitutions the other way around i.e. use the keys as the\n strings we want to appear in the output strings and the values as the strings\n we wish to substitute.\n\n case_insensitive : bool\n If True, the substitutions will be made in a case insensitive way.\n\n unused_substitutions : {\"ignore\", \"warn\", \"raise\"}, default ignore\n Behaviour when one or more of the inputs does not have a corresponding\n substitution. If \"ignore\", nothing happens. If \"warn\", a warning is issued. If\n \"raise\", an error is raised. See the examples.\n\n Returns\n -------\n ``type(input)``\n The input with substitutions performed.\n\n Examples\n --------\n >>> apply_string_substitutions(\"Hello JimBob\", {\"Jim\": \"Bob\"})\n 'Hello BobBob'\n\n >>> apply_string_substitutions(\"Hello JimBob\", {\"Jim\": \"Bob\"}, inverse=True)\n 'Hello JimJim'\n\n >>> apply_string_substitutions([\"Hello JimBob\", \"Jim says, 'Hi Bob'\"], {\"Jim\": \"Bob\"})\n ['Hello BobBob', \"Bob says, 'Hi Bob'\"]\n\n >>> apply_string_substitutions([\"Hello JimBob\", \"Jim says, 'Hi Bob'\"], {\"Jim\": \"Bob\"}, inverse=True)\n ['Hello JimJim', \"Jim says, 'Hi Jim'\"]\n\n >>> apply_string_substitutions(\"Muttons Butter\", {\"M\": \"B\", \"Button\": \"Zip\"})\n 'Buttons Butter'\n # Substitutions don't cascade. If they did, Muttons would become Buttons, then the\n # substitutions \"Button\" --> \"Zip\" would be applied and we would end up with\n # \"Zips Butter\".\n\n >>> apply_string_substitutions(\"Muttons Butter\", {\"Mutton\": \"Gutter\", \"tt\": \"zz\"})\n 'Gutters Buzzer'\n # Longer substitutions take precedent. Hence Mutton becomes Gutter, not Muzzon.\n\n >>> apply_string_substitutions(\"Butter\", {\"buTTer\": \"Gutter\"}, case_insensitive=True)\n 'Gutter'\n\n >>> apply_string_substitutions(\"Butter\", {\"teeth\": \"tooth\"})\n 'Butter'\n\n >>> apply_string_substitutions(\"Butter\", {\"teeth\": \"tooth\"}, unused_substitutions=\"ignore\")\n 'Butter'\n\n >>> apply_string_substitutions(\"Butter\", {\"teeth\": \"tooth\"}, unused_substitutions=\"warn\")\n ...pymagicc/utils.py:50: UserWarning: No substitution available for {'Butter'} warnings.warn(msg)\n 'Butter'\n\n >>> apply_string_substitutions(\"Butter\", {\"teeth\": \"tooth\"}, unused_substitutions=\"raise\")\n ValueError: No substitution available for {'Butter'}\n \"\"\"\n if inverse:\n substitutions = {v: k for k, v in substitutions.items()}\n\n # only possible to have conflicting substitutions when case insensitive\n if case_insensitive:\n _check_duplicate_substitutions(substitutions)\n\n if unused_substitutions != \"ignore\":\n _check_unused_substitutions(\n substitutions, inputs, unused_substitutions, case_insensitive\n )\n\n compiled_regexp = _compile_replacement_regexp(\n substitutions, case_insensitive=case_insensitive\n )\n\n inputs_return = deepcopy(inputs)\n if isinstance(inputs_return, str):\n inputs_return = _multiple_replace(inputs_return, substitutions, compiled_regexp)\n else:\n inputs_return = [\n _multiple_replace(v, substitutions, compiled_regexp) for v in inputs_return\n ]\n\n return inputs_return", "response": "Apply a number of substitutions to a string."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the list of items from the store based on the given time range.", "response": "def get_range(self,\n base_key: Key,\n start_time: datetime,\n end_time: datetime = None,\n count: int = 0) -> List[Tuple[Key, Any]]:\n \"\"\"\n Returns the list of items from the store based on the given time range or count.\n :param base_key: Items which don't start with the base_key are filtered out.\n :param start_time: Start time to for the range query\n :param end_time: End time of the range query. If None count is used.\n :param count: The number of items to be returned. Used if end_time is not specified.\n \"\"\"\n if end_time and count:\n raise ValueError('Only one of `end` or `count` can be set')\n\n if count:\n end_time = datetime.min.replace(\n tzinfo=timezone.utc) if count < 0 else datetime.max.replace(tzinfo=timezone.utc)\n\n end_time = self._add_timezone_if_required(end_time)\n start_time = self._add_timezone_if_required(start_time)\n\n if end_time < start_time:\n start_time, end_time = end_time, start_time\n\n if base_key.key_type == KeyType.TIMESTAMP:\n start_key = Key(KeyType.TIMESTAMP, base_key.identity, base_key.group, [], start_time)\n end_key = Key(KeyType.TIMESTAMP, base_key.identity, base_key.group, [], end_time)\n return self._get_range_timestamp_key(start_key, end_key, count)\n else:\n return self._get_range_dimension_key(base_key, start_time, end_time, count)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_range_timestamp_key(self, start: Key, end: Key,\n count: int = 0) -> List[Tuple[Key, Any]]:\n \"\"\"\n Returns the list of items from the store based on the given time range or count.\n\n This is used when the key being used is a TIMESTAMP key.\n \"\"\"\n raise NotImplementedError()", "response": "Returns the list of items from the store based on the given time range."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _get_range_dimension_key(self,\n base_key: Key,\n start_time: datetime,\n end_time: datetime,\n count: int = 0) -> List[Tuple[Key, Any]]:\n \"\"\"\n Returns the list of items from the store based on the given time range or count.\n\n This is used when the key being used is a DIMENSION key.\n \"\"\"\n raise NotImplementedError()", "response": "Returns the list of items from the store based on the given time range or count."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _restrict_items_to_count(items: List[Tuple[Key, Any]], count: int) -> List[Tuple[Key, Any]]:\n if abs(count) > len(items):\n count = Store._sign(count) * len(items)\n\n if count < 0:\n return items[count:]\n else:\n return items[:count]", "response": "Restricts items to count number if the number of items is larger than abs ( count."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbuild an expression object.", "response": "def build_expression(self, attribute: str) -> Optional[Expression]:\n \"\"\" Builds an expression object. Adds an error if expression creation has errors. \"\"\"\n\n expression_string = self._spec.get(attribute, None)\n if expression_string:\n try:\n return Expression(str(expression_string))\n except Exception as err:\n self.add_errors(\n InvalidExpressionError(self.fully_qualified_name, self._spec, attribute, err))\n\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_errors(self, *errors: Union[BaseSchemaError, List[BaseSchemaError]]) -> None:\n self.schema_loader.add_errors(*errors)", "response": "Adds errors to the error repository."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nvalidates that the schema contains a series of required attributes.", "response": "def validate_required_attributes(self, *attributes: str) -> None:\n \"\"\" Validates that the schema contains a series of required attributes \"\"\"\n self.add_errors(\n validate_required_attributes(self.fully_qualified_name, self._spec, *attributes))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nvalidates that the attribute contains a numeric value within boundaries.", "response": "def validate_number_attribute(self,\n attribute: str,\n value_type: Union[Type[int], Type[float]] = int,\n minimum: Optional[Union[int, float]] = None,\n maximum: Optional[Union[int, float]] = None) -> None:\n \"\"\" Validates that the attribute contains a numeric value within boundaries if specified \"\"\"\n self.add_errors(\n validate_number_attribute(self.fully_qualified_name, self._spec, attribute, value_type,\n minimum, maximum))"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nvalidate that the attribute value is among the candidates.", "response": "def validate_enum_attribute(self, attribute: str,\n candidates: Set[Union[str, int, float]]) -> None:\n \"\"\" Validates that the attribute value is among the candidates \"\"\"\n self.add_errors(\n validate_enum_attribute(self.fully_qualified_name, self._spec, attribute, candidates))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef validate_schema_spec(self) -> None:\n self.add_errors(\n validate_empty_attributes(self.fully_qualified_name, self._spec, *self._spec.keys()))\n self.add_errors(\n validate_python_identifier_attributes(self.fully_qualified_name, self._spec,\n self.ATTRIBUTE_NAME))", "response": "This method is called by the validation routines that are called by the base class to validate the schema specification."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _needs_evaluation(self) -> bool:\n return self._schema.when is None or self._schema.when.evaluate(self._evaluation_context)", "response": "Returns True when the schema needs evaluation"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nevaluating the current item and returns an evaluation result object containing the result and reasons why the evaluation failed", "response": "def run_evaluate(self, *args, **kwargs) -> None:\n \"\"\"\n Evaluates the current item\n :returns An evaluation result object containing the result, or reasons why\n evaluation failed\n \"\"\"\n if self._needs_evaluation:\n for _, item in self._nested_items.items():\n item.run_evaluate()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _snapshot(self) -> Dict[str, Any]:\n try:\n return {name: item._snapshot for name, item in self._nested_items.items()}\n except Exception as e:\n raise SnapshotError('Error while creating snapshot for {}'.format(self._name)) from e", "response": "Creates a snapshot of the current item."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run_restore(self, snapshot: Dict[Union[str, Key], Any]) -> 'BaseItemCollection':\n try:\n\n for name, snap in snapshot.items():\n if isinstance(name, Key):\n self._nested_items[name.group].run_restore(snap)\n else:\n self._nested_items[name].run_restore(snap)\n return self\n\n except Exception as e:\n raise SnapshotError('Error while restoring snapshot: {}'.format(self._snapshot)) from e", "response": "Restores the state of a collection from a snapshot"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\npreparing the window for the current window.", "response": "def _prepare_window(self, start_time: datetime) -> None:\n \"\"\"\n Prepares window if any is specified.\n :param start_time: The anchor block start_time from where the window\n should be generated.\n \"\"\"\n # evaluate window first which sets the correct window in the store\n store = self._schema.schema_loader.get_store(\n self._schema.source.store_schema.fully_qualified_name)\n if Type.is_type_equal(self._schema.window_type, Type.DAY) or Type.is_type_equal(\n self._schema.window_type, Type.HOUR):\n block_list = self._load_blocks(\n store.get_range(\n Key(self._schema.source.key_type, self._identity, self._schema.source.name),\n start_time, self._get_end_time(start_time)))\n else:\n block_list = self._load_blocks(\n store.get_range(\n Key(self._schema.source.key_type, self._identity, self._schema.source.name),\n start_time, None, self._schema.window_value))\n\n self._window_source = _WindowSource(block_list)\n self._validate_view()"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating the end time to be used for the store range query.", "response": "def _get_end_time(self, start_time: datetime) -> datetime:\n \"\"\"\n Generates the end time to be used for the store range query.\n :param start_time: Start time to use as an offset to calculate the end time\n based on the window type in the schema.\n :return:\n \"\"\"\n if Type.is_type_equal(self._schema.window_type, Type.DAY):\n return start_time + timedelta(days=self._schema.window_value)\n elif Type.is_type_equal(self._schema.window_type, Type.HOUR):\n return start_time + timedelta(hours=self._schema.window_value)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert [ ( Key block ) to [ BlockAggregate ]", "response": "def _load_blocks(self, blocks: List[Tuple[Key, Any]]) -> List[TimeAggregate]:\n \"\"\"\n Converts [(Key, block)] to [BlockAggregate]\n :param blocks: List of (Key, block) blocks.\n :return: List of BlockAggregate\n \"\"\"\n return [\n TypeLoader.load_item(self._schema.source.type)(self._schema.source, self._identity,\n EvaluationContext()).run_restore(block)\n for (_, block) in blocks\n ]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef execute_per_identity_records(\n self,\n identity: str,\n records: List[TimeAndRecord],\n old_state: Optional[Dict[Key, Any]] = None) -> Tuple[str, Tuple[Dict, List]]:\n \"\"\"\n Executes the streaming and window BTS on the given records. An option old state can provided\n which initializes the state for execution. This is useful for batch execution where the\n previous state is written out to storage and can be loaded for the next batch run.\n\n :param identity: Identity of the records.\n :param records: List of TimeAndRecord to be processed.\n :param old_state: Streaming BTS state dictionary from a previous execution.\n :return: Tuple[Identity, Tuple[Identity, Tuple[Streaming BTS state dictionary,\n List of window BTS output]].\n \"\"\"\n schema_loader = SchemaLoader()\n if records:\n records.sort(key=lambda x: x[0])\n else:\n records = []\n\n block_data = self._execute_stream_bts(records, identity, schema_loader, old_state)\n window_data = self._execute_window_bts(identity, schema_loader)\n\n return identity, (block_data, window_data)", "response": "Executes the streaming and window BTS on the given records."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_per_identity_records(self, events: Iterable, data_processor: DataProcessor\n ) -> Generator[Tuple[str, TimeAndRecord], None, None]:\n \"\"\"\n Uses the given iteratable events and the data processor convert the event into a list of\n Records along with its identity and time.\n :param events: iteratable events.\n :param data_processor: DataProcessor to process each event in events.\n :return: yields Tuple[Identity, TimeAndRecord] for all Records in events,\n \"\"\"\n schema_loader = SchemaLoader()\n stream_bts_name = schema_loader.add_schema_spec(self._stream_bts)\n stream_transformer_schema: StreamingTransformerSchema = schema_loader.get_schema_object(\n stream_bts_name)\n for event in events:\n try:\n for record in data_processor.process_data(event):\n try:\n id = stream_transformer_schema.get_identity(record)\n time = stream_transformer_schema.get_time(record)\n yield (id, (time, record))\n except Exception as err:\n logging.error('{} in parsing Record {}.'.format(err, record))\n except Exception as err:\n logging.error('{} in parsing Event {}.'.format(err, event))", "response": "Returns a generator that yields all Records in the given events and the time and record."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstrings representation of the object.", "response": "def to_string(self, hdr, other):\n \"\"\"String representation with additional information\"\"\"\n result = \"%s[%s,%s\" % (\n hdr, self.get_type(self.type), self.get_clazz(self.clazz))\n if self.unique:\n result += \"-unique,\"\n else:\n result += \",\"\n result += self.name\n if other is not None:\n result += \",%s]\" % (other)\n else:\n result += \"]\"\n return result"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn true if the question is answered by the record", "response": "def answered_by(self, rec):\n \"\"\"Returns true if the question is answered by the record\"\"\"\n return self.clazz == rec.clazz and \\\n (self.type == rec.type or self.type == _TYPE_ANY) and \\\n self.name == rec.name"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef reset_ttl(self, other):\n self.created = other.created\n self.ttl = other.ttl", "response": "Sets this record s TTL and created time to that of\n other record."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nstrings representation of the record with addtional information", "response": "def to_string(self, other):\n \"\"\"String representation with addtional information\"\"\"\n arg = \"%s/%s,%s\" % (\n self.ttl, self.get_remaining_ttl(current_time_millis()), other)\n return DNSEntry.to_string(self, \"record\", arg)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwrite the packet to the output stream.", "response": "def write(self, out):\n \"\"\"Used in constructing an outgoing packet\"\"\"\n out.write_string(self.address, len(self.address))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write(self, out):\n out.write_string(self.cpu, len(self.cpu))\n out.write_string(self.os, len(self.os))", "response": "Write the CPU and OS information to the packet out."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef write(self, out):\n out.write_string(self.text, len(self.text))", "response": "Write the text to the packet out."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_property(self, key, value):\n self.properties[key] = value\n self.sync_properties()", "response": "Set a property in the dict"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write(self, out):\n out.write_short(self.priority)\n out.write_short(self.weight)\n out.write_short(self.port)\n out.write_name(self.server)", "response": "Write the attributes of this object to the packet out."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read_header(self):\n format = '!HHHHHH'\n length = struct.calcsize(format)\n info = struct.unpack(format,\n self.data[self.offset:self.offset + length])\n self.offset += length\n\n self.id = info[0]\n self.flags = info[1]\n self.num_questions = info[2]\n self.num_answers = info[3]\n self.num_authorities = info[4]\n self.num_additionals = info[5]", "response": "Reads the header portion of the packet"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_questions(self):\n format = '!HH'\n length = struct.calcsize(format)\n for i in range(0, self.num_questions):\n name = self.read_name()\n info = struct.unpack(format,\n self.data[self.offset:self.offset + length])\n self.offset += length\n\n question = DNSQuestion(name, info[0], info[1])\n self.questions.append(question)", "response": "Reads questions section of packet"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef read_int(self):\n format = '!I'\n length = struct.calcsize(format)\n info = struct.unpack(format,\n self.data[self.offset:self.offset + length])\n self.offset += length\n return info[0]", "response": "Reads an integer from the packet"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread a character string from the packet", "response": "def read_character_string(self):\n \"\"\"Reads a character string from the packet\"\"\"\n length = ord(self.data[self.offset])\n self.offset += 1\n return self.read_string(length)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading a string from the packet", "response": "def read_string(self, len):\n \"\"\"Reads a string of a given length from the packet\"\"\"\n format = '!' + str(len) + 's'\n length = struct.calcsize(format)\n info = struct.unpack(format,\n self.data[self.offset:self.offset + length])\n self.offset += length\n return info[0]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_others(self):\n format = '!HHiH'\n length = struct.calcsize(format)\n n = self.num_answers + self.num_authorities + self.num_additionals\n for i in range(0, n):\n domain = self.read_name()\n info = struct.unpack(format,\n self.data[self.offset:self.offset + length])\n self.offset += length\n\n rec = None\n if info[0] == _TYPE_A:\n rec = DNSAddress(domain,\n info[0], info[1], info[2],\n self.read_string(4))\n elif info[0] == _TYPE_CNAME or info[0] == _TYPE_PTR:\n rec = DNSPointer(domain,\n info[0], info[1], info[2],\n self.read_name())\n elif info[0] == _TYPE_TXT:\n rec = DNSText(domain,\n info[0], info[1], info[2],\n self.read_string(info[3]))\n elif info[0] == _TYPE_SRV:\n rec = DNSService(domain,\n info[0], info[1], info[2],\n self.read_unsigned_short(),\n self.read_unsigned_short(),\n self.read_unsigned_short(),\n self.read_name())\n elif info[0] == _TYPE_HINFO:\n rec = DNSHinfo(domain,\n info[0], info[1], info[2],\n self.read_character_string(),\n self.read_character_string())\n elif info[0] == _TYPE_RRSIG:\n rec = DNSSignatureI(domain,\n info[0], info[1], info[2],\n self.read_string(18),\n self.read_name(),\n self.read_character_string())\n elif info[0] == _TYPE_AAAA:\n rec = DNSAddress(domain,\n info[0], info[1], info[2],\n self.read_string(16))\n else:\n # Try to ignore types we don't know about\n # this may mean the rest of the name is\n # unable to be parsed, and may show errors\n # so this is left for debugging. New types\n # encountered need to be parsed properly.\n #\n #print \"UNKNOWN TYPE = \" + str(info[0])\n #raise BadTypeInNameException\n pass\n\n if rec is not None:\n self.answers.append(rec)", "response": "Reads the answers authorities and additionals section of the packet"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nread a UTF - 8 string of a given length from the packet", "response": "def read_utf(self, offset, len):\n \"\"\"Reads a UTF-8 string of a given length from the packet\"\"\"\n try:\n result = self.data[offset:offset + len].decode('utf-8')\n except UnicodeDecodeError:\n result = str('')\n return result"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread a domain name from the packet.", "response": "def read_name(self):\n \"\"\"Reads a domain name from the packet\"\"\"\n result = ''\n off = self.offset\n next = -1\n first = off\n\n while 1:\n len = ord(self.data[off])\n off += 1\n if len == 0:\n break\n t = len & 0xC0\n if t == 0x00:\n result = ''.join((result, self.read_utf(off, len) + '.'))\n off += len\n elif t == 0xC0:\n if next < 0:\n next = off + 1\n off = ((len & 0x3F) << 8) | ord(self.data[off])\n if off >= first:\n raise Exception(\n \"Bad domain name (circular) at \" + str(off))\n first = off\n else:\n raise Exception(\"Bad domain name at \" + str(off))\n\n if next >= 0:\n self.offset = next\n else:\n self.offset = off\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_answer(self, inp, record):\n if not record.suppressed_by(inp):\n self.add_answer_at_time(record, 0)", "response": "Adds an answer to the list of answer records."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_answer_at_time(self, record, now):\n if record is not None:\n if now == 0 or not record.is_expired(now):\n self.answers.append((record, now))\n if record.rrsig is not None:\n self.answers.append((record.rrsig, now))", "response": "Adds an answer if does not expire by a certain time"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write_byte(self, value):\n format = '!B'\n self.data.append(struct.pack(format, value))\n self.size += 1", "response": "Writes a single byte to the packet"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninsert an unsigned short in a certain position in the packet", "response": "def insert_short(self, index, value):\n \"\"\"Inserts an unsigned short in a certain position in the packet\"\"\"\n format = '!H'\n self.data.insert(index, struct.pack(format, value))\n self.size += 2"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write_int(self, value):\n format = '!I'\n self.data.append(struct.pack(format, int(value)))\n self.size += 4", "response": "Writes an unsigned integer to the packet"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write_string(self, value, length):\n format = '!' + str(length) + 's'\n self.data.append(struct.pack(format, value))\n self.size += length", "response": "Writes a string to the packet"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef write_utf(self, s):\n utfstr = s.encode('utf-8')\n length = len(utfstr)\n if length > 64:\n raise NamePartTooLongException\n self.write_byte(length)\n self.write_string(utfstr, length)", "response": "Writes a UTF - 8 string to the packet"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites a domain name to the packet", "response": "def write_name(self, name):\n \"\"\"Writes a domain name to the packet\"\"\"\n\n try:\n # Find existing instance of this name in packet\n #\n index = self.names[name]\n except KeyError:\n # No record of this name already, so write it\n # out as normal, recording the location of the name\n # for future pointers to it.\n #\n self.names[name] = self.size\n parts = name.split('.')\n if parts[-1] == '':\n parts = parts[:-1]\n for part in parts:\n self.write_utf(part)\n self.write_byte(0)\n return\n\n # An index was found, so write a pointer to it\n #\n self.write_byte((index >> 8) | 0xC0)\n self.write_byte(index)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef write_question(self, question):\n self.write_name(question.name)\n self.write_short(question.type)\n self.write_short(question.clazz)", "response": "Writes a question to the packet"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite a record to the packet", "response": "def write_record(self, record, now):\n \"\"\"Writes a record (answer, authoritative answer, additional) to\n the packet\"\"\"\n self.write_name(record.name)\n self.write_short(record.type)\n if record.unique and self.multicast:\n self.write_short(record.clazz | _CLASS_UNIQUE)\n else:\n self.write_short(record.clazz)\n if now == 0:\n self.write_int(record.ttl)\n else:\n self.write_int(record.get_remaining_ttl(now))\n index = len(self.data)\n # Adjust size for the short we will write before this record\n #\n self.size += 2\n record.write(self)\n self.size -= 2\n\n length = len(b''.join(self.data[index:]))\n self.insert_short(index, length)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef packet(self):\n if not self.finished:\n self.finished = 1\n for question in self.questions:\n self.write_question(question)\n for answer, time in self.answers:\n self.write_record(answer, time)\n for authority in self.authorities:\n self.write_record(authority, 0)\n for additional in self.additionals:\n self.write_record(additional, 0)\n\n self.insert_short(0, len(self.additionals))\n self.insert_short(0, len(self.authorities))\n self.insert_short(0, len(self.answers))\n self.insert_short(0, len(self.questions))\n self.insert_short(0, self.flags)\n if self.multicast:\n self.insert_short(0, 0)\n else:\n self.insert_short(0, self.id)\n return b''.join(self.data)", "response": "Returns a string containing the packet s bytes\n ."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add(self, entry):\n if self.get(entry) is not None:\n return\n try:\n list = self.cache[entry.key]\n except:\n list = self.cache[entry.key] = []\n list.append(entry)", "response": "Adds an entry to the cache"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sign(self, entry, signer=None):\n if (self.get(entry) is not None):\n return\n if (entry.rrsig is None) and (self.private is not None):\n entry.rrsig = DNSSignatureS(entry.name,\n _TYPE_RRSIG, _CLASS_IN, entry, self.private, signer)\n self.add(entry)\n if (self.private is not None):\n self.add(entry.rrsig)", "response": "Adds and sign an entry"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nremoves an entry from the cache", "response": "def remove(self, entry):\n \"\"\"Removes an entry\"\"\"\n try:\n list = self.cache[entry.key]\n list.remove(entry)\n except:\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets an entry by key. Will return None if there is no matching entry.", "response": "def get(self, entry):\n \"\"\"Gets an entry by key. Will return None if there is no\n matching entry.\"\"\"\n try:\n list = self.cache[entry.key]\n return list[list.index(entry)]\n except:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_by_details(self, name, type, clazz):\n entry = DNSEntry(name, type, clazz)\n return self.get(entry)", "response": "Gets an entry by details. Will return None if there is no matching entry."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef entries(self):\n def add(x, y):\n return x + y\n try:\n return reduce(add, list(self.cache.values()))\n except:\n return []", "response": "Returns a list of all entries"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_record(self, zeroconf, now, record):\n if record.type == _TYPE_PTR and record.name == self.type:\n expired = record.is_expired(now)\n try:\n oldrecord = self.services[record.alias.lower()]\n if not expired:\n oldrecord.reset_ttl(record)\n else:\n del(self.services[record.alias.lower()])\n callback = lambda x: self.listener.remove_service(x,\n self.type, record.alias)\n self.list.append(callback)\n return\n except:\n if not expired:\n self.services[record.alias.lower()] = record\n callback = lambda x: self.listener.add_service(x,\n self.type, record.alias)\n self.list.append(callback)\n\n expires = record.get_expiration_time(75)\n if expires < self.next_time:\n self.next_time = expires", "response": "Callback invoked by Zeroconf when new information arrives."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_properties(self, properties):\n if isinstance(properties, dict):\n self.properties = properties\n self.sync_properties()\n else:\n self.text = properties", "response": "Sets properties and text of this info from a dictionary"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_text(self, text):\n self.text = text\n try:\n self.properties = text_to_dict(text)\n except:\n traceback.print_exc()\n self.properties = None", "response": "Sets properties and text given a text field"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nname accessor for the class name.", "response": "def get_name(self):\n \"\"\"Name accessor\"\"\"\n if self.type is not None and self.name.endswith(\".\" + self.type):\n return self.name[:len(self.name) - len(self.type) - 1]\n return self.name"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update_record(self, zeroconf, now, record):\n if record is not None and not record.is_expired(now):\n if record.type == _TYPE_A:\n if record.name == self.name:\n if not record.address in self.address:\n self.address.append(record.address)\n elif record.type == _TYPE_SRV:\n if record.name == self.name:\n self.server = record.server\n self.port = record.port\n self.weight = record.weight\n self.priority = record.priority\n self.address = []\n self.update_record(zeroconf, now,\n zeroconf.cache.get_by_details(self.server,\n _TYPE_A, _CLASS_IN))\n elif record.type == _TYPE_TXT:\n if record.name == self.name:\n self.set_text(record.text)", "response": "Updates service information from a DNS record."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef request(self, zeroconf, timeout):\n now = current_time_millis()\n delay = _LISTENER_TIME\n next = now + delay\n last = now + timeout\n result = 0\n try:\n zeroconf.add_listener(self,\n DNSQuestion(self.name, _TYPE_ANY, _CLASS_IN))\n while self.server is None or \\\n len(self.address) == 0 or \\\n self.text is None:\n if last <= now:\n return 0\n if next <= now:\n out = DNSOutgoing(_FLAGS_QR_QUERY)\n out.add_question(DNSQuestion(self.name,\n _TYPE_SRV, _CLASS_IN))\n out.add_answer_at_time(\n zeroconf.cache.get_by_details(self.name,\n _TYPE_SRV, _CLASS_IN), now)\n out.add_question(\n DNSQuestion(self.name, _TYPE_TXT, _CLASS_IN))\n out.add_answer_at_time(\n zeroconf.cache.get_by_details(self.name,\n _TYPE_TXT, _CLASS_IN), now)\n if self.server is not None:\n out.add_question(\n DNSQuestion(self.server, _TYPE_A, _CLASS_IN))\n out.add_answer_at_time(\n zeroconf.cache.get_by_details(self.server,\n _TYPE_A, _CLASS_IN), now)\n zeroconf.send(out)\n next = now + delay\n delay = delay * 2\n\n zeroconf.wait(min(next, last) - now)\n now = current_time_millis()\n result = 1\n finally:\n zeroconf.remove_listener(self)\n\n return result", "response": "Returns true if the service could be discovered on the zeroconf network and updates this object with details discovered."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wait(self, timeout):\n self.condition.acquire()\n self.condition.wait(timeout // 1000)\n self.condition.release()", "response": "Calling thread waits for a given number of milliseconds or\n until notified."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nnotifies all waiting threads", "response": "def notify_all(self):\n \"\"\"Notifies all waiting threads\"\"\"\n self.condition.acquire()\n # python 3.x\n try:\n self.condition.notify_all()\n except:\n self.condition.notifyAll()\n self.condition.release()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the service information for a particular name and type.", "response": "def get_service_info(self, type, name, timeout=3000):\n \"\"\"Returns network's service information for a particular\n name and type, or None if no service matches by the timeout,\n which defaults to 3 seconds.\"\"\"\n info = ServiceInfo(type, name)\n if info.request(self, timeout):\n return info\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_serviceListener(self, type, listener):\n self.remove_service_listener(listener)\n self.browsers.append(ServiceBrowser(self, type, listener))", "response": "Adds a listener for a particular service type."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef remove_service_listener(self, listener):\n for browser in self.browsers:\n if browser.listener == listener:\n browser.cancel()\n del(browser)", "response": "Removes a listener from the set that is currently listening."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef register_service(self, info):\n self.check_service(info)\n self.services[info.name.lower()] = info\n\n # zone transfer\n self.transfer_zone(info.type)\n self.announce_service(info.name)", "response": "Registers a service with a default TTL\n of 60 seconds."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck the network for a unique service name modifying the the passed in if it is not unique.", "response": "def check_service(self, info):\n \"\"\"Checks the network for a unique service name, modifying the\n ServiceInfo passed in if it is not unique.\"\"\"\n now = current_time_millis()\n next_time = now\n i = 0\n while i < 3:\n for record in self.cache.entries_with_name(info.type):\n if record.type == _TYPE_PTR and \\\n not record.is_expired(now) and \\\n record.alias == info.name:\n if (info.name.find('.') < 0):\n info.name = info.name + \".[\" + \\\n info.address + \\\n \":\" + info.port + \\\n \"].\" + info.type\n self.check_service(info)\n return\n raise NonUniqueNameException\n if now < next_time:\n self.wait(next_time - now)\n now = current_time_millis()\n continue\n out = DNSOutgoing(_FLAGS_QR_QUERY | _FLAGS_AA)\n self.debug = out\n out.add_question(\n DNSQuestion(info.type, _TYPE_PTR, _CLASS_IN))\n out.add_authorative_answer(\n DNSPointer(info.type,\n _TYPE_PTR, _CLASS_IN, info.ttl, info.name))\n self.send(out)\n i += 1\n next_time += _CHECK_TIME"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a listener to the list of listeners that have been added to the cache.", "response": "def add_listener(self, listener, question):\n \"\"\"Adds a listener for a given question. The listener will have\n its update_record method called when information is available to\n answer the question.\"\"\"\n now = current_time_millis()\n self.listeners.append(listener)\n if question is not None:\n for record in self.cache.entries_with_name(question.name):\n if question.answered_by(record) and not record.is_expired(now):\n listener.update_record(self, now, record)\n self.notify_all()"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nuses to notify listeners of new information that has updated a record.", "response": "def update_record(self, now, rec):\n \"\"\"Used to notify listeners of new information that has updated\n a record.\"\"\"\n for listener in self.listeners:\n listener.update_record(self, now, rec)\n self.notify_all()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ndealing with incoming response packets.", "response": "def handle_response(self, msg, address):\n \"\"\"Deal with incoming response packets. All answers\n are held in the cache, and listeners are notified.\"\"\"\n now = current_time_millis()\n\n sigs = []\n precache = []\n\n for record in msg.answers:\n if isinstance(record, DNSSignature):\n sigs.append(record)\n else:\n precache.append(record)\n\n for e in precache:\n for s in sigs:\n if self.verify(e, s):\n # print \"DNS: %s verified with %s\" % (e,s)\n\n if self.adaptive and e.type == _TYPE_A:\n if e.address == '\\x00\\x00\\x00\\x00':\n e.address = socket.inet_aton(address)\n\n if e in self.cache.entries():\n if e.is_expired(now):\n for i in self.hooks:\n try:\n i.remove(e)\n except:\n pass\n self.cache.remove(e)\n self.cache.remove(s)\n else:\n entry = self.cache.get(e)\n sig = self.cache.get(s)\n if (entry is not None) and (sig is not None):\n for i in self.hooks:\n try:\n i.update(e)\n except:\n pass\n entry.reset_ttl(e)\n sig.reset_ttl(s)\n else:\n e.rrsig = s\n self.cache.add(e)\n self.cache.add(s)\n for i in self.hooks:\n try:\n i.add(e)\n except:\n pass\n\n precache.remove(e)\n sigs.remove(s)\n self.update_record(now, record)\n\n if self.bypass:\n for e in precache:\n if e in self.cache.entries():\n if e.is_expired(now):\n for i in self.hooks:\n try:\n i.remove(e)\n except:\n pass\n self.cache.remove(e)\n else:\n entry = self.cache.get(e)\n if (entry is not None):\n for i in self.hooks:\n try:\n i.update(e)\n except:\n pass\n entry.reset_ttl(e)\n else:\n self.cache.add(e)\n for i in self.hooks:\n try:\n i.add(e)\n except:\n pass\n\n self.update_record(now, record)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndeal with incoming query packets.", "response": "def handle_query(self, msg, addr, port, orig):\n \"\"\"\n Deal with incoming query packets. Provides a response if\n possible.\n\n msg - message to process\n addr - dst addr\n port - dst port\n orig - originating address (for adaptive records)\n \"\"\"\n out = None\n\n # Support unicast client responses\n #\n if port != _MDNS_PORT:\n out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA, 0)\n for question in msg.questions:\n out.add_question(question)\n for question in msg.questions:\n if question.type == _TYPE_PTR:\n for service in self.services.values():\n if question.name == service.type:\n # FIXME: sometimes we just not in time filling cache\n answer = self.cache.get(\n DNSPointer(service.type,\n _TYPE_PTR, _CLASS_IN,\n service.ttl, service.name))\n if out is None and answer is not None:\n out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA)\n if answer:\n out.add_answer(msg, answer)\n if question.type == _TYPE_AXFR:\n if question.name in list(self.zones.keys()):\n if out is None:\n out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA)\n for i in self.zones[question.name].services.values():\n out.add_answer(msg, i)\n else:\n try:\n if out is None:\n out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA)\n\n service = self.services.get(question.name.lower(), None)\n try:\n rs = service.records\n except:\n rs = []\n\n # Answer A record queries for any service addresses we know\n if (question.type == _TYPE_A or \\\n question.type == _TYPE_ANY) \\\n and (_TYPE_A in rs):\n for service in self.services.values():\n if service.server == question.name.lower():\n for i in service.address:\n out.add_answer(msg, self.cache.get(\n DNSAddress(question.name,\n _TYPE_A, _CLASS_IN | _CLASS_UNIQUE,\n service.ttl, i)))\n\n if not service:\n continue\n\n if (question.type == _TYPE_SRV or \\\n question.type == _TYPE_ANY) and (_TYPE_SRV in rs):\n out.add_answer(msg, self.cache.get(\n DNSService(question.name,\n _TYPE_SRV, _CLASS_IN | _CLASS_UNIQUE,\n service.ttl, service.priority, service.weight,\n service.port, service.server)))\n if (question.type == _TYPE_TXT or \\\n question.type == _TYPE_ANY) and \\\n (_TYPE_TXT in rs):\n out.add_answer(msg, self.cache.get(\n DNSText(question.name,\n _TYPE_TXT, _CLASS_IN | _CLASS_UNIQUE,\n service.ttl, service.text)))\n if (question.type == _TYPE_SRV) and (_TYPE_SRV in rs):\n for i in service.address:\n out.add_additional_answer(self.cache.get(\n DNSAddress(service.server,\n _TYPE_A, _CLASS_IN | _CLASS_UNIQUE,\n service.ttl, i)))\n except:\n traceback.print_exc()\n\n if out is not None and out.answers:\n out.id = msg.id\n self.send(out, addr, port)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef send(self, out, addr=_MDNS_ADDR, port=_MDNS_PORT):\n # This is a quick test to see if we can parse the packets we generate\n #temp = DNSIncoming(out.packet())\n for i in self.intf.values():\n try:\n return i.sendto(out.packet(), 0, (addr, port))\n except:\n traceback.print_exc()\n # Ignore this, it may be a temporary loss of network connection\n return -1", "response": "Sends an outgoing packet."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef close(self):\n if globals()['_GLOBAL_DONE'] == 0:\n globals()['_GLOBAL_DONE'] = 1\n self.notify_all()\n self.engine.notify()\n self.unregister_all_services()\n for i in self.intf.values():\n try:\n # there are cases, when we start mDNS without network\n i.setsockopt(socket.SOL_IP, socket.IP_DROP_MEMBERSHIP,\n socket.inet_aton(_MDNS_ADDR) + \\\n socket.inet_aton('0.0.0.0'))\n except:\n pass\n i.close()", "response": "Ends the background threads and prevent this instance from servicing further queries."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute Blurr BTS with the given records.", "response": "def execute(self, identity_records: 'RDD', old_state_rdd: Optional['RDD'] = None) -> 'RDD':\n \"\"\"\n Executes Blurr BTS with the given records. old_state_rdd can be provided to load an older\n state from a previous run.\n\n :param identity_records: RDD of the form Tuple[Identity, List[TimeAndRecord]]\n :param old_state_rdd: A previous streaming BTS state RDD as Tuple[Identity, Streaming BTS\n State]\n :return: RDD[Identity, Tuple[Streaming BTS State, List of Window BTS output]]\n \"\"\"\n identity_records_with_state = identity_records\n if old_state_rdd:\n identity_records_with_state = identity_records.fullOuterJoin(old_state_rdd)\n return identity_records_with_state.map(lambda x: self._execute_per_identity_records(x))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads the data from the given json_files path and converts them into the Record s format for the given object.", "response": "def get_record_rdd_from_json_files(self,\n json_files: List[str],\n data_processor: DataProcessor = SimpleJsonDataProcessor(),\n spark_session: Optional['SparkSession'] = None) -> 'RDD':\n \"\"\"\n Reads the data from the given json_files path and converts them into the `Record`s format for\n processing. `data_processor` is used to process the per event data in those files to convert\n them into `Record`.\n\n :param json_files: List of json file paths. Regular Spark path wildcards are accepted.\n :param data_processor: `DataProcessor` to process each event in the json files.\n :param spark_session: `SparkSession` to use for execution. If None is provided then a basic\n `SparkSession` is created.\n :return: RDD containing Tuple[Identity, List[TimeAndRecord]] which can be used in\n `execute()`\n \"\"\"\n spark_context = get_spark_session(spark_session).sparkContext\n raw_records: 'RDD' = spark_context.union(\n [spark_context.textFile(file) for file in json_files])\n return raw_records.mapPartitions(\n lambda x: self.get_per_identity_records(x, data_processor)).groupByKey().mapValues(list)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert an RDD of raw events into the Record s format for processing.", "response": "def get_record_rdd_from_rdd(\n self,\n rdd: 'RDD',\n data_processor: DataProcessor = SimpleDictionaryDataProcessor(),\n ) -> 'RDD':\n \"\"\"\n Converts a RDD of raw events into the `Record`s format for processing. `data_processor` is\n used to process the per row data to convert them into `Record`.\n\n :param rdd: RDD containing the raw events.\n :param data_processor: `DataProcessor` to process each row in the given `rdd`.\n :return: RDD containing Tuple[Identity, List[TimeAndRecord]] which can be used in\n `execute()`\n \"\"\"\n return rdd.mapPartitions(\n lambda x: self.get_per_identity_records(x, data_processor)).groupByKey().mapValues(list)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite the data to disk.", "response": "def write_output_file(self,\n path: str,\n per_identity_data: 'RDD',\n spark_session: Optional['SparkSession'] = None) -> None:\n \"\"\"\n Basic helper function to persist data to disk.\n\n If window BTS was provided then the window BTS output to written in csv format, otherwise,\n the streaming BTS output is written in JSON format to the `path` provided\n\n :param path: Path where the output should be written.\n :param per_identity_data: Output of the `execute()` call.\n :param spark_session: `SparkSession` to use for execution. If None is provided then a basic\n `SparkSession` is created.\n :return:\n \"\"\"\n _spark_session_ = get_spark_session(spark_session)\n if not self._window_bts:\n per_identity_data.flatMap(\n lambda x: [json.dumps(data, cls=BlurrJSONEncoder) for data in x[1][0].items()]\n ).saveAsTextFile(path)\n else:\n # Convert to a DataFrame first so that the data can be saved as a CSV\n _spark_session_.createDataFrame(per_identity_data.flatMap(lambda x: x[1][1])).write.csv(\n path, header=True)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef print_output(self, per_identity_data: 'RDD') -> None:\n if not self._window_bts:\n data = per_identity_data.flatMap(\n lambda x: [json.dumps(data, cls=BlurrJSONEncoder) for data in x[1][0].items()])\n else:\n # Convert to a DataFrame first so that the data can be saved as a CSV\n data = per_identity_data.map(\n lambda x: json.dumps((x[0], x[1][1]), cls=BlurrJSONEncoder))\n for row in data.collect():\n print(row)", "response": "Basic helper function to write data to stdout."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfind an executable in the path.", "response": "def find_executable(executable, path=None):\n \"\"\"\n As distutils.spawn.find_executable, but on Windows, look up\n every extension declared in PATHEXT instead of just `.exe`\n \"\"\"\n if sys.platform != 'win32':\n return distutils.spawn.find_executable(executable, path)\n\n if path is None:\n path = os.environ['PATH']\n\n paths = path.split(os.pathsep)\n extensions = os.environ.get('PATHEXT', '.exe').split(os.pathsep)\n base, ext = os.path.splitext(executable)\n\n if not os.path.isfile(executable):\n for p in paths:\n for ext in extensions:\n f = os.path.join(p, base + ext)\n if os.path.isfile(f):\n return f\n return None\n else:\n return executable"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_environment_dict(overrides):\n result = os.environ.copy()\n result.update(overrides or {})\n return result", "response": "Create and return a copy of os. environ with the specified overrides"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get(self, server):\n if not isinstance(server, six.binary_type):\n server = server.encode('utf-8')\n data = self._execute('get', server)\n result = json.loads(data.decode('utf-8'))\n\n # docker-credential-pass will return an object for inexistent servers\n # whereas other helpers will exit with returncode != 0. For\n # consistency, if no significant data is returned,\n # raise CredentialsNotFound\n if result['Username'] == '' and result['Secret'] == '':\n raise errors.CredentialsNotFound(\n 'No matching credentials in {}'.format(self.program)\n )\n\n return result", "response": "Retrieve credentials for server."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nstore credentials for server. Raises a StoreError if an error occurs.", "response": "def store(self, server, username, secret):\n \"\"\" Store credentials for `server`. Raises a `StoreError` if an error\n occurs.\n \"\"\"\n data_input = json.dumps({\n 'ServerURL': server,\n 'Username': username,\n 'Secret': secret\n }).encode('utf-8')\n return self._execute('store', data_input)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef erase(self, server):\n if not isinstance(server, six.binary_type):\n server = server.encode('utf-8')\n self._execute('erase', server)", "response": "Erase the credentials for the specified server. Raises a StoreError if an error occurs."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nevaluates and returns the identity as specified in the schema.", "response": "def get_identity(self, record: Record) -> str:\n \"\"\"\n Evaluates and returns the identity as specified in the schema.\n :param record: Record which is used to determine the identity.\n :return: The evaluated identity\n :raises: IdentityError if identity cannot be determined.\n \"\"\"\n context = self.schema_context.context\n context.add_record(record)\n identity = self.identity.evaluate(context)\n if not identity:\n raise IdentityError('Could not determine identity using {}. Record is {}'.format(\n self.identity.code_string, record))\n context.remove_record()\n return identity"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nevaluating and updates data in the StreamingTransformer.", "response": "def run_evaluate(self, record: Record):\n \"\"\"\n Evaluates and updates data in the StreamingTransformer.\n :param record: The 'source' record used for the update.\n :raises: IdentityError if identity is different from the one used during\n initialization.\n \"\"\"\n record_identity = self._schema.get_identity(record)\n if self._identity != record_identity:\n raise IdentityError(\n 'Identity in transformer ({}) and new record ({}) do not match'.format(\n self._identity, record_identity))\n\n # Add source record and time to the global context\n self._evaluation_context.add_record(record)\n self._evaluation_context.global_add(\n 'time',\n DateTimeFieldSchema.sanitize_object(\n self._schema.time.evaluate(self._evaluation_context)))\n super().run_evaluate()\n\n # Cleanup source and time form the context\n self._evaluation_context.remove_record()\n self._evaluation_context.global_remove('time')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nextend the schema spec with the identity field.", "response": "def extend_schema_spec(self) -> None:\n \"\"\" Injects the identity field \"\"\"\n super().extend_schema_spec()\n\n identity_field = {\n 'Name': '_identity',\n 'Type': BtsType.STRING,\n 'Value': 'identity',\n ATTRIBUTE_INTERNAL: True\n }\n\n if self.ATTRIBUTE_FIELDS in self._spec:\n self._spec[self.ATTRIBUTE_FIELDS].insert(0, identity_field)\n self.schema_loader.add_schema_spec(identity_field, self.fully_qualified_name)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _persist(self) -> None:\n if self._store:\n self._store.save(self._key, self._snapshot)", "response": "Persist the current data to the persistent store."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_schema_spec(self, spec: Dict[str, Any],\n fully_qualified_parent_name: str = None) -> Optional[str]:\n \"\"\"\n Add a schema dictionary to the schema loader. The given schema is stored\n against fully_qualified_parent_name + ITEM_SEPARATOR('.') + schema.name.\n :param spec: Schema specification.\n :param fully_qualified_parent_name: Full qualified name of the parent.\n If None is passed then the schema is stored against the schema name.\n :return: The fully qualified name against which the spec is stored.\n None is returned if the given spec is not a dictionary or the spec does not\n contain a 'name' key.\n \"\"\"\n if not isinstance(spec, dict) or ATTRIBUTE_NAME not in spec:\n return None\n\n name = spec[ATTRIBUTE_NAME]\n fully_qualified_name = name if fully_qualified_parent_name is None else self.get_fully_qualified_name(\n fully_qualified_parent_name, name)\n\n # Ensure that basic validation for each spec part is done before it is added to spec cache\n if isinstance(spec, dict):\n self._error_cache.add(\n validate_required_attributes(fully_qualified_name, spec, ATTRIBUTE_NAME,\n ATTRIBUTE_TYPE))\n if ATTRIBUTE_TYPE in spec and not Type.contains(spec[ATTRIBUTE_TYPE]):\n self._error_cache.add(\n InvalidTypeError(fully_qualified_name, spec, ATTRIBUTE_TYPE,\n InvalidTypeError.Reason.TYPE_NOT_DEFINED))\n\n self._spec_cache[fully_qualified_name] = spec\n for key, val in spec.items():\n if isinstance(val, list):\n for item in val:\n self.add_schema_spec(item, fully_qualified_name)\n self.add_schema_spec(val, fully_qualified_name)\n\n return spec[ATTRIBUTE_NAME]", "response": "Adds a schema dictionary to the schema loader."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_errors(self, *errors: Union[BaseSchemaError, SchemaErrorCollection]) -> None:\n for error in errors:\n self._error_cache.add(error)", "response": "Adds errors to the error store for the schema."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_schema_object(self, fully_qualified_name: str) -> 'BaseSchema':\n\n if fully_qualified_name not in self._schema_cache:\n spec = self.get_schema_spec(fully_qualified_name)\n\n if spec:\n try:\n self._schema_cache[fully_qualified_name] = TypeLoader.load_schema(\n spec.get(ATTRIBUTE_TYPE, None))(fully_qualified_name, self)\n except TypeLoaderError as err:\n self.add_errors(\n InvalidTypeError(fully_qualified_name, spec, ATTRIBUTE_TYPE,\n InvalidTypeError.Reason.TYPE_NOT_LOADED,\n err.type_class_name))\n\n return self._schema_cache.get(fully_qualified_name, None)", "response": "Returns an initialized schema object for the given fully_qualified_name."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_store(self, fully_qualified_name: str) -> Optional['Store']:\n\n if fully_qualified_name not in self._store_cache:\n schema = self.get_schema_object(fully_qualified_name)\n if not schema:\n return None\n\n if Type.is_store_type(schema.type):\n self._store_cache[fully_qualified_name] = TypeLoader.load_item(schema.type)(schema)\n else:\n self.add_errors(\n InvalidTypeError(fully_qualified_name, {}, ATTRIBUTE_TYPE,\n InvalidTypeError.Reason.INCORRECT_BASE, schema.type,\n InvalidTypeError.BaseTypes.STORE))\n\n return self._store_cache.get(fully_qualified_name, None)", "response": "Returns an initialized store object for the given fully qualified name."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nusing to generate a schema object from the given fully qualified parent name and the given nested item name.", "response": "def get_nested_schema_object(self, fully_qualified_parent_name: str,\n nested_item_name: str) -> Optional['BaseSchema']:\n \"\"\"\n Used to generate a schema object from the given fully_qualified_parent_name\n and the nested_item_name.\n :param fully_qualified_parent_name: The fully qualified name of the parent.\n :param nested_item_name: The nested item name.\n :return: An initialized schema object of the nested item.\n \"\"\"\n return self.get_schema_object(\n self.get_fully_qualified_name(fully_qualified_parent_name, nested_item_name))"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the fully qualified name of the nested item.", "response": "def get_fully_qualified_name(fully_qualified_parent_name: str, nested_item_name: str) -> str:\n \"\"\"\n Returns the fully qualified name by combining the fully_qualified_parent_name\n and nested_item_name.\n :param fully_qualified_parent_name: The fully qualified name of the parent.\n :param nested_item_name: The nested item name.\n :return: The fully qualified name of the nested item.\n \"\"\"\n return fully_qualified_parent_name + SchemaLoader.ITEM_SEPARATOR + nested_item_name"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_schema_spec(self, fully_qualified_name: str) -> Dict[str, Any]:\n\n if fully_qualified_name not in self._spec_cache:\n self.add_errors(SpecNotFoundError(fully_qualified_name, {}))\n\n return self._spec_cache.get(fully_qualified_name, None)", "response": "Used to retrieve the specifications of the given schema from the given fully_qualified_name of the schema."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_schema_specs_of_type(self, *schema_types: Type) -> Dict[str, Dict[str, Any]]:\n\n return {\n fq_name: schema\n for fq_name, schema in self._spec_cache.items()\n if Type.is_type_in(schema.get(ATTRIBUTE_TYPE, ''), list(schema_types))\n }", "response": "Returns a list of fully qualified names and schema dictionary tuples for the given schema types."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef global_add(self, key: str, value: Any) -> None:\n self.global_context[key] = value", "response": "Adds a key and value to the global dictionary."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmerge the provided evaluation context to the current evaluation context.", "response": "def merge(self, evaluation_context: 'EvaluationContext') -> None:\n \"\"\"\n Merges the provided evaluation context to the current evaluation context.\n :param evaluation_context: Evaluation context to merge.\n \"\"\"\n self.global_context.merge(evaluation_context.global_context)\n self.local_context.merge(evaluation_context.local_context)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nevaluate the expression with the context provided.", "response": "def evaluate(self, evaluation_context: EvaluationContext) -> Any:\n \"\"\"\n Evaluates the expression with the context provided. If the execution\n results in failure, an ExpressionEvaluationException encapsulating the\n underlying exception is raised.\n :param evaluation_context: Global and local context dictionary to be passed for evaluation\n \"\"\"\n try:\n\n if self.type == ExpressionType.EVAL:\n return eval(self.code_object, evaluation_context.global_context,\n evaluation_context.local_context)\n\n elif self.type == ExpressionType.EXEC:\n return exec(self.code_object, evaluation_context.global_context,\n evaluation_context.local_context)\n\n except Exception as err:\n # Evaluation exceptions are expected because of missing fields in the source 'Record'.\n logging.debug('{} in evaluating expression {}. Error: {}'.format(\n type(err).__name__, self.code_string, err))\n # These should result in an exception being raised:\n # NameError - Exceptions thrown because of using names in the expression which are not\n # present in EvaluationContext. A common cause for this is typos in the BTS.\n # MissingAttributeError - Exception thrown when a BTS nested item is used which does not\n # exist. Should only happen for erroneous BTSs.\n # ImportError - Thrown when there is a failure in importing other modules.\n if isinstance(err, (NameError, MissingAttributeError, ImportError)):\n raise err\n return None"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncopying all the files in source directory to target.", "response": "def _copy_files(source, target):\n \"\"\"\n Copy all the files in source directory to target.\n\n Ignores subdirectories.\n \"\"\"\n source_files = listdir(source)\n if not exists(target):\n makedirs(target)\n for filename in source_files:\n full_filename = join(source, filename)\n if isfile(full_filename):\n shutil.copy(full_filename, target)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_copy(self):\n if self.executable is None or not isfile(self.executable):\n raise FileNotFoundError(\n \"Could not find MAGICC{} executable: {}\".format(\n self.version, self.executable\n )\n )\n if self.is_temp:\n assert (\n self.root_dir is None\n ), \"A temp copy for this instance has already been created\"\n self.root_dir = mkdtemp(prefix=\"pymagicc-\")\n\n if exists(self.run_dir):\n raise Exception(\"A copy of MAGICC has already been created.\")\n if not exists(self.root_dir):\n makedirs(self.root_dir)\n\n exec_dir = basename(self.original_dir)\n\n # Copy a subset of folders from the MAGICC `original_dir`\n # Also copy anything which is in the root of the MAGICC distribution\n # Assumes that the MAGICC binary is in a folder one level below the root\n # of the MAGICC distribution. i.e. /run/magicc.exe or /bin/magicc\n dirs_to_copy = [\".\", \"bin\", \"run\"]\n # Check that the executable is in a valid sub directory\n assert exec_dir in dirs_to_copy, \"binary must be in bin/ or run/ directory\"\n\n for d in dirs_to_copy:\n source_dir = abspath(join(self.original_dir, \"..\", d))\n if exists(source_dir):\n _copy_files(source_dir, join(self.root_dir, d))\n\n # Create an empty out dir\n # MAGICC assumes that the 'out' directory already exists\n makedirs(join(self.root_dir, \"out\"))\n\n # Create basic configuration files so magicc can run\n self.set_years()\n self.set_config()", "response": "Creates a temporary directory structure and copy of MAGICC configuration files and binary."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef run(self, scenario=None, only=None, **kwargs):\n if not exists(self.root_dir):\n raise FileNotFoundError(self.root_dir)\n\n if self.executable is None:\n raise ValueError(\n \"MAGICC executable not found, try setting an environment variable `MAGICC_EXECUTABLE_{}=/path/to/binary`\".format(\n self.version\n )\n )\n\n if scenario is not None:\n kwargs = self.set_emission_scenario_setup(scenario, kwargs)\n\n yr_config = {}\n if \"startyear\" in kwargs:\n yr_config[\"startyear\"] = kwargs.pop(\"startyear\")\n if \"endyear\" in kwargs:\n yr_config[\"endyear\"] = kwargs.pop(\"endyear\")\n if yr_config:\n self.set_years(**yr_config)\n\n # should be able to do some other nice metadata stuff re how magicc was run\n # etc. here\n kwargs.setdefault(\"rundate\", get_date_time_string())\n\n self.update_config(**kwargs)\n\n self.check_config()\n\n exec_dir = basename(self.original_dir)\n command = [join(self.root_dir, exec_dir, self.binary_name)]\n\n if not IS_WINDOWS and self.binary_name.endswith(\".exe\"): # pragma: no cover\n command.insert(0, \"wine\")\n\n # On Windows shell=True is required.\n subprocess.check_call(command, cwd=self.run_dir, shell=IS_WINDOWS)\n\n outfiles = self._get_output_filenames()\n\n read_cols = {\"climate_model\": [\"MAGICC{}\".format(self.version)]}\n if scenario is not None:\n read_cols[\"model\"] = scenario[\"model\"].unique().tolist()\n read_cols[\"scenario\"] = scenario[\"scenario\"].unique().tolist()\n else:\n read_cols.setdefault(\"model\", [\"unspecified\"])\n read_cols.setdefault(\"scenario\", [\"unspecified\"])\n\n mdata = None\n for filepath in outfiles:\n try:\n openscm_var = _get_openscm_var_from_filepath(filepath)\n if only is None or openscm_var in only:\n tempdata = MAGICCData(\n join(self.out_dir, filepath), columns=deepcopy(read_cols)\n )\n mdata = mdata.append(tempdata) if mdata is not None else tempdata\n\n except (NoReaderWriterError, InvalidTemporalResError):\n continue\n\n if mdata is None:\n error_msg = \"No output found for only={}\".format(only)\n raise ValueError(error_msg)\n\n try:\n run_paras = self.read_parameters()\n self.config = run_paras\n mdata.metadata[\"parameters\"] = run_paras\n except FileNotFoundError:\n pass\n\n return mdata", "response": "Run MAGICC and parse the output."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchecks that our MAGICC. CFG files are set to safely work with Pymagicc.", "response": "def check_config(self):\n \"\"\"Check that our MAGICC ``.CFG`` files are set to safely work with PYMAGICC\n\n For further detail about why this is required, please see :ref:`MAGICC flags`.\n\n Raises\n ------\n ValueError\n If we are not certain that the config written by PYMAGICC will overwrite\n all other config i.e. that there will be no unexpected behaviour. A\n ValueError will also be raised if the user tries to use more than one\n scenario file.\n \"\"\"\n cfg_error_msg = (\n \"PYMAGICC is not the only tuning model that will be used by \"\n \"`MAGCFG_USER.CFG`: your run is likely to fail/do odd things\"\n )\n emisscen_error_msg = (\n \"You have more than one `FILE_EMISSCEN_X` flag set. Using more than \"\n \"one emissions scenario is hard to debug and unnecessary with \"\n \"Pymagicc's dataframe scenario input. Please combine all your \"\n \"scenarios into one dataframe with Pymagicc and pandas, then feed \"\n \"this single Dataframe into Pymagicc's run API.\"\n )\n\n nml_to_check = \"nml_allcfgs\"\n usr_cfg = read_cfg_file(join(self.run_dir, \"MAGCFG_USER.CFG\"))\n for k in usr_cfg[nml_to_check]:\n if k.startswith(\"file_tuningmodel\"):\n first_tuningmodel = k in [\"file_tuningmodel\", \"file_tuningmodel_1\"]\n if first_tuningmodel:\n if usr_cfg[nml_to_check][k] != \"PYMAGICC\":\n raise ValueError(cfg_error_msg)\n elif usr_cfg[nml_to_check][k] not in [\"USER\", \"\"]:\n raise ValueError(cfg_error_msg)\n\n elif k.startswith(\"file_emisscen_\"):\n if usr_cfg[nml_to_check][k] not in [\"NONE\", \"\"]:\n raise ValueError(emisscen_error_msg)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting an input file to disk", "response": "def write(self, mdata, name):\n \"\"\"Write an input file to disk\n\n Parameters\n ----------\n mdata : :obj:`pymagicc.io.MAGICCData`\n A MAGICCData instance with the data to write\n\n name : str\n The name of the file to write. The file will be written to the MAGICC\n instance's run directory i.e. ``self.run_dir``\n \"\"\"\n mdata.write(join(self.run_dir, name), self.version)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef read_parameters(self):\n param_fname = join(self.out_dir, \"PARAMETERS.OUT\")\n\n if not exists(param_fname):\n raise FileNotFoundError(\"No PARAMETERS.OUT found\")\n\n with open(param_fname) as nml_file:\n parameters = dict(f90nml.read(nml_file))\n for group in [\"nml_years\", \"nml_allcfgs\", \"nml_outputcfgs\"]:\n parameters[group] = dict(parameters[group])\n for k, v in parameters[group].items():\n parameters[group][k] = _clean_value(v)\n parameters[group.replace(\"nml_\", \"\")] = parameters.pop(group)\n self.config = parameters\n return parameters", "response": "Read a parameters. out file and return a dictionary containing all the configuration used by MAGICC\n "} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef remove_temp_copy(self):\n if self.is_temp and self.root_dir is not None:\n shutil.rmtree(self.root_dir)\n self.root_dir = None", "response": "Removes a temporary copy of the MAGICC version shipped with Pymagicc."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a configuration file for the current version of the MAGICC.", "response": "def set_config(\n self, filename=\"MAGTUNE_PYMAGICC.CFG\", top_level_key=\"nml_allcfgs\", **kwargs\n ):\n \"\"\"\n Create a configuration file for MAGICC.\n\n Writes a fortran namelist in run_dir.\n\n Parameters\n ----------\n filename : str\n Name of configuration file to write\n\n top_level_key : str\n Name of namelist to be written in the\n configuration file\n\n kwargs\n Other parameters to pass to the configuration file. No\n validation on the parameters is performed.\n\n Returns\n -------\n dict\n The contents of the namelist which was written to file\n \"\"\"\n kwargs = self._format_config(kwargs)\n\n fname = join(self.run_dir, filename)\n conf = {top_level_key: kwargs}\n f90nml.write(conf, fname, force=True)\n\n return conf"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nupdate a configuration file for MAGICC and returns the contents of the file.", "response": "def update_config(\n self, filename=\"MAGTUNE_PYMAGICC.CFG\", top_level_key=\"nml_allcfgs\", **kwargs\n ):\n \"\"\"Updates a configuration file for MAGICC\n\n Updates the contents of a fortran namelist in the run directory,\n creating a new namelist if none exists.\n\n Parameters\n ----------\n filename : str\n Name of configuration file to write\n\n top_level_key : str\n Name of namelist to be written in the\n configuration file\n\n kwargs\n Other parameters to pass to the configuration file. No\n validation on the parameters is performed.\n\n Returns\n -------\n dict\n The contents of the namelist which was written to file\n \"\"\"\n kwargs = self._format_config(kwargs)\n fname = join(self.run_dir, filename)\n\n if exists(fname):\n conf = f90nml.read(fname)\n else:\n conf = {top_level_key: {}}\n\n conf[top_level_key].update(kwargs)\n f90nml.write(conf, fname, force=True)\n\n return conf"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_zero_config(self):\n # zero_emissions is imported from scenarios module\n zero_emissions.write(join(self.run_dir, self._scen_file_name), self.version)\n\n time = zero_emissions.filter(variable=\"Emissions|CH4\", region=\"World\")[\n \"time\"\n ].values\n no_timesteps = len(time)\n # value doesn't actually matter as calculations are done from difference but\n # chose sensible value nonetheless\n ch4_conc_pi = 722\n ch4_conc = ch4_conc_pi * np.ones(no_timesteps)\n ch4_conc_df = pd.DataFrame(\n {\n \"time\": time,\n \"scenario\": \"idealised\",\n \"model\": \"unspecified\",\n \"climate_model\": \"unspecified\",\n \"variable\": \"Atmospheric Concentrations|CH4\",\n \"unit\": \"ppb\",\n \"todo\": \"SET\",\n \"region\": \"World\",\n \"value\": ch4_conc,\n }\n )\n ch4_conc_writer = MAGICCData(ch4_conc_df)\n ch4_conc_filename = \"HIST_CONSTANT_CH4_CONC.IN\"\n ch4_conc_writer.metadata = {\n \"header\": \"Constant pre-industrial CH4 concentrations\"\n }\n ch4_conc_writer.write(join(self.run_dir, ch4_conc_filename), self.version)\n\n fgas_conc_pi = 0\n fgas_conc = fgas_conc_pi * np.ones(no_timesteps)\n # MAGICC6 doesn't read this so not a problem, for MAGICC7 we might have to\n # write each file separately\n varname = \"FGAS_CONC\"\n fgas_conc_df = pd.DataFrame(\n {\n \"time\": time,\n \"scenario\": \"idealised\",\n \"model\": \"unspecified\",\n \"climate_model\": \"unspecified\",\n \"variable\": varname,\n \"unit\": \"ppt\",\n \"todo\": \"SET\",\n \"region\": \"World\",\n \"value\": fgas_conc,\n }\n )\n fgas_conc_writer = MAGICCData(fgas_conc_df)\n fgas_conc_filename = \"HIST_ZERO_{}.IN\".format(varname)\n fgas_conc_writer.metadata = {\"header\": \"Zero concentrations\"}\n fgas_conc_writer.write(join(self.run_dir, fgas_conc_filename), self.version)\n\n emis_config = self._fix_any_backwards_emissions_scen_key_in_config(\n {\"file_emissionscenario\": self._scen_file_name}\n )\n self.set_config(\n **emis_config,\n rf_initialization_method=\"ZEROSTARTSHIFT\",\n rf_total_constantafteryr=10000,\n file_co2i_emis=\"\",\n file_co2b_emis=\"\",\n co2_switchfromconc2emis_year=1750,\n file_ch4i_emis=\"\",\n file_ch4b_emis=\"\",\n file_ch4n_emis=\"\",\n file_ch4_conc=ch4_conc_filename,\n ch4_switchfromconc2emis_year=10000,\n file_n2oi_emis=\"\",\n file_n2ob_emis=\"\",\n file_n2on_emis=\"\",\n file_n2o_conc=\"\",\n n2o_switchfromconc2emis_year=1750,\n file_noxi_emis=\"\",\n file_noxb_emis=\"\",\n file_noxi_ot=\"\",\n file_noxb_ot=\"\",\n file_noxt_rf=\"\",\n file_soxnb_ot=\"\",\n file_soxi_ot=\"\",\n file_soxt_rf=\"\",\n file_soxi_emis=\"\",\n file_soxb_emis=\"\",\n file_soxn_emis=\"\",\n file_oci_emis=\"\",\n file_ocb_emis=\"\",\n file_oci_ot=\"\",\n file_ocb_ot=\"\",\n file_oci_rf=\"\",\n file_ocb_rf=\"\",\n file_bci_emis=\"\",\n file_bcb_emis=\"\",\n file_bci_ot=\"\",\n file_bcb_ot=\"\",\n file_bci_rf=\"\",\n file_bcb_rf=\"\",\n bcoc_switchfromrf2emis_year=1750,\n file_nh3i_emis=\"\",\n file_nh3b_emis=\"\",\n file_nmvoci_emis=\"\",\n file_nmvocb_emis=\"\",\n file_coi_emis=\"\",\n file_cob_emis=\"\",\n file_mineraldust_rf=\"\",\n file_landuse_rf=\"\",\n file_bcsnow_rf=\"\",\n # rf_fgassum_scale=0, # this appears to do nothing, hence the next two lines\n file_fgas_conc=[fgas_conc_filename] * 12,\n fgas_switchfromconc2emis_year=10000,\n rf_mhalosum_scale=0,\n mhalo_switch_conc2emis_yr=1750,\n stratoz_o3scale=0,\n rf_volcanic_scale=0,\n rf_solar_scale=0,\n )", "response": "Set the config such that radiative forcing and temperature output will be zero."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the start and end dates of the simulations.", "response": "def set_years(self, startyear=1765, endyear=2100):\n \"\"\"\n Set the start and end dates of the simulations.\n\n Parameters\n ----------\n startyear : int\n Start year of the simulation\n\n endyear : int\n End year of the simulation\n\n Returns\n -------\n dict\n The contents of the namelist\n \"\"\"\n # TODO: test altering stepsperyear, I think 1, 2 and 24 should all work\n return self.set_config(\n \"MAGCFG_NMLYEARS.CFG\",\n \"nml_years\",\n endyear=endyear,\n startyear=startyear,\n stepsperyear=12,\n )"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_output_variables(self, write_ascii=True, write_binary=False, **kwargs):\n\n assert (\n write_ascii or write_binary\n ), \"write_binary and/or write_ascii must be configured\"\n if write_binary and write_ascii:\n ascii_binary = \"BOTH\"\n elif write_ascii:\n ascii_binary = \"ASCII\"\n else:\n ascii_binary = \"BINARY\"\n\n # defaults\n outconfig = {\n \"out_emissions\": 0,\n \"out_gwpemissions\": 0,\n \"out_sum_gwpemissions\": 0,\n \"out_concentrations\": 0,\n \"out_carboncycle\": 0,\n \"out_forcing\": 0,\n \"out_surfaceforcing\": 0,\n \"out_permafrost\": 0,\n \"out_temperature\": 0,\n \"out_sealevel\": 0,\n \"out_parameters\": 0,\n \"out_misc\": 0,\n \"out_timeseriesmix\": 0,\n \"out_rcpdata\": 0,\n \"out_summaryidx\": 0,\n \"out_inverseemis\": 0,\n \"out_tempoceanlayers\": 0,\n \"out_heatuptake\": 0,\n \"out_ascii_binary\": ascii_binary,\n \"out_warnings\": 0,\n \"out_precipinput\": 0,\n \"out_aogcmtuning\": 0,\n \"out_ccycletuning\": 0,\n \"out_observationaltuning\": 0,\n \"out_keydata_1\": 0,\n \"out_keydata_2\": 0,\n }\n if self.version == 7:\n outconfig[\"out_oceanarea\"] = 0\n outconfig[\"out_lifetimes\"] = 0\n\n for kw in kwargs:\n val = 1 if kwargs[kw] else 0 # convert values to 0/1 instead of booleans\n outconfig[\"out_\" + kw.lower()] = val\n\n self.update_config(**outconfig)", "response": "Set the output configuration minimising output as much as possible."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef diagnose_tcr_ecs(self, **kwargs):\n if self.version == 7:\n raise NotImplementedError(\"MAGICC7 cannot yet diagnose ECS and TCR\")\n self._diagnose_tcr_ecs_config_setup(**kwargs)\n timeseries = self.run(\n scenario=None,\n only=[\n \"Atmospheric Concentrations|CO2\",\n \"Radiative Forcing\",\n \"Surface Temperature\",\n ],\n )\n tcr, ecs = self._get_tcr_ecs_from_diagnosis_results(timeseries)\n return {\"tcr\": tcr, \"ecs\": ecs, \"timeseries\": timeseries}", "response": "Diagnose TCR and ECS and return a dictionary of the relevant model input and output times."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset the emissions flags correctly.", "response": "def set_emission_scenario_setup(self, scenario, config_dict):\n \"\"\"Set the emissions flags correctly.\n\n Parameters\n ----------\n scenario : :obj:`pymagicc.io.MAGICCData`\n Scenario to run.\n\n config_dict : dict\n Dictionary with current input configurations which is to be validated and\n updated where necessary.\n\n Returns\n -------\n dict\n Updated configuration\n \"\"\"\n self.write(scenario, self._scen_file_name)\n # can be lazy in this line as fix backwards key handles errors for us\n config_dict[\"file_emissionscenario\"] = self._scen_file_name\n config_dict = self._fix_any_backwards_emissions_scen_key_in_config(config_dict)\n\n return config_dict"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if a type is defined", "response": "def contains(value: Union[str, 'Type']) -> bool:\n \"\"\" Checks if a type is defined \"\"\"\n if isinstance(value, str):\n return any(value.lower() == i.value for i in Type)\n\n return any(value == i for i in Type)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef xml_replace(filename, **replacements):\n keywords = set(replacements.keys())\n templatename = f'{filename}.xmlt'\n targetname = f'{filename}.xml'\n print(f'template file: {templatename}')\n print(f'target file: {targetname}')\n print('replacements:')\n with open(templatename) as templatefile:\n templatebody = templatefile.read()\n parts = templatebody.replace('').split('|-->')\n defaults = {}\n for idx, part in enumerate(parts):\n if idx % 2:\n subparts = part.partition('=')\n if subparts[2]:\n parts[idx] = subparts[0]\n if subparts[0] not in replacements:\n if ((subparts[0] in defaults) and\n (defaults[subparts[0]] != str(subparts[2]))):\n raise RuntimeError(\n f'Template file `{templatename}` defines '\n f'different default values for marker '\n f'`{subparts[0]}`.')\n defaults[subparts[0]] = str(subparts[2])\n markers = parts[1::2]\n try:\n unused_keywords = keywords.copy()\n for idx, part in enumerate(parts):\n if idx % 2:\n argument_info = 'given argument'\n newpart = replacements.get(part)\n if newpart is None:\n argument_info = 'default argument'\n newpart = defaults.get(part)\n if newpart is None:\n raise RuntimeError(\n f'Marker `{part}` cannot be replaced.')\n print(f' {part} --> {newpart} ({argument_info})')\n parts[idx] = str(newpart)\n unused_keywords.discard(part)\n targetbody = ''.join(parts)\n if unused_keywords:\n raise RuntimeError(\n f'Keyword(s) `{objecttools.enumeration(unused_keywords)}` '\n f'cannot be used.')\n with open(targetname, 'w') as targetfile:\n targetfile.write(targetbody)\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to replace the markers '\n f'`{objecttools.enumeration(sorted(set(markers)))}` of the '\n f'XML template file `{templatename}` with the available '\n f'keywords `{objecttools.enumeration(sorted(keywords))}`')", "response": "Read the content of an XML template file and apply the given replacements to the given XML file."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the required indexes based on the given lambda function and the Timegrids object handled by module hydpy. pub.", "response": "def _calcidxs(func):\n \"\"\"Return the required indexes based on the given lambda function\n and the |Timegrids| object handled by module |pub|. Raise a\n |RuntimeError| if the latter is not available.\n \"\"\"\n timegrids = hydpy.pub.get('timegrids')\n if timegrids is None:\n raise RuntimeError(\n 'An Indexer object has been asked for an %s array. Such an '\n 'array has neither been determined yet nor can it be '\n 'determined automatically at the moment. Either define an '\n '%s array manually and pass it to the Indexer object, or make '\n 'a proper Timegrids object available within the pub module. '\n 'In usual HydPy applications, the latter is done '\n 'automatically.'\n % (func.__name__, func.__name__))\n idxs = numpy.empty(len(timegrids.init), dtype=int)\n for jdx, date in enumerate(hydpy.pub.timegrids.init):\n idxs[jdx] = func(date)\n return idxs"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef dayofyear(self):\n def _dayofyear(date):\n return (date.dayofyear-1 +\n ((date.month > 2) and (not date.leapyear)))\n return _dayofyear", "response": "Return the day of the year index."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef timeofyear(self):\n refgrid = timetools.Timegrid(\n timetools.Date('2000.01.01'),\n timetools.Date('2001.01.01'),\n hydpy.pub.timegrids.stepsize)\n\n def _timeofyear(date):\n date = copy.deepcopy(date)\n date.year = 2000\n return refgrid[date]\n\n return _timeofyear", "response": "Return a function that returns the time of the year index for the current year."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nassigns the given docstring to the property instance and if possible to the module of its owner class.", "response": "def set_doc(self, doc: str):\n \"\"\"Assign the given docstring to the property instance and, if\n possible, to the `__test__` dictionary of the module of its\n owner class.\"\"\"\n self.__doc__ = doc\n if hasattr(self, 'module'):\n ref = f'{self.objtype.__name__}.{self.name}'\n self.module.__dict__['__test__'][ref] = doc"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getter_(self, fget) -> 'BaseProperty':\n self.fget = fget\n self.set_doc(fget.__doc__)\n return self", "response": "Add the given getter function and its docstring to the\n property and return it."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning |True| or |False| to indicate if the protected property is ready for the given object.", "response": "def isready(self, obj) -> bool:\n \"\"\"Return |True| or |False| to indicate if the protected\n property is ready for the given object. If the object is\n unknow, |ProtectedProperty| returns |False|.\"\"\"\n return vars(obj).get(self.name, False)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef allready(self, obj) -> bool:\n for prop in self.__properties:\n if not prop.isready(obj):\n return False\n return True", "response": "Return |True| or |False| to indicate whether all protected\n properties are ready or not."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef call_fget(self, obj) -> Any:\n custom = vars(obj).get(self.name)\n if custom is None:\n return self.fget(obj)\n return custom", "response": "Return the predefined custom value when available otherwise return the value defined by the getter function."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef call_fset(self, obj, value) -> None:\n vars(obj)[self.name] = self.fset(obj, value)", "response": "Store the given custom value and call the setter function."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef call_fdel(self, obj) -> None:\n self.fdel(obj)\n try:\n del vars(obj)[self.name]\n except KeyError:\n pass", "response": "Remove the predefined custom value and call the delete function."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef trim(self, lower=None, upper=None):\n if lower is None:\n lower = getattr(self.subpars.relwb, 'value', None)\n lland_parameters.ParameterSoil.trim(self, lower, upper)", "response": "Trim upper values in accordance with RelWB \\\\ leq RelWZ."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntrims upper values in accordance with RelWB \\\\ leq RelWZ.", "response": "def trim(self, lower=None, upper=None):\n \"\"\"Trim upper values in accordance with :math:`RelWB \\\\leq RelWZ`.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(3)\n >>> lnk(ACKER)\n >>> relwz.values = 0.5\n >>> relwb(0.2, 0.5, 0.8)\n >>> relwb\n relwb(0.2, 0.5, 0.5)\n \"\"\"\n if upper is None:\n upper = getattr(self.subpars.relwz, 'value', None)\n lland_parameters.ParameterSoil.trim(self, lower, upper)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef trim(self, lower=None, upper=None):\n if lower is None:\n lower = getattr(self.subpars.eqi1, 'value', None)\n super().trim(lower, upper)", "response": "Trim lower values in accordance with : math : EQI1 \\\\ leq EQB."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef trim(self, lower=None, upper=None):\n if lower is None:\n lower = getattr(self.subpars.eqi2, 'value', None)\n if upper is None:\n upper = getattr(self.subpars.eqb, 'value', None)\n super().trim(lower, upper)", "response": "Trim lower values in accordance with\n ."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of digital time choices every half hour from 00 to 23 to 30.", "response": "def time_choices():\n \"\"\"Return digital time choices every half hour from 00:00 to 23:30.\"\"\"\n hours = list(range(0, 24))\n times = []\n for h in hours:\n hour = str(h).zfill(2)\n times.append(hour+':00')\n times.append(hour+':30')\n return list(zip(times, times))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _add_lines(specification, module):\n caption = _all_spec2capt.get(specification, 'dummy')\n if caption.split()[-1] in ('parameters', 'sequences', 'Masks'):\n exists_collectionclass = True\n name_collectionclass = caption.title().replace(' ', '')\n else:\n exists_collectionclass = False\n lines = []\n if specification == 'model':\n lines += [f'',\n f'.. autoclass:: {module.__name__}.Model',\n f' :members:',\n f' :show-inheritance:',\n f' :exclude-members: {\", \".join(EXCLUDE_MEMBERS)}']\n elif exists_collectionclass:\n lines += [f'',\n f'.. autoclass:: {module.__name__}.{name_collectionclass}',\n f' :members:',\n f' :show-inheritance:',\n f' :exclude-members: {\", \".join(EXCLUDE_MEMBERS)}']\n lines += ['',\n '.. automodule:: ' + module.__name__,\n ' :members:',\n ' :show-inheritance:']\n if specification == 'model':\n lines += [' :exclude-members: Model']\n elif exists_collectionclass:\n lines += [' :exclude-members: ' + name_collectionclass]\n return lines", "response": "Return a list of autodoc commands for a basemodels docstring."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef autodoc_basemodel(module):\n autodoc_tuple2doc(module)\n namespace = module.__dict__\n doc = namespace.get('__doc__')\n if doc is None:\n doc = ''\n basemodulename = namespace['__name__'].split('.')[-1]\n modules = {key: value for key, value in namespace.items()\n if (isinstance(value, types.ModuleType) and\n key.startswith(basemodulename+'_'))}\n substituter = Substituter(hydpy.substituter)\n lines = []\n specification = 'model'\n modulename = basemodulename+'_'+specification\n if modulename in modules:\n module = modules[modulename]\n lines += _add_title('Model features', '-')\n lines += _add_lines(specification, module)\n substituter.add_module(module)\n for (title, spec2capt) in (('Parameter features', _PAR_SPEC2CAPT),\n ('Sequence features', _SEQ_SPEC2CAPT),\n ('Auxiliary features', _AUX_SPEC2CAPT)):\n found_module = False\n new_lines = _add_title(title, '-')\n for (specification, caption) in spec2capt.items():\n modulename = basemodulename+'_'+specification\n module = modules.get(modulename)\n if module:\n found_module = True\n new_lines += _add_title(caption, '.')\n new_lines += _add_lines(specification, module)\n substituter.add_module(module)\n if found_module:\n lines += new_lines\n doc += '\\n'.join(lines)\n namespace['__doc__'] = doc\n basemodule = importlib.import_module(namespace['__name__'])\n substituter.add_module(basemodule)\n substituter.update_masters()\n namespace['substituter'] = substituter", "response": "Add an exhaustive docstring to the given module of a basemodel."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef autodoc_applicationmodel(module):\n autodoc_tuple2doc(module)\n name_applicationmodel = module.__name__\n name_basemodel = name_applicationmodel.split('_')[0]\n module_basemodel = importlib.import_module(name_basemodel)\n substituter = Substituter(module_basemodel.substituter)\n substituter.add_module(module)\n substituter.update_masters()\n module.substituter = substituter", "response": "Improves the docstrings of application models when called\n at the bottom of the respective module."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\npreparing and return a |Substituter| object for the main __init__ object of HydPy *.", "response": "def prepare_mainsubstituter():\n \"\"\"Prepare and return a |Substituter| object for the main `__init__`\n file of *HydPy*.\"\"\"\n substituter = Substituter()\n for module in (builtins, numpy, datetime, unittest, doctest, inspect, io,\n os, sys, time, collections, itertools, subprocess, scipy,\n typing):\n substituter.add_module(module)\n for subpackage in (auxs, core, cythons, exe):\n for dummy, name, dummy in pkgutil.walk_packages(subpackage.__path__):\n full_name = subpackage.__name__ + '.' + name\n substituter.add_module(importlib.import_module(full_name))\n substituter.add_modules(models)\n for cymodule in (annutils, smoothutils, pointerutils):\n substituter.add_module(cymodule, cython=True)\n substituter._short2long['|pub|'] = ':mod:`~hydpy.pub`'\n substituter._short2long['|config|'] = ':mod:`~hydpy.config`'\n return substituter"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _number_of_line(member_tuple):\n member = member_tuple[1]\n try:\n return member.__code__.co_firstlineno\n except AttributeError:\n pass\n try:\n return inspect.findsource(member)[1]\n except BaseException:\n pass\n for value in vars(member).values():\n try:\n return value.__code__.co_firstlineno\n except AttributeError:\n pass\n return 0", "response": "Try to return the number of the first line of the definition of a\n member of a module."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef autodoc_module(module):\n doc = getattr(module, '__doc__')\n if doc is None:\n doc = ''\n members = []\n for name, member in inspect.getmembers(module):\n if ((not name.startswith('_')) and\n (inspect.getmodule(member) is module)):\n members.append((name, member))\n members = sorted(members, key=_number_of_line)\n if members:\n lines = ['\\n\\nModule :mod:`~%s` implements the following members:\\n'\n % module.__name__]\n for (name, member) in members:\n if inspect.isfunction(member):\n type_ = 'func'\n elif inspect.isclass(member):\n type_ = 'class'\n else:\n type_ = 'obj'\n lines.append(' * :%s:`~%s` %s'\n % (type_, name, objecttools.description(member)))\n doc = doc + '\\n\\n' + '\\n'.join(lines) + '\\n\\n' + 80*'_'\n module.__doc__ = doc", "response": "Add a short summary of all implemented members to a modules docstring."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef autodoc_tuple2doc(module):\n modulename = module.__name__\n for membername, member in inspect.getmembers(module):\n for tuplename, descr in _name2descr.items():\n tuple_ = getattr(member, tuplename, None)\n if tuple_:\n logstring = f'{modulename}.{membername}.{tuplename}'\n if logstring not in _loggedtuples:\n _loggedtuples.add(logstring)\n lst = [f'\\n\\n\\n {descr}:']\n if tuplename == 'CLASSES':\n type_ = 'func'\n else:\n type_ = 'class'\n for cls in tuple_:\n lst.append(\n f' * '\n f':{type_}:`{cls.__module__}.{cls.__name__}`'\n f' {objecttools.description(cls)}')\n doc = getattr(member, '__doc__')\n if doc is None:\n doc = ''\n member.__doc__ = doc + '\\n'.join(l for l in lst)", "response": "Include tuples as CLASS of ControlParameters and RUN_METHODS of Models into the respective docstring."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef consider_member(name_member, member, module, class_=None):\n if name_member.startswith('_'):\n return False\n if inspect.ismodule(member):\n return False\n real_module = getattr(member, '__module__', None)\n if not real_module:\n return True\n if real_module != module.__name__:\n if class_ and hasattr(member, '__get__'):\n return True\n if 'hydpy' in real_module:\n return False\n if module.__name__ not in real_module:\n return False\n return True", "response": "Return |True| if the given member should be added to the\n substitutions. If not return |False|."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_role(member, cython=False):\n if inspect.isroutine(member) or isinstance(member, numpy.ufunc):\n return 'func'\n elif inspect.isclass(member):\n return 'class'\n elif cython:\n return 'func'\n return 'const'", "response": "Return the reStructuredText role func class or const for the given member."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_substitution(self, short, medium, long, module):\n name = module.__name__\n if 'builtin' in name:\n self._short2long[short] = long.split('~')[0] + long.split('.')[-1]\n else:\n if ('hydpy' in name) and (short not in self._blacklist):\n if short in self._short2long:\n if self._short2long[short] != long:\n self._blacklist.add(short)\n del self._short2long[short]\n else:\n self._short2long[short] = long\n self._medium2long[medium] = long", "response": "Add the given substitutions both as a short2long mapping and a medium2long mapping."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd the given module to the internal dictionary.", "response": "def add_module(self, module, cython=False):\n \"\"\"Add the given module, its members, and their submembers.\n\n The first examples are based on the site-package |numpy|: which\n is passed to method |Substituter.add_module|:\n\n >>> from hydpy.core.autodoctools import Substituter\n >>> substituter = Substituter()\n >>> import numpy\n >>> substituter.add_module(numpy)\n\n Firstly, the module itself is added:\n\n >>> substituter.find('|numpy|')\n |numpy| :mod:`~numpy`\n\n Secondly, constants like |numpy.nan| are added:\n\n >>> substituter.find('|numpy.nan|')\n |numpy.nan| :const:`~numpy.nan`\n\n Thirdly, functions like |numpy.clip| are added:\n\n >>> substituter.find('|numpy.clip|')\n |numpy.clip| :func:`~numpy.clip`\n\n Fourthly, clases line |numpy.ndarray| are added:\n\n >>> substituter.find('|numpy.ndarray|')\n |numpy.ndarray| :class:`~numpy.ndarray`\n\n When adding Cython modules, the `cython` flag should be set |True|:\n\n >>> from hydpy.cythons import pointerutils\n >>> substituter.add_module(pointerutils, cython=True)\n >>> substituter.find('set_pointer')\n |PPDouble.set_pointer| \\\n:func:`~hydpy.cythons.autogen.pointerutils.PPDouble.set_pointer`\n |pointerutils.PPDouble.set_pointer| \\\n:func:`~hydpy.cythons.autogen.pointerutils.PPDouble.set_pointer`\n \"\"\"\n name_module = module.__name__.split('.')[-1]\n short = ('|%s|'\n % name_module)\n long = (':mod:`~%s`'\n % module.__name__)\n self._short2long[short] = long\n for (name_member, member) in vars(module).items():\n if self.consider_member(\n name_member, member, module):\n role = self.get_role(member, cython)\n short = ('|%s|'\n % name_member)\n medium = ('|%s.%s|'\n % (name_module,\n name_member))\n long = (':%s:`~%s.%s`'\n % (role,\n module.__name__,\n name_member))\n self.add_substitution(short, medium, long, module)\n if inspect.isclass(member):\n for name_submember, submember in vars(member).items():\n if self.consider_member(\n name_submember, submember, module, member):\n role = self.get_role(submember, cython)\n short = ('|%s.%s|'\n % (name_member,\n name_submember))\n medium = ('|%s.%s.%s|'\n % (name_module,\n name_member,\n name_submember))\n long = (':%s:`~%s.%s.%s`'\n % (role,\n module.__name__,\n name_member,\n name_submember))\n self.add_substitution(short, medium, long, module)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_modules(self, package):\n for name in os.listdir(package.__path__[0]):\n if name.startswith('_'):\n continue\n name = name.split('.')[0]\n short = '|%s|' % name\n long = ':mod:`~%s.%s`' % (package.__package__, name)\n self._short2long[short] = long", "response": "Add the modules of the given package without their members."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating all master objects and slaves.", "response": "def update_masters(self):\n \"\"\"Update all `master` |Substituter| objects.\n\n If a |Substituter| object is passed to the constructor of another\n |Substituter| object, they become `master` and `slave`:\n\n >>> from hydpy.core.autodoctools import Substituter\n >>> sub1 = Substituter()\n >>> from hydpy.core import devicetools\n >>> sub1.add_module(devicetools)\n >>> sub2 = Substituter(sub1)\n >>> sub3 = Substituter(sub2)\n >>> sub3.master.master is sub1\n True\n >>> sub2 in sub1.slaves\n True\n\n During initialization, all mappings handled by the master object\n are passed to its new slave:\n\n >>> sub3.find('Node|')\n |Node| :class:`~hydpy.core.devicetools.Node`\n |devicetools.Node| :class:`~hydpy.core.devicetools.Node`\n\n Updating a slave, does not affect its master directly:\n\n >>> from hydpy.core import hydpytools\n >>> sub3.add_module(hydpytools)\n >>> sub3.find('HydPy|')\n |HydPy| :class:`~hydpy.core.hydpytools.HydPy`\n |hydpytools.HydPy| :class:`~hydpy.core.hydpytools.HydPy`\n >>> sub2.find('HydPy|')\n\n Through calling |Substituter.update_masters|, the `medium2long`\n mappings are passed from the slave to its master:\n\n >>> sub3.update_masters()\n >>> sub2.find('HydPy|')\n |hydpytools.HydPy| :class:`~hydpy.core.hydpytools.HydPy`\n\n Then each master object updates its own master object also:\n\n >>> sub1.find('HydPy|')\n |hydpytools.HydPy| :class:`~hydpy.core.hydpytools.HydPy`\n\n In reverse, subsequent updates of master objects to not affect\n their slaves directly:\n\n >>> from hydpy.core import masktools\n >>> sub1.add_module(masktools)\n >>> sub1.find('Masks|')\n |Masks| :class:`~hydpy.core.masktools.Masks`\n |masktools.Masks| :class:`~hydpy.core.masktools.Masks`\n >>> sub2.find('Masks|')\n\n Through calling |Substituter.update_slaves|, the `medium2long`\n mappings are passed the master to all of its slaves:\n\n >>> sub1.update_slaves()\n >>> sub2.find('Masks|')\n |masktools.Masks| :class:`~hydpy.core.masktools.Masks`\n >>> sub3.find('Masks|')\n |masktools.Masks| :class:`~hydpy.core.masktools.Masks`\n \"\"\"\n if self.master is not None:\n self.master._medium2long.update(self._medium2long)\n self.master.update_masters()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate all |Slave| objects.", "response": "def update_slaves(self):\n \"\"\"Update all `slave` |Substituter| objects.\n\n See method |Substituter.update_masters| for further information.\n \"\"\"\n for slave in self.slaves:\n slave._medium2long.update(self._medium2long)\n slave.update_slaves()"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_commands(self, source=None):\n commands = []\n for key, value in self:\n if (source is None) or (key in source):\n commands.append('.. %s replace:: %s' % (key, value))\n return '\\n'.join(commands)", "response": "Return a string containing multiple reStructuredText commands with the substitutions currently defined."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprints all substitutions that include the given text string.", "response": "def find(self, text):\n \"\"\"Print all substitutions that include the given text string.\"\"\"\n for key, value in self:\n if (text in key) or (text in value):\n print(key, value)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd print commands time to the given function informing about execution time. To show how the |print_progress| decorator works, we need to modify the functions used by |print_progress| to gain system time information available in module |time|. First, we mock the functions |time.strftime| and |time.perf_counter|: >>> import time >>> from unittest import mock >>> strftime = time.strftime >>> perf_counter = time.perf_counter >>> strftime_mock = mock.MagicMock() >>> time.strftime = strftime_mock >>> time.perf_counter = mock.MagicMock() The mock of |time.strftime| shall respond to two calls, as if the first call to a decorated function occurs at quarter past eight, and the second one two seconds later: >>> time.strftime.side_effect = '20:15:00', '20:15:02' The mock of |time.perf_counter| shall respond to four calls, as if the subsequent calls by decorated functions occur at second 1, 3, 4, and 7: >>> time.perf_counter.side_effect = 1, 3, 4, 7 Now we decorate two test functions. The first one does nothing; the second one only calls the first one: >>> from hydpy.core.printtools import print_progress >>> @print_progress ... def test1(): ... pass >>> @print_progress ... def test2(): ... test1() The first example shows that the output is appropriately indented, tat the returned times are at the right place, that the calculated execution the is correct, and that the mock of |time.strftime| received a valid format string: >>> from hydpy import pub >>> pub.options.printprogress = True >>> test2() method test2 started at 20:15:00 method test1 started at 20:15:02 seconds elapsed: 1 seconds elapsed: 6 >>> strftime_mock.call_args call('%H:%M:%S') The second example verifies that resetting the indentation works: >>> time.strftime.side_effect = '20:15:00', '20:15:02' >>> time.perf_counter.side_effect = 1, 3, 4, 7 >>> test2() method test2 started at 20:15:00 method test1 started at 20:15:02 seconds elapsed: 1 seconds elapsed: 6 The last example shows that disabling the |Options.printprogress| option works as expected: >>> pub.options.printprogress = False >>> test2() >>> time.strftime = strftime >>> time.perf_counter = perf_counter", "response": "def print_progress(wrapped, _=None, args=None, kwargs=None):\n \"\"\"Add print commands time to the given function informing about\n execution time.\n\n To show how the |print_progress| decorator works, we need to modify the\n functions used by |print_progress| to gain system time information\n available in module |time|.\n\n First, we mock the functions |time.strftime| and |time.perf_counter|:\n\n >>> import time\n >>> from unittest import mock\n >>> strftime = time.strftime\n >>> perf_counter = time.perf_counter\n >>> strftime_mock = mock.MagicMock()\n >>> time.strftime = strftime_mock\n >>> time.perf_counter = mock.MagicMock()\n\n The mock of |time.strftime| shall respond to two calls, as if the first\n call to a decorated function occurs at quarter past eight, and the second\n one two seconds later:\n\n >>> time.strftime.side_effect = '20:15:00', '20:15:02'\n\n The mock of |time.perf_counter| shall respond to four calls, as if the\n subsequent calls by decorated functions occur at second 1, 3, 4, and 7:\n\n >>> time.perf_counter.side_effect = 1, 3, 4, 7\n\n Now we decorate two test functions. The first one does nothing; the\n second one only calls the first one:\n\n >>> from hydpy.core.printtools import print_progress\n >>> @print_progress\n ... def test1():\n ... pass\n >>> @print_progress\n ... def test2():\n ... test1()\n\n The first example shows that the output is appropriately indented,\n tat the returned times are at the right place, that the calculated\n execution the is correct, and that the mock of |time.strftime|\n received a valid format string:\n\n >>> from hydpy import pub\n >>> pub.options.printprogress = True\n >>> test2()\n method test2 started at 20:15:00\n method test1 started at 20:15:02\n seconds elapsed: 1\n seconds elapsed: 6\n >>> strftime_mock.call_args\n call('%H:%M:%S')\n\n The second example verifies that resetting the indentation works:\n\n >>> time.strftime.side_effect = '20:15:00', '20:15:02'\n >>> time.perf_counter.side_effect = 1, 3, 4, 7\n >>> test2()\n method test2 started at 20:15:00\n method test1 started at 20:15:02\n seconds elapsed: 1\n seconds elapsed: 6\n\n The last example shows that disabling the |Options.printprogress|\n option works as expected:\n\n >>> pub.options.printprogress = False\n >>> test2()\n\n >>> time.strftime = strftime\n >>> time.perf_counter = perf_counter\n \"\"\"\n global _printprogress_indentation\n _printprogress_indentation += 4\n try:\n if hydpy.pub.options.printprogress:\n blanks = ' ' * _printprogress_indentation\n name = wrapped.__name__\n time_ = time.strftime('%H:%M:%S')\n with PrintStyle(color=34, font=1):\n print(f'{blanks}method {name} started at {time_}')\n seconds = time.perf_counter()\n sys.stdout.flush()\n wrapped(*args, **kwargs)\n blanks = ' ' * (_printprogress_indentation+4)\n seconds = time.perf_counter()-seconds\n with PrintStyle(color=34, font=1):\n print(f'{blanks}seconds elapsed: {seconds}')\n sys.stdout.flush()\n else:\n wrapped(*args, **kwargs)\n finally:\n _printprogress_indentation -= 4"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nprinting a simple progress bar while processing the given iterable.", "response": "def progressbar(iterable, length=23):\n \"\"\"Print a simple progress bar while processing the given iterable.\n\n Function |progressbar| does print the progress bar when option\n `printprogress` is activted:\n\n >>> from hydpy import pub\n >>> pub.options.printprogress = True\n\n You can pass an iterable object. Say you want to calculate the the sum\n of all integer values from 1 to 100 and print the progress of the\n calculation. Using function |range| (which returns a list in Python 2\n and an iterator in Python3, but both are fine), one just has to\n interpose function |progressbar|:\n\n >>> from hydpy.core.printtools import progressbar\n >>> x_sum = 0\n >>> for x in progressbar(range(1, 101)):\n ... x_sum += x\n |---------------------|\n ***********************\n >>> x_sum\n 5050\n\n To prevent possible interim print commands from dismembering the status\n bar, they are delayed until the status bar is complete. For intermediate\n print outs of each fiftieth calculation, the result looks as follows:\n\n >>> x_sum = 0\n >>> for x in progressbar(range(1, 101)):\n ... x_sum += x\n ... if not x % 50:\n ... print(x, x_sum)\n |---------------------|\n ***********************\n 50 1275\n 100 5050\n\n\n The number of characters of the progress bar can be changed:\n\n >>> for i in progressbar(range(100), length=50):\n ... continue\n |------------------------------------------------|\n **************************************************\n\n But its maximum number of characters is restricted by the length of the\n given iterable:\n\n >>> for i in progressbar(range(10), length=50):\n ... continue\n |--------|\n **********\n\n The smallest possible progress bar has two characters:\n\n >>> for i in progressbar(range(2)):\n ... continue\n ||\n **\n\n For iterables of length one or zero, no progress bar is plottet:\n\n\n >>> for i in progressbar(range(1)):\n ... continue\n\n\n The same is True when the `printprogress` option is inactivated:\n\n >>> pub.options.printprogress = False\n >>> for i in progressbar(range(100)):\n ... continue\n \"\"\"\n if hydpy.pub.options.printprogress and (len(iterable) > 1):\n temp_name = os.path.join(tempfile.gettempdir(),\n 'HydPy_progressbar_stdout')\n temp_stdout = open(temp_name, 'w')\n real_stdout = sys.stdout\n try:\n sys.stdout = temp_stdout\n nmbstars = min(len(iterable), length)\n nmbcounts = len(iterable)/nmbstars\n indentation = ' '*max(_printprogress_indentation, 0)\n with PrintStyle(color=36, font=1, file=real_stdout):\n print(' %s|%s|\\n%s ' % (indentation,\n '-'*(nmbstars-2),\n indentation),\n end='',\n file=real_stdout)\n counts = 1.\n for next_ in iterable:\n counts += 1.\n if counts >= nmbcounts:\n print(end='*', file=real_stdout)\n counts -= nmbcounts\n yield next_\n finally:\n try:\n temp_stdout.close()\n except BaseException:\n pass\n sys.stdout = real_stdout\n print()\n with open(temp_name, 'r') as temp_stdout:\n sys.stdout.write(temp_stdout.read())\n sys.stdout.flush()\n else:\n for next_ in iterable:\n yield next_"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef start_server(socket, projectname, xmlfilename: str) -> None:\n state.initialise(projectname, xmlfilename)\n server = http.server.HTTPServer(('', int(socket)), HydPyServer)\n server.serve_forever()", "response": "Start the HydPy server using the given socket."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef await_server(port, seconds):\n now = time.perf_counter()\n end = now + float(seconds)\n while now <= end:\n try:\n urllib.request.urlopen(f'http://localhost:{port}/status')\n break\n except urllib.error.URLError:\n time.sleep(0.1)\n now = time.perf_counter()\n else:\n raise urllib.error.URLError(\n f'Waited for {seconds} seconds without response on port {port}.')", "response": "Block the current process until the hydpy server is responding on the given port or the given number of seconds elapsed."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef initialise(self, projectname: str, xmlfile: str) -> None:\n write = commandtools.print_textandtime\n write(f'Start HydPy project `{projectname}`')\n hp = hydpytools.HydPy(projectname)\n write(f'Read configuration file `{xmlfile}`')\n interface = xmltools.XMLInterface(xmlfile)\n write('Interpret the defined options')\n interface.update_options()\n write('Interpret the defined period')\n interface.update_timegrids()\n write('Read all network files')\n hp.prepare_network()\n write('Activate the selected network')\n hp.update_devices(interface.fullselection)\n write('Read the required control files')\n hp.init_models()\n write('Read the required condition files')\n interface.conditions_io.load_conditions()\n write('Read the required time series files')\n interface.series_io.prepare_series()\n interface.exchange.prepare_series()\n interface.series_io.load_series()\n self.hp = hp\n self.parameteritems = interface.exchange.parameteritems\n self.conditionitems = interface.exchange.conditionitems\n self.getitems = interface.exchange.getitems\n self.conditions = {}\n self.parameteritemvalues = collections.defaultdict(lambda: {})\n self.modifiedconditionitemvalues = collections.defaultdict(lambda: {})\n self.getitemvalues = collections.defaultdict(lambda: {})\n self.init_conditions = hp.conditions\n self.timegrids = {}", "response": "Initialise a HydPy project based on the given XML configuration file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef POST_evaluate(self) -> None:\n for name, value in self._inputs.items():\n result = eval(value)\n self._outputs[name] = objecttools.flatten_repr(result)", "response": "Evaluate any valid Python expression with the * HydPy * server\n process and get its result."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstop and close the HydPy server.", "response": "def GET_close_server(self) -> None:\n \"\"\"Stop and close the *HydPy* server.\"\"\"\n def _close_server():\n self.server.shutdown()\n self.server.server_close()\n shutter = threading.Thread(target=_close_server)\n shutter.deamon = True\n shutter.start()"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GET_parameteritemtypes(self) -> None:\n for item in state.parameteritems:\n self._outputs[item.name] = self._get_itemtype(item)", "response": "Get the types of all current exchange items supposed to change\n the values of |Parameter| objects."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GET_conditionitemtypes(self) -> None:\n for item in state.conditionitems:\n self._outputs[item.name] = self._get_itemtype(item)", "response": "Get the types of all current exchange items supposed to change\n the values of StateSequence or LogSequence objects."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GET_getitemtypes(self) -> None:\n for item in state.getitems:\n type_ = self._get_itemtype(item)\n for name, _ in item.yield_name2value():\n self._outputs[name] = type_", "response": "Get the types of all current exchange items supposed to return\n the values of Parameter or Sequence objects or the time series\n of IOSequence objects or the time series\n of IOParameter objects or the time series\n of IOSequence objects."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchanges the current simulation |Timegrid|.", "response": "def POST_timegrid(self) -> None:\n \"\"\"Change the current simulation |Timegrid|.\"\"\"\n init = hydpy.pub.timegrids.init\n sim = hydpy.pub.timegrids.sim\n sim.firstdate = self._inputs['firstdate']\n sim.lastdate = self._inputs['lastdate']\n state.idx1 = init[sim.firstdate]\n state.idx2 = init[sim.lastdate]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GET_parameteritemvalues(self) -> None:\n for item in state.parameteritems:\n self._outputs[item.name] = item.value", "response": "Get the values of all |ChangeItem| objects handling |Parameter| objects."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the values of all |ChangeItem| objects handling |StateSequence| or |LogSequence| objects handling |StateSequence| or |LogSequence| objects handling |StateSequence| or |LogSequence| objects handling |StateSequence| or |LogSequence| objects.", "response": "def GET_conditionitemvalues(self) -> None:\n \"\"\"Get the values of all |ChangeItem| objects handling |StateSequence|\n or |LogSequence| objects.\"\"\"\n for item in state.conditionitems:\n self._outputs[item.name] = item.value"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GET_getitemvalues(self) -> None:\n for item in state.getitems:\n for name, value in item.yield_name2value(state.idx1, state.idx2):\n self._outputs[name] = value", "response": "Get the values of all Variable objects observed by the\n current objects."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GET_load_conditionvalues(self) -> None:\n try:\n state.hp.conditions = state.conditions[self._id][state.idx1]\n except KeyError:\n if state.idx1:\n self._statuscode = 500\n raise RuntimeError(\n f'Conditions for ID `{self._id}` and time point '\n f'`{hydpy.pub.timegrids.sim.firstdate}` are required, '\n f'but have not been calculated so far.')\n else:\n state.hp.conditions = state.init_conditions", "response": "Assign the |StateSequence| or |LogSequence| object values available to the current |HydPy| instance."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GET_save_conditionvalues(self) -> None:\n state.conditions[self._id] = state.conditions.get(self._id, {})\n state.conditions[self._id][state.idx2] = state.hp.conditions", "response": "Save the |StateSequence| and |LogSequence| object values of the current simulation endpoint for the current simulation endpoint."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsave the values of those which are handling |Parameter| objects.", "response": "def GET_save_parameteritemvalues(self) -> None:\n \"\"\"Save the values of those |ChangeItem| objects which are\n handling |Parameter| objects.\"\"\"\n for item in state.parameteritems:\n state.parameteritemvalues[self._id][item.name] = item.value.copy()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GET_savedparameteritemvalues(self) -> None:\n dict_ = state.parameteritemvalues.get(self._id)\n if dict_ is None:\n self.GET_parameteritemvalues()\n else:\n for name, value in dict_.items():\n self._outputs[name] = value", "response": "Get the previously saved values of those |ChangeItem| objects which are handling |Parameter| objects."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GET_save_modifiedconditionitemvalues(self) -> None:\n for item in state.conditionitems:\n state.modifiedconditionitemvalues[self._id][item.name] = \\\n list(item.device2target.values())[0].value", "response": "Save modified condition items values to the state."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GET_savedmodifiedconditionitemvalues(self) -> None:\n dict_ = state.modifiedconditionitemvalues.get(self._id)\n if dict_ is None:\n self.GET_conditionitemvalues()\n else:\n for name, value in dict_.items():\n self._outputs[name] = value", "response": "Get the modified condition items values."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GET_save_getitemvalues(self) -> None:\n for item in state.getitems:\n for name, value in item.yield_name2value(state.idx1, state.idx2):\n state.getitemvalues[self._id][name] = value", "response": "Save the values of all current |GetItem| objects."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GET_savedgetitemvalues(self) -> None:\n dict_ = state.getitemvalues.get(self._id)\n if dict_ is None:\n self.GET_getitemvalues()\n else:\n for name, value in dict_.items():\n self._outputs[name] = value", "response": "Get the previously saved values of all |GetItem| objects."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GET_save_timegrid(self) -> None:\n state.timegrids[self._id] = copy.deepcopy(hydpy.pub.timegrids.sim)", "response": "Save the current simulation period."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GET_savedtimegrid(self) -> None:\n try:\n self._write_timegrid(state.timegrids[self._id])\n except KeyError:\n self._write_timegrid(hydpy.pub.timegrids.init)", "response": "Get the previously saved simulation period."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntrimming the value(s) of a |Variable| instance. Usually, users do not need to apply function |trim| directly. Instead, some |Variable| subclasses implement their own `trim` methods relying on function |trim|. Model developers should implement individual `trim` methods for their |Parameter| or |Sequence| subclasses when their boundary values depend on the actual project configuration (one example is soil moisture; its lowest possible value should possibly be zero in all cases, but its highest possible value could depend on another parameter defining the maximum storage capacity). For the following examples, we prepare a simple (not fully functional) |Variable| subclass, making use of function |trim| without any modifications. Function |trim| works slightly different for variables handling |float|, |int|, and |bool| values. We start with the most common content type |float|: >>> from hydpy.core.variabletools import trim, Variable >>> class Var(Variable): ... NDIM = 0 ... TYPE = float ... SPAN = 1.0, 3.0 ... trim = trim ... initinfo = 2.0, False ... __hydpy__connect_variable2subgroup__ = None First, we enable the printing of warning messages raised by function |trim|: >>> from hydpy import pub >>> pub.options.warntrim = True When not passing boundary values, function |trim| extracts them from class attribute `SPAN` of the given |Variable| instance, if available: >>> var = Var(None) >>> var.value = 2.0 >>> var.trim() >>> var var(2.0) >>> var.value = 0.0 >>> var.trim() Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `0.0` and `1.0`, respectively. >>> var var(1.0) >>> var.value = 4.0 >>> var.trim() Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `4.0` and `3.0`, respectively. >>> var var(3.0) In the examples above, outlier values are set to the respective boundary value, accompanied by suitable warning messages. For very tiny deviations, which might be due to precision problems only, outliers are trimmed but not reported: >>> var.value = 1.0 - 1e-15 >>> var == 1.0 False >>> trim(var) >>> var == 1.0 True >>> var.value = 3.0 + 1e-15 >>> var == 3.0 False >>> var.trim() >>> var == 3.0 True Use arguments `lower` and `upper` to override the (eventually) available `SPAN` entries: >>> var.trim(lower=4.0) Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `3.0` and `4.0`, respectively. >>> var.trim(upper=3.0) Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `4.0` and `3.0`, respectively. Function |trim| interprets both |None| and |numpy.nan| values as if no boundary value exists: >>> import numpy >>> var.value = 0.0 >>> var.trim(lower=numpy.nan) >>> var.value = 5.0 >>> var.trim(upper=numpy.nan) You can disable function |trim| via option |Options.trimvariables|: >>> with pub.options.trimvariables(False): ... var.value = 5.0 ... var.trim() >>> var var(5.0) Alternatively, you can omit the warning messages only: >>> with pub.options.warntrim(False): ... var.value = 5.0 ... var.trim() >>> var var(3.0) If a |Variable| subclass does not have (fixed) boundaries, give it either no `SPAN` attribute or a |tuple| containing |None| values: >>> del Var.SPAN >>> var.value = 5.0 >>> var.trim() >>> var var(5.0) >>> Var.SPAN = (None, None) >>> var.trim() >>> var var(5.0) The above examples deal with a 0-dimensional |Variable| subclass. The following examples repeat the most relevant examples for a 2-dimensional subclass: >>> Var.SPAN = 1.0, 3.0 >>> Var.NDIM = 2 >>> var.shape = 1, 3 >>> var.values = 2.0 >>> var.trim() >>> var.values = 0.0, 1.0, 2.0 >>> var.trim() Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `[[ 0. 1. 2.]]` and `[[ 1. 1. 2.]]`, \\ respectively. >>> var var([[1.0, 1.0, 2.0]]) >>> var.values = 2.0, 3.0, 4.0 >>> var.trim() Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `[[ 2. 3. 4.]]` and `[[ 2. 3. 3.]]`, \\ respectively. >>> var var([[2.0, 3.0, 3.0]]) >>> var.values = 1.0-1e-15, 2.0, 3.0+1e-15 >>> var.values == (1.0, 2.0, 3.0) array([[False, True, False]], dtype=bool) >>> var.trim() >>> var.values == (1.0, 2.0, 3.0) array([[ True, True, True]], dtype=bool) >>> var.values = 0.0, 2.0, 4.0 >>> var.trim(lower=numpy.nan, upper=numpy.nan) >>> var var([[0.0, 2.0, 4.0]]) >>> var.trim(lower=[numpy.nan, 3.0, 3.0]) Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `[[ 0. 2. 4.]]` and `[[ 0. 3. 3.]]`, \\ respectively. >>> var.values = 0.0, 2.0, 4.0 >>> var.trim(upper=[numpy.nan, 1.0, numpy.nan]) Traceback (most recent call last): ... UserWarning: For variable `var` at least one value needed to be trimmed. \\ The old and the new value(s) are `[[ 0. 2. 4.]]` and `[[ 1. 1. 4.]]`, \\ respectively. For |Variable| subclasses handling |float| values, setting outliers to the respective boundary value might often be an acceptable approach. However, this is often not the case for subclasses handling |int| values, which often serve as option flags (e.g. to enable/disable a certain hydrological process for different land-use types). Hence, function |trim| raises an exception instead of a warning and does not modify the wrong |int| value: >>> Var.TYPE = int >>> Var.NDIM = 0 >>> Var.SPAN = 1, 3 >>> var.value = 2 >>> var.trim() >>> var var(2) >>> var.value = 0 >>> var.trim() Traceback (most recent call last): ... ValueError: The value `0` of parameter `var` of element `?` is not valid. >>> var var(0) >>> var.value = 4 >>> var.trim() Traceback (most recent call last): ... ValueError: The value `4` of parameter `var` of element `?` is not valid. >>> var var(4) >>> from hydpy import INT_NAN >>> var.value = 0 >>> var.trim(lower=0) >>> var.trim(lower=INT_NAN) >>> var.value = 4 >>> var.trim(upper=4) >>> var.trim(upper=INT_NAN) >>> Var.SPAN = 1, None >>> var.value = 0 >>> var.trim() Traceback (most recent call last): ... ValueError: The value `0` of parameter `var` of element `?` is not valid. >>> var var(0) >>> Var.SPAN = None, 3 >>> var.value = 0 >>> var.trim() >>> var.value = 4 >>> var.trim() Traceback (most recent call last): ... ValueError: The value `4` of parameter `var` of element `?` is not valid. >>> del Var.SPAN >>> var.value = 0 >>> var.trim() >>> var.value = 4 >>> var.trim() >>> Var.SPAN = 1, 3 >>> Var.NDIM = 2 >>> var.shape = (1, 3) >>> var.values = 2 >>> var.trim() >>> var.values = 0, 1, 2 >>> var.trim() Traceback (most recent call last): ... ValueError: At least one value of parameter `var` of element `?` \\ is not valid. >>> var var([[0, 1, 2]]) >>> var.values = 2, 3, 4 >>> var.trim() Traceback (most recent call last): ... ValueError: At least one value of parameter `var` of element `?` \\ is not valid. >>> var var([[2, 3, 4]]) >>> var.values = 0, 0, 2 >>> var.trim(lower=[0, INT_NAN, 2]) >>> var.values = 2, 4, 4 >>> var.trim(upper=[2, INT_NAN, 4]) For |bool| values, defining outliers does not make much sense, which is why function |trim| does nothing when applied on variables handling |bool| values: >>> Var.TYPE = bool >>> var.trim() If function |trim| encounters an unmanageable type, it raises an exception like the following: >>> Var.TYPE = str >>> var.trim() Traceback (most recent call last): ... NotImplementedError: Method `trim` can only be applied on parameters \\ handling floating point, integer, or boolean values, but the \"value type\" \\ of parameter `var` is `str`. >>> pub.options.warntrim = False", "response": "def trim(self: 'Variable', lower=None, upper=None) -> None:\n \"\"\"Trim the value(s) of a |Variable| instance.\n\n Usually, users do not need to apply function |trim| directly.\n Instead, some |Variable| subclasses implement their own `trim`\n methods relying on function |trim|. Model developers should\n implement individual `trim` methods for their |Parameter| or\n |Sequence| subclasses when their boundary values depend on the\n actual project configuration (one example is soil moisture;\n its lowest possible value should possibly be zero in all cases,\n but its highest possible value could depend on another parameter\n defining the maximum storage capacity).\n\n For the following examples, we prepare a simple (not fully\n functional) |Variable| subclass, making use of function |trim|\n without any modifications. Function |trim| works slightly\n different for variables handling |float|, |int|, and |bool|\n values. We start with the most common content type |float|:\n\n >>> from hydpy.core.variabletools import trim, Variable\n >>> class Var(Variable):\n ... NDIM = 0\n ... TYPE = float\n ... SPAN = 1.0, 3.0\n ... trim = trim\n ... initinfo = 2.0, False\n ... __hydpy__connect_variable2subgroup__ = None\n\n First, we enable the printing of warning messages raised by function\n |trim|:\n\n >>> from hydpy import pub\n >>> pub.options.warntrim = True\n\n When not passing boundary values, function |trim| extracts them from\n class attribute `SPAN` of the given |Variable| instance, if available:\n\n >>> var = Var(None)\n >>> var.value = 2.0\n >>> var.trim()\n >>> var\n var(2.0)\n\n >>> var.value = 0.0\n >>> var.trim()\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `0.0` and `1.0`, respectively.\n >>> var\n var(1.0)\n\n >>> var.value = 4.0\n >>> var.trim()\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `4.0` and `3.0`, respectively.\n >>> var\n var(3.0)\n\n In the examples above, outlier values are set to the respective\n boundary value, accompanied by suitable warning messages. For very\n tiny deviations, which might be due to precision problems only,\n outliers are trimmed but not reported:\n\n >>> var.value = 1.0 - 1e-15\n >>> var == 1.0\n False\n >>> trim(var)\n >>> var == 1.0\n True\n\n >>> var.value = 3.0 + 1e-15\n >>> var == 3.0\n False\n >>> var.trim()\n >>> var == 3.0\n True\n\n Use arguments `lower` and `upper` to override the (eventually)\n available `SPAN` entries:\n\n >>> var.trim(lower=4.0)\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `3.0` and `4.0`, respectively.\n\n >>> var.trim(upper=3.0)\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `4.0` and `3.0`, respectively.\n\n Function |trim| interprets both |None| and |numpy.nan| values as if\n no boundary value exists:\n\n >>> import numpy\n >>> var.value = 0.0\n >>> var.trim(lower=numpy.nan)\n >>> var.value = 5.0\n >>> var.trim(upper=numpy.nan)\n\n You can disable function |trim| via option |Options.trimvariables|:\n\n >>> with pub.options.trimvariables(False):\n ... var.value = 5.0\n ... var.trim()\n >>> var\n var(5.0)\n\n Alternatively, you can omit the warning messages only:\n\n >>> with pub.options.warntrim(False):\n ... var.value = 5.0\n ... var.trim()\n >>> var\n var(3.0)\n\n If a |Variable| subclass does not have (fixed) boundaries, give it\n either no `SPAN` attribute or a |tuple| containing |None| values:\n\n >>> del Var.SPAN\n >>> var.value = 5.0\n >>> var.trim()\n >>> var\n var(5.0)\n\n >>> Var.SPAN = (None, None)\n >>> var.trim()\n >>> var\n var(5.0)\n\n The above examples deal with a 0-dimensional |Variable| subclass.\n The following examples repeat the most relevant examples for a\n 2-dimensional subclass:\n\n >>> Var.SPAN = 1.0, 3.0\n >>> Var.NDIM = 2\n >>> var.shape = 1, 3\n >>> var.values = 2.0\n >>> var.trim()\n\n >>> var.values = 0.0, 1.0, 2.0\n >>> var.trim()\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `[[ 0. 1. 2.]]` and `[[ 1. 1. 2.]]`, \\\nrespectively.\n >>> var\n var([[1.0, 1.0, 2.0]])\n\n >>> var.values = 2.0, 3.0, 4.0\n >>> var.trim()\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `[[ 2. 3. 4.]]` and `[[ 2. 3. 3.]]`, \\\nrespectively.\n >>> var\n var([[2.0, 3.0, 3.0]])\n\n >>> var.values = 1.0-1e-15, 2.0, 3.0+1e-15\n >>> var.values == (1.0, 2.0, 3.0)\n array([[False, True, False]], dtype=bool)\n >>> var.trim()\n >>> var.values == (1.0, 2.0, 3.0)\n array([[ True, True, True]], dtype=bool)\n\n >>> var.values = 0.0, 2.0, 4.0\n >>> var.trim(lower=numpy.nan, upper=numpy.nan)\n >>> var\n var([[0.0, 2.0, 4.0]])\n\n >>> var.trim(lower=[numpy.nan, 3.0, 3.0])\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `[[ 0. 2. 4.]]` and `[[ 0. 3. 3.]]`, \\\nrespectively.\n\n >>> var.values = 0.0, 2.0, 4.0\n >>> var.trim(upper=[numpy.nan, 1.0, numpy.nan])\n Traceback (most recent call last):\n ...\n UserWarning: For variable `var` at least one value needed to be trimmed. \\\nThe old and the new value(s) are `[[ 0. 2. 4.]]` and `[[ 1. 1. 4.]]`, \\\nrespectively.\n\n For |Variable| subclasses handling |float| values, setting outliers\n to the respective boundary value might often be an acceptable approach.\n However, this is often not the case for subclasses handling |int|\n values, which often serve as option flags (e.g. to enable/disable\n a certain hydrological process for different land-use types). Hence,\n function |trim| raises an exception instead of a warning and does\n not modify the wrong |int| value:\n\n >>> Var.TYPE = int\n >>> Var.NDIM = 0\n >>> Var.SPAN = 1, 3\n\n >>> var.value = 2\n >>> var.trim()\n >>> var\n var(2)\n\n >>> var.value = 0\n >>> var.trim()\n Traceback (most recent call last):\n ...\n ValueError: The value `0` of parameter `var` of element `?` is not valid.\n >>> var\n var(0)\n >>> var.value = 4\n >>> var.trim()\n Traceback (most recent call last):\n ...\n ValueError: The value `4` of parameter `var` of element `?` is not valid.\n >>> var\n var(4)\n\n >>> from hydpy import INT_NAN\n >>> var.value = 0\n >>> var.trim(lower=0)\n >>> var.trim(lower=INT_NAN)\n\n >>> var.value = 4\n >>> var.trim(upper=4)\n >>> var.trim(upper=INT_NAN)\n\n >>> Var.SPAN = 1, None\n >>> var.value = 0\n >>> var.trim()\n Traceback (most recent call last):\n ...\n ValueError: The value `0` of parameter `var` of element `?` is not valid.\n >>> var\n var(0)\n\n >>> Var.SPAN = None, 3\n >>> var.value = 0\n >>> var.trim()\n >>> var.value = 4\n >>> var.trim()\n Traceback (most recent call last):\n ...\n ValueError: The value `4` of parameter `var` of element `?` is not valid.\n\n >>> del Var.SPAN\n >>> var.value = 0\n >>> var.trim()\n >>> var.value = 4\n >>> var.trim()\n\n >>> Var.SPAN = 1, 3\n >>> Var.NDIM = 2\n >>> var.shape = (1, 3)\n >>> var.values = 2\n >>> var.trim()\n\n >>> var.values = 0, 1, 2\n >>> var.trim()\n Traceback (most recent call last):\n ...\n ValueError: At least one value of parameter `var` of element `?` \\\nis not valid.\n >>> var\n var([[0, 1, 2]])\n >>> var.values = 2, 3, 4\n >>> var.trim()\n Traceback (most recent call last):\n ...\n ValueError: At least one value of parameter `var` of element `?` \\\nis not valid.\n >>> var\n var([[2, 3, 4]])\n\n\n >>> var.values = 0, 0, 2\n >>> var.trim(lower=[0, INT_NAN, 2])\n\n >>> var.values = 2, 4, 4\n >>> var.trim(upper=[2, INT_NAN, 4])\n\n For |bool| values, defining outliers does not make much sense,\n which is why function |trim| does nothing when applied on\n variables handling |bool| values:\n\n >>> Var.TYPE = bool\n >>> var.trim()\n\n If function |trim| encounters an unmanageable type, it raises an\n exception like the following:\n\n >>> Var.TYPE = str\n >>> var.trim()\n Traceback (most recent call last):\n ...\n NotImplementedError: Method `trim` can only be applied on parameters \\\nhandling floating point, integer, or boolean values, but the \"value type\" \\\nof parameter `var` is `str`.\n\n >>> pub.options.warntrim = False\n \"\"\"\n if hydpy.pub.options.trimvariables:\n if lower is None:\n lower = self.SPAN[0]\n if upper is None:\n upper = self.SPAN[1]\n type_ = getattr(self, 'TYPE', float)\n if type_ is float:\n if self.NDIM == 0:\n _trim_float_0d(self, lower, upper)\n else:\n _trim_float_nd(self, lower, upper)\n elif type_ is int:\n if self.NDIM == 0:\n _trim_int_0d(self, lower, upper)\n else:\n _trim_int_nd(self, lower, upper)\n elif type_ is bool:\n pass\n else:\n raise NotImplementedError(\n f'Method `trim` can only be applied on parameters '\n f'handling floating point, integer, or boolean values, '\n f'but the \"value type\" of parameter `{self.name}` is '\n f'`{objecttools.classname(self.TYPE)}`.')"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_tolerance(values):\n tolerance = numpy.abs(values*1e-15)\n if hasattr(tolerance, '__setitem__'):\n tolerance[numpy.isinf(tolerance)] = 0.\n elif numpy.isinf(tolerance):\n tolerance = 0.\n return tolerance", "response": "Return some numerical accuracy to be expected for the\n given floating point values."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a function usable as a comparison method for class |Variable|.", "response": "def _compare_variables_function_generator(\n method_string, aggregation_func):\n \"\"\"Return a function usable as a comparison method for class |Variable|.\n\n Pass the specific method (e.g. `__eq__`) and the corresponding\n operator (e.g. `==`) as strings. Also pass either |numpy.all| or\n |numpy.any| for aggregating multiple boolean values.\n \"\"\"\n def comparison_function(self, other):\n \"\"\"Wrapper for comparison functions for class |Variable|.\"\"\"\n if self is other:\n return method_string in ('__eq__', '__le__', '__ge__')\n method = getattr(self.value, method_string)\n try:\n if hasattr(type(other), '__hydpy__get_value__'):\n other = other.__hydpy__get_value__()\n result = method(other)\n if result is NotImplemented:\n return result\n return aggregation_func(result)\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to compare variable '\n f'{objecttools.elementphrase(self)} with object '\n f'`{other}` of type `{objecttools.classname(other)}`')\n return comparison_function"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a valid string representation for the given |Variable| object.", "response": "def to_repr(self: Variable, values, brackets1d: Optional[bool] = False) \\\n -> str:\n \"\"\"Return a valid string representation for the given |Variable|\n object.\n\n Function |to_repr| it thought for internal purposes only, more\n specifically for defining string representations of subclasses\n of class |Variable| like the following:\n\n >>> from hydpy.core.variabletools import to_repr, Variable\n >>> class Var(Variable):\n ... NDIM = 0\n ... TYPE = int\n ... __hydpy__connect_variable2subgroup__ = None\n ... initinfo = 1.0, False\n >>> var = Var(None)\n >>> var.value = 2\n >>> var\n var(2)\n\n The following examples demonstrate all covered cases. Note that\n option `brackets1d` allows choosing between a \"vararg\" and an\n \"iterable\" string representation for 1-dimensional variables\n (the first one being the default):\n\n >>> print(to_repr(var, 2))\n var(2)\n\n >>> Var.NDIM = 1\n >>> var = Var(None)\n >>> var.shape = 3\n >>> print(to_repr(var, range(3)))\n var(0, 1, 2)\n >>> print(to_repr(var, range(3), True))\n var([0, 1, 2])\n >>> print(to_repr(var, range(30)))\n var(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,\n 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29)\n >>> print(to_repr(var, range(30), True))\n var([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,\n 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])\n\n >>> Var.NDIM = 2\n >>> var = Var(None)\n >>> var.shape = (2, 3)\n >>> print(to_repr(var, [range(3), range(3, 6)]))\n var([[0, 1, 2],\n [3, 4, 5]])\n >>> print(to_repr(var, [range(30), range(30, 60)]))\n var([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,\n 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29],\n [30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,\n 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]])\n \"\"\"\n prefix = f'{self.name}('\n if isinstance(values, str):\n string = f'{self.name}({values})'\n elif self.NDIM == 0:\n string = f'{self.name}({objecttools.repr_(values)})'\n elif self.NDIM == 1:\n if brackets1d:\n string = objecttools.assignrepr_list(values, prefix, 72) + ')'\n else:\n string = objecttools.assignrepr_values(\n values, prefix, 72) + ')'\n else:\n string = objecttools.assignrepr_list2(values, prefix, 72) + ')'\n return '\\n'.join(self.commentrepr + [string])"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nraise a |RuntimeError| if at least one of the required values of a variable object is not set.", "response": "def verify(self) -> None:\n \"\"\"Raises a |RuntimeError| if at least one of the required values\n of a |Variable| object is |None| or |numpy.nan|. The descriptor\n `mask` defines, which values are considered to be necessary.\n\n Example on a 0-dimensional |Variable|:\n\n >>> from hydpy.core.variabletools import Variable\n >>> class Var(Variable):\n ... NDIM = 0\n ... TYPE = float\n ... __hydpy__connect_variable2subgroup__ = None\n ... initinfo = 0.0, False\n >>> var = Var(None)\n >>> import numpy\n >>> var.shape = ()\n >>> var.value = 1.0\n >>> var.verify()\n >>> var.value = numpy.nan\n >>> var.verify()\n Traceback (most recent call last):\n ...\n RuntimeError: For variable `var`, 1 required value has not been set yet.\n\n Example on a 2-dimensional |Variable|:\n\n >>> Var.NDIM = 2\n >>> var = Var(None)\n >>> var.shape = (2, 3)\n >>> var.value = numpy.ones((2,3))\n >>> var.value[:, 1] = numpy.nan\n >>> var.verify()\n Traceback (most recent call last):\n ...\n RuntimeError: For variable `var`, 2 required values \\\nhave not been set yet.\n\n >>> Var.mask = var.mask\n >>> Var.mask[0, 1] = False\n >>> var.verify()\n Traceback (most recent call last):\n ...\n RuntimeError: For variable `var`, 1 required value has not been set yet.\n\n >>> Var.mask[1, 1] = False\n >>> var.verify()\n \"\"\"\n nmbnan: int = numpy.sum(numpy.isnan(\n numpy.array(self.value)[self.mask]))\n if nmbnan:\n if nmbnan == 1:\n text = 'value has'\n else:\n text = 'values have'\n raise RuntimeError(\n f'For variable {objecttools.devicephrase(self)}, '\n f'{nmbnan} required {text} not been set yet.')"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef average_values(self, *args, **kwargs) -> float:\n try:\n if not self.NDIM:\n return self.value\n mask = self.get_submask(*args, **kwargs)\n if numpy.any(mask):\n weights = self.refweights[mask]\n return numpy.sum(weights*self[mask])/numpy.sum(weights)\n return numpy.nan\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to calculate the mean value of variable '\n f'{objecttools.devicephrase(self)}')", "response": "A method that calculates the actual values of the |Variable| object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_submask(self, *args, **kwargs) -> masktools.CustomMask:\n if args or kwargs:\n masks = self.availablemasks\n mask = masktools.CustomMask(numpy.full(self.shape, False))\n for arg in args:\n mask = mask + self._prepare_mask(arg, masks)\n for key, value in kwargs.items():\n mask = mask + self._prepare_mask(key, masks, **value)\n if mask not in self.mask:\n raise ValueError(\n f'Based on the arguments `{args}` and `{kwargs}` '\n f'the mask `{repr(mask)}` has been determined, '\n f'which is not a submask of `{repr(self.mask)}`.')\n else:\n mask = self.mask\n return mask", "response": "Get a sub - mask of the object."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef commentrepr(self) -> List[str]:\n if hydpy.pub.options.reprcomments:\n return [f'# {line}' for line in\n textwrap.wrap(objecttools.description(self), 72)]\n return []", "response": "A list with comments for making string representations\n more informative."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd a table into a worksheet.", "response": "def AddTable(p_workSheet = None, p_headerDict = None, p_startColumn = 1, p_startRow = 1, p_headerHeight = None, p_data = None, p_mainTable = False, p_conditionalFormatting = None, p_tableStyleInfo = None, p_withFilters = True):\n \"\"\"Insert a table in a given worksheet.\n\n Args:\n p_workSheet (openpyxl.worksheet.worksheet.Worksheet): the worksheet where the table will be inserted. Defaults to None.\n p_headerDict (collections.OrderedDict): an ordered dict that contains table header columns.\n Notes:\n Each entry is in the following form:\n Key: Name of the column to be searched in p_data.Columns.\n Value: Spartacus.Report.Field instance.\n Examples:\n p_headerDict = collections.OrderedDict([\n (\n 'field_one',\n Field(\n p_name = 'Code',\n p_width = 15,\n p_data = Data(\n p_type = 'int'\n )\n )\n ),\n (\n 'field_two',\n Field(\n p_name = 'Result',\n p_width = 15,\n p_data = Data(\n p_type = 'int_formula'\n )\n )\n )\n ])\n p_startColumn (int): the column number where the table should start. Defaults to 1.\n Notes:\n Must be a positive integer.\n p_startRow (int): the row number where the table should start. Defaults to 1.\n Notes:\n Must be a positive integer.\n p_headerHeight (float): the header row height in pt. Defaults to None.\n Notes:\n Must be a non-negative number or None.\n p_data (Spartacus.Database.DataTable): the datatable that contains the data that will be inserted into the excel table. Defaults to None.\n Notes:\n If the corresponding column data type in p_headerDict is some kind of formula, then below wildcards can be used:\n #row#: the current row.\n #column_columname#: will be replaced by the letter of the column.\n Examples:\n p_data = Spartacus.Database.DataTable that contains:\n Columns: ['field_one', 'field_two'].\n Rows: [\n [\n 'HAHAHA',\n '=if(#column_field_one##row# = \"HAHAHA\", 1, 0)'\n ],\n [\n 'HEHEHE',\n '=if(#column_field_one##row# = \"HAHAHA\", 1, 0)'\n ]\n ]\n p_mainTable (bool): if this table is the main table of the current worksheet. Defaults to False.\n Notes:\n If it's the main table, then it will consider p_width, p_hidden and freeze panes in the first table row. The 3 parameters are ignored otherwise.\n p_conditionalFormatting (Spartacus.Report.ConditionalFormatting): a conditional formatting that should be applied to data rows. Defaults to None.\n Notes:\n Will be applied to all data rows of this table.\n A wildcard can be used and be replaced properly:\n #row#: the current data row.\n #column_columname#: will be replaced by the letter of the column.\n Examples:\n p_conditionalFormatting = ConditionalFormatting(\n p_formula = '$Y#row# = 2',\n p_differentialStyle = openpyxl.styles.differential.DifferentialStyle(\n fill = openpyxl.styles.PatternFill(\n bgColor = 'D3D3D3'\n )\n )\n )\n p_tableStyleInfo (openpyxl.worksheet.table.TableStyleInfo): a style to be applied to this table. Defaults to None.\n Notes:\n Will not be applied to summaries, if any.\n Examples:\n p_tableStyleInfo = openpyxl.worksheet.table.TableStyleInfo(\n name = 'TableStyleMedium23',\n showFirstColumn = True,\n showLastColumn = True,\n showRowStripes = True,\n showColumnStripes = False\n )\n\n p_withFilters (bool): if the table must contain auto-filters.\n\n Yields:\n int: Every 1000 lines inserted into the table, yields actual line number.\n\n Raises:\n Spartacus.Report.Exception: custom exceptions occurred in this script.\n \"\"\"\n\n if not isinstance(p_workSheet, openpyxl.worksheet.worksheet.Worksheet):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_workSheet\" must be of type \"openpyxl.worksheet.worksheet.Worksheet\".')\n\n if not isinstance(p_headerDict, collections.OrderedDict):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_headerDict\" must be of type \"collections.OrderedDict\".')\n\n if not isinstance(p_startColumn, int):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_startColumn\" must be of type \"int\".')\n\n if p_startColumn < 1:\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_startColumn\" must be a positive integer.')\n\n if not isinstance(p_startRow, int):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_startRow\" must be of type \"int\".')\n\n if p_startRow < 1:\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_startRow\" must be a positive integer.')\n\n if p_headerHeight is not None and not isinstance(p_headerHeight, int) and not isinstance(p_headerHeight, float):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_headerHeight\" must be None or of type \"int\" or \"float\".')\n\n if not isinstance(p_data, Spartacus.Database.DataTable):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_data\" must be of type \"Spartacus.Database.DataTable\".')\n\n if not isinstance(p_mainTable, bool):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_mainTable\" must be of type \"bool\".')\n\n if p_conditionalFormatting is not None and not isinstance(p_conditionalFormatting, ConditionalFormatting):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_conditionalFormatting\" must be None or of type \"Spartacus.Report.ConditionalFormatting\".')\n\n if p_tableStyleInfo is not None and not isinstance(p_tableStyleInfo, openpyxl.worksheet.table.TableStyleInfo):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_tableStyleInfo\" must be None or of type \"openpyxl.worksheet.table.TableStyleInfo\".')\n\n if p_withFilters is not None and not isinstance(p_withFilters, bool):\n raise Spartacus.Report.Exception('Error during execution of method \"Static.AddTable\": Parameter \"p_withFilters\" must be None or of type \"bool\".')\n\n #Format Header\n if p_headerHeight is not None:\n p_workSheet.row_dimensions[p_startRow].height = p_headerHeight\n\n v_headerList = list(p_headerDict.keys())\n\n for i in range(len(v_headerList)):\n v_header = p_headerDict[v_headerList[i]]\n v_letter = openpyxl.utils.get_column_letter(i + p_startColumn)\n\n v_cell = p_workSheet['{0}{1}'.format(v_letter, p_startRow)]\n v_cell.value = v_header.name\n\n if p_mainTable:\n p_workSheet.column_dimensions[v_letter].width = v_header.width\n p_workSheet.column_dimensions[v_letter].hidden = v_header.hidden\n\n if v_header.comment is not None:\n v_cell.comment = v_header.comment\n\n if v_header.border is not None:\n v_cell.border = v_header.border\n\n if v_header.font is not None:\n v_cell.font = v_header.font\n\n if v_header.fill is not None:\n v_cell.fill = v_header.fill\n\n if v_header.alignment is not None:\n v_cell.alignment = v_header.alignment\n\n if p_mainTable:\n p_workSheet.freeze_panes = 'A{0}'.format(p_startRow + 1)\n\n #used in formula fields, if it's the case\n v_pattern = re.compile(r'#column_[^\\n\\r#]*#')\n\n v_line = 0\n\n #Fill content\n for v_row in p_data.Rows:\n v_line += 1\n\n for i in range(len(v_headerList)):\n v_headerData = p_headerDict[v_headerList[i]].data\n v_letter = openpyxl.utils.get_column_letter(i + p_startColumn)\n\n v_cell = p_workSheet['{0}{1}'.format(v_letter, v_line + p_startRow)] #Plus p_startRow to \"jump\" report header lines\n\n if v_headerData.border is not None:\n v_cell.border = v_headerData.border\n\n if v_headerData.font is not None:\n v_cell.font = v_headerData.font\n\n if v_headerData.fill is not None:\n v_cell.fill = v_headerData.fill\n\n if v_headerData.alignment is not None:\n v_cell.alignment = v_headerData.alignment\n\n if v_headerData.type == 'int':\n v_key = str(v_row[v_headerList[i]])\n\n if v_key in v_headerData.valueMapping:\n v_cell.value = v_headerData.valueMapping[v_key]\n else:\n try:\n v_cell.value = int(v_row[v_headerList[i]])\n except (Exception, TypeError, ValueError):\n v_cell.value = v_row[v_headerList[i]] if v_row[v_headerList[i]] is not None else ''\n\n v_cell.number_format = '0'\n elif v_headerData.type == 'float':\n v_key = str(v_row[v_headerList[i]])\n\n if v_key in v_headerData.valueMapping:\n v_cell.value = v_headerData.valueMapping[v_key]\n else:\n try:\n v_cell.value = float(v_row[v_headerList[i]])\n except (Exception, TypeError, ValueError):\n v_cell.value = v_row[v_headerList[i]] if v_row[v_headerList[i]] is not None else ''\n\n v_cell.number_format = '#,##0.00'\n elif v_headerData.type == 'float4':\n v_key = str(v_row[v_headerList[i]])\n\n if v_key in v_headerData.valueMapping:\n v_cell.value = v_headerData.valueMapping[v_key]\n else:\n try:\n v_cell.value = float(v_row[v_headerList[i]])\n except (Exception, TypeError, ValueError):\n v_cell.value = v_row[v_headerList[i]] if v_row[v_headerList[i]] is not None else ''\n\n v_cell.number_format = '#,##0.0000'\n elif v_headerData.type == 'percent':\n v_key = str(v_row[v_headerList[i]])\n\n if v_key in v_headerData.valueMapping:\n v_cell.value = v_headerData.valueMapping[v_key]\n else:\n try:\n v_cell.value = float(v_row[v_headerList[i]])\n except (Exception, TypeError, ValueError):\n v_cell.value = v_row[v_headerList[i]] if v_row[v_headerList[i]] is not None else ''\n\n v_cell.number_format = '0.00%'\n elif v_headerData.type == 'date':\n v_key = str(v_row[v_headerList[i]])\n\n if v_key in v_headerData.valueMapping:\n v_cell.value = v_headerData.valueMapping[v_key]\n else:\n v_cell.value = v_row[v_headerList[i]] if v_row[v_headerList[i]] is not None else ''\n\n v_cell.number_format = 'DD/MM/YYYY'\n elif v_headerData.type == 'str':\n v_key = str(v_row[v_headerList[i]])\n\n if v_key in v_headerData.valueMapping:\n v_cell.value = v_headerData.valueMapping[v_key]\n else:\n v_cell.value = v_row[v_headerList[i]] if v_row[v_headerList[i]] is not None else ''\n elif v_headerData.type == 'bool':\n v_key = str(v_row[v_headerList[i]])\n\n if v_key in v_headerData.valueMapping:\n v_cell.value = v_headerData.valueMapping[v_key]\n else:\n try:\n v_cell.value = bool(v_row[v_headerList[i]]) if v_row[v_headerList[i]] is not None and str(v_row[v_headerList[i]]).strip() != '' else ''\n except (Exception, TypeError, ValueError):\n v_cell.value = v_row[v_headerList[i]] if v_row[v_headerList[i]] is not None else ''\n if v_headerData.type == 'int_formula':\n v_value = v_row[v_headerList[i]].replace('#row#', str(p_startRow + v_line))\n v_match = re.search(v_pattern, v_value)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_value[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_value = v_value[:v_start] + v_matchColumn + v_value[v_end:]\n v_match = re.search(v_pattern, v_value)\n\n v_cell.value = v_value\n v_cell.number_format = '0'\n elif v_headerData.type == 'float_formula':\n v_value = v_row[v_headerList[i]].replace('#row#', str(p_startRow + v_line))\n v_match = re.search(v_pattern, v_value)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_value[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_value = v_value[:v_start] + v_matchColumn + v_value[v_end:]\n v_match = re.search(v_pattern, v_value)\n\n v_cell.value = v_value\n v_cell.number_format = '#,##0.00'\n elif v_headerData.type == 'float4_formula':\n v_value = v_row[v_headerList[i]].replace('#row#', str(p_startRow + v_line))\n v_match = re.search(v_pattern, v_value)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_value[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_value = v_value[:v_start] + v_matchColumn + v_value[v_end:]\n v_match = re.search(v_pattern, v_value)\n\n v_cell.value = v_value\n v_cell.number_format = '#,##0.0000'\n elif v_headerData.type == 'percent_formula':\n v_value = v_row[v_headerList[i]].replace('#row#', str(p_startRow + v_line))\n v_match = re.search(v_pattern, v_value)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_value[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_value = v_value[:v_start] + v_matchColumn + v_value[v_end:]\n v_match = re.search(v_pattern, v_value)\n\n v_cell.value = v_value\n v_cell.number_format = '0.00%'\n elif v_headerData.type == 'date_formula':\n v_value = v_row[v_headerList[i]].replace('#row#', str(p_startRow + v_line))\n v_match = re.search(v_pattern, v_value)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_value[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_value = v_value[:v_start] + v_matchColumn + v_value[v_end:]\n v_match = re.search(v_pattern, v_value)\n\n v_cell.value = v_value\n v_cell.number_format = 'DD/MM/YYYY'\n elif v_headerData.type == 'str_formula':\n v_value = v_row[v_headerList[i]].replace('#row#', str(p_startRow + v_line))\n v_match = re.search(v_pattern, v_value)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_value[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_value = v_value[:v_start] + v_matchColumn + v_value[v_end:]\n v_match = re.search(v_pattern, v_value)\n\n v_cell.value = v_value\n\n if v_line % 1000 == 0:\n yield v_line\n\n v_lastLine = len(p_data.Rows) + p_startRow\n\n #Apply conditional formatting, if any\n if p_conditionalFormatting is not None:\n v_startLetter = openpyxl.utils.get_column_letter(p_startColumn)\n v_finalLetter = openpyxl.utils.get_column_letter(len(v_headerList) + p_startColumn - 1)\n\n v_formula = p_conditionalFormatting.formula.replace('#row#', str(p_startRow + 1))\n\n v_match = re.search(v_pattern, v_formula)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_formula[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_formula = v_formula[:v_start] + v_matchColumn + v_formula[v_end:]\n v_match = re.search(v_pattern, v_formula)\n\n v_rule = openpyxl.formatting.rule.Rule(\n type = 'expression',\n formula = [v_formula],\n dxf = p_conditionalFormatting.differentialStyle\n )\n\n p_workSheet.conditional_formatting.add(\n '{0}{1}:{2}{3}'.format(v_startLetter, p_startRow + 1, v_finalLetter, v_lastLine),\n v_rule\n )\n\n #Build Summary\n for i in range(len(v_headerList)):\n v_headerSummaryList = p_headerDict[v_headerList[i]].summaryList\n\n for v_headerSummary in v_headerSummaryList:\n v_letter = openpyxl.utils.get_column_letter(i + p_startColumn)\n\n v_index = p_startRow - 1\n\n if v_headerSummary.index < 0:\n v_index = p_startRow + v_headerSummary.index\n elif v_headerSummary.index > 0:\n v_index = v_lastLine + v_headerSummary.index\n\n v_value = v_headerSummary.function.replace('#column#', v_letter).replace('#start_row#', str(p_startRow + 1)).replace('#end_row#', str(v_lastLine))\n\n v_match = re.search(v_pattern, v_value)\n\n while v_match is not None:\n v_start = v_match.start()\n v_end = v_match.end()\n v_matchColumn = openpyxl.utils.get_column_letter(p_startColumn + v_headerList.index(v_value[v_start + 8 : v_end - 1])) #Discard starting #column_ and ending # in match\n v_value = v_value[:v_start] + v_matchColumn + v_value[v_end:]\n v_match = re.search(v_pattern, v_value)\n\n v_cell = p_workSheet['{0}{1}'.format(v_letter, v_index)]\n v_cell.value = v_value\n\n if v_headerSummary.border is not None:\n v_cell.border = v_headerSummary.border\n\n if v_headerSummary.font is not None:\n v_cell.font = v_headerSummary.font\n\n if v_headerSummary.fill is not None:\n v_cell.fill = v_headerSummary.fill\n\n if v_headerSummary.type == 'int':\n v_cell.number_format = '0'\n elif v_headerSummary.type == 'float':\n v_cell.number_format = '#,##0.00'\n elif v_headerSummary.type == 'float4':\n v_cell.number_format = '#,##0.0000'\n elif v_headerSummary.type == 'percent':\n v_cell.number_format = '0.00%'\n\n #Create a new table and add it to worksheet\n v_name = 'Table_{0}_{1}'.format(p_workSheet.title.replace(' ', ''), len(p_workSheet._tables) + 1) #excel doesn't accept same displayName in more than one table.\n v_name = ''.join([c for c in v_name if c.isalnum()]) #Excel doesn't accept non-alphanumeric characters.\n\n v_table = openpyxl.worksheet.table.Table(\n displayName = v_name,\n ref = '{0}{1}:{2}{3}'.format(\n openpyxl.utils.get_column_letter(p_startColumn),\n p_startRow,\n openpyxl.utils.get_column_letter(p_startColumn + len(v_headerList) - 1),\n v_lastLine\n )\n )\n\n if p_tableStyleInfo is not None:\n v_table.tableStyleInfo = p_tableStyleInfo\n\n if not p_withFilters:\n v_table.headerRowCount = 0\n\n p_workSheet.add_table(v_table)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_controlfileheader(\n model: Union[str, 'modeltools.Model'],\n parameterstep: timetools.PeriodConstrArg = None,\n simulationstep: timetools.PeriodConstrArg = None) -> str:\n \"\"\"Return the header of a regular or auxiliary parameter control file.\n\n The header contains the default coding information, the import command\n for the given model and the actual parameter and simulation step sizes.\n\n The first example shows that, if you pass the model argument as a\n string, you have to take care that this string makes sense:\n\n >>> from hydpy.core.parametertools import get_controlfileheader, Parameter\n >>> from hydpy import Period, prepare_model, pub, Timegrids, Timegrid\n >>> print(get_controlfileheader(model='no model class',\n ... parameterstep='-1h',\n ... simulationstep=Period('1h')))\n # -*- coding: utf-8 -*-\n \n from hydpy.models.no model class import *\n \n simulationstep('1h')\n parameterstep('-1h')\n \n \n\n The second example shows the saver option to pass the proper model\n object. It also shows that function |get_controlfileheader| tries\n to gain the parameter and simulation step sizes from the global\n |Timegrids| object contained in the module |pub| when necessary:\n\n >>> model = prepare_model('lland_v1')\n >>> _ = Parameter.parameterstep('1d')\n >>> pub.timegrids = '2000.01.01', '2001.01.01', '1h'\n >>> print(get_controlfileheader(model=model))\n # -*- coding: utf-8 -*-\n \n from hydpy.models.lland_v1 import *\n \n simulationstep('1h')\n parameterstep('1d')\n \n \n \"\"\"\n with Parameter.parameterstep(parameterstep):\n if simulationstep is None:\n simulationstep = Parameter.simulationstep\n else:\n simulationstep = timetools.Period(simulationstep)\n return (f\"# -*- coding: utf-8 -*-\\n\\n\"\n f\"from hydpy.models.{model} import *\\n\\n\"\n f\"simulationstep('{simulationstep}')\\n\"\n f\"parameterstep('{Parameter.parameterstep}')\\n\\n\")", "response": "Return the header of a regular or auxiliary parameter control file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nassigning docstrings to the constants handled by |Constants| to make them available in the interactive mode of Python.", "response": "def _prepare_docstrings(self, frame):\n \"\"\"Assign docstrings to the constants handled by |Constants|\n to make them available in the interactive mode of Python.\"\"\"\n if config.USEAUTODOC:\n filename = inspect.getsourcefile(frame)\n with open(filename) as file_:\n sources = file_.read().split('\"\"\"')\n for code, doc in zip(sources[::2], sources[1::2]):\n code = code.strip()\n key = code.split('\\n')[-1].split()[0]\n value = self.get(key)\n if value:\n value.__doc__ = doc"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nupdate the internal state of the secondary parameters.", "response": "def update(self) -> None:\n \"\"\"Call method |Parameter.update| of all \"secondary\" parameters.\n\n Directly after initialisation, neither the primary (`control`)\n parameters nor the secondary (`derived`) parameters of\n application model |hstream_v1| are ready for usage:\n\n >>> from hydpy.models.hstream_v1 import *\n >>> parameterstep('1d')\n >>> simulationstep('1d')\n >>> derived\n nmbsegments(?)\n c1(?)\n c3(?)\n c2(?)\n\n Trying to update the values of the secondary parameters while the\n primary ones are still not defined, raises errors like the following:\n\n >>> model.parameters.update()\n Traceback (most recent call last):\n ...\n AttributeError: While trying to update parameter ``nmbsegments` \\\nof element `?``, the following error occurred: For variable `lag`, \\\nno value has been defined so far.\n\n With proper values both for parameter |hstream_control.Lag| and\n |hstream_control.Damp|, updating the derived parameters succeeds:\n\n >>> lag(0.0)\n >>> damp(0.0)\n >>> model.parameters.update()\n >>> derived\n nmbsegments(0)\n c1(0.0)\n c3(0.0)\n c2(1.0)\n \"\"\"\n for subpars in self.secondary_subpars:\n for par in subpars:\n try:\n par.update()\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to update parameter '\n f'`{objecttools.elementphrase(par)}`')"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef save_controls(self, filepath: Optional[str] = None,\n parameterstep: timetools.PeriodConstrArg = None,\n simulationstep: timetools.PeriodConstrArg = None,\n auxfiler: 'auxfiletools.Auxfiler' = None):\n \"\"\"Write the control parameters to file.\n\n Usually, a control file consists of a header (see the documentation\n on the method |get_controlfileheader|) and the string representations\n of the individual |Parameter| objects handled by the `control`\n |SubParameters| object.\n\n The main functionality of method |Parameters.save_controls| is\n demonstrated in the documentation on the method |HydPy.save_controls|\n of class |HydPy|, which one would apply to write the parameter\n information of complete *HydPy* projects. However, to call\n |Parameters.save_controls| on individual |Parameters| objects\n offers the advantage to choose an arbitrary file path, as shown\n in the following example:\n\n >>> from hydpy.models.hstream_v1 import *\n >>> parameterstep('1d')\n >>> simulationstep('1h')\n >>> lag(1.0)\n >>> damp(0.5)\n\n >>> from hydpy import Open\n >>> with Open():\n ... model.parameters.save_controls('otherdir/otherfile.py')\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n otherdir/otherfile.py\n -------------------------------------\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1h')\n parameterstep('1d')\n \n lag(1.0)\n damp(0.5)\n \n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n Without a given file path and a proper project configuration,\n method |Parameters.save_controls| raises the following error:\n\n >>> model.parameters.save_controls()\n Traceback (most recent call last):\n ...\n RuntimeError: To save the control parameters of a model to a file, \\\nits filename must be known. This can be done, by passing a filename to \\\nfunction `save_controls` directly. But in complete HydPy applications, \\\nit is usally assumed to be consistent with the name of the element \\\nhandling the model.\n \"\"\"\n if self.control:\n variable2auxfile = getattr(auxfiler, str(self.model), None)\n lines = [get_controlfileheader(\n self.model, parameterstep, simulationstep)]\n with Parameter.parameterstep(parameterstep):\n for par in self.control:\n if variable2auxfile:\n auxfilename = variable2auxfile.get_filename(par)\n if auxfilename:\n lines.append(\n f\"{par.name}(auxfile='{auxfilename}')\\n\")\n continue\n lines.append(repr(par) + '\\n')\n text = ''.join(lines)\n if filepath:\n with open(filepath, mode='w', encoding='utf-8') as controlfile:\n controlfile.write(text)\n else:\n filename = objecttools.devicename(self)\n if filename == '?':\n raise RuntimeError(\n 'To save the control parameters of a model to a file, '\n 'its filename must be known. This can be done, by '\n 'passing a filename to function `save_controls` '\n 'directly. But in complete HydPy applications, it is '\n 'usally assumed to be consistent with the name of the '\n 'element handling the model.')\n hydpy.pub.controlmanager.save_file(filename, text)", "response": "Write the control parameters to file."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ntries to return the parameter values from the auxiliary control file with the given name.", "response": "def _get_values_from_auxiliaryfile(self, auxfile):\n \"\"\"Try to return the parameter values from the auxiliary control file\n with the given name.\n\n Things are a little complicated here. To understand this method, you\n should first take a look at the |parameterstep| function.\n \"\"\"\n try:\n frame = inspect.currentframe().f_back.f_back\n while frame:\n namespace = frame.f_locals\n try:\n subnamespace = {'model': namespace['model'],\n 'focus': self}\n break\n except KeyError:\n frame = frame.f_back\n else:\n raise RuntimeError(\n 'Cannot determine the corresponding model. Use the '\n '`auxfile` keyword in usual parameter control files only.')\n filetools.ControlManager.read2dict(auxfile, subnamespace)\n try:\n subself = subnamespace[self.name]\n except KeyError:\n raise RuntimeError(\n f'The selected file does not define value(s) for '\n f'parameter {self.name}')\n return subself.values\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to extract information for parameter '\n f'`{self.name}` from file `{auxfile}`')"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef initinfo(self) -> Tuple[Union[float, int, bool], bool]:\n init = self.INIT\n if (init is not None) and hydpy.pub.options.usedefaultvalues:\n with Parameter.parameterstep('1d'):\n return self.apply_timefactor(init), True\n return variabletools.TYPE2MISSINGVALUE[self.TYPE], False", "response": "The actual initial value of the given parameter."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfactor to adjust a new value of a time-dependent parameter. For a time-dependent parameter, its effective value depends on the simulation step size. Method |Parameter.get_timefactor| returns the fraction between the current simulation step size and the current parameter step size. .. testsetup:: >>> from hydpy import pub >>> del pub.timegrids >>> from hydpy.core.parametertools import Parameter >>> Parameter.simulationstep.delete() Period() Method |Parameter.get_timefactor| raises the following error when time information is not available: >>> from hydpy.core.parametertools import Parameter >>> Parameter.get_timefactor() Traceback (most recent call last): ... RuntimeError: To calculate the conversion factor for adapting the \\ values of the time-dependent parameters, you need to define both a \\ parameter and a simulation time step size first. One can define both time step sizes directly: >>> _ = Parameter.parameterstep('1d') >>> _ = Parameter.simulationstep('6h') >>> Parameter.get_timefactor() 0.25 As usual, the \"global\" simulation step size of the |Timegrids| object of module |pub| is prefered: >>> from hydpy import pub >>> pub.timegrids = '2000-01-01', '2001-01-01', '12h' >>> Parameter.get_timefactor() 0.5", "response": "def get_timefactor(cls) -> float:\n \"\"\"Factor to adjust a new value of a time-dependent parameter.\n\n For a time-dependent parameter, its effective value depends on the\n simulation step size. Method |Parameter.get_timefactor| returns\n the fraction between the current simulation step size and the\n current parameter step size.\n\n .. testsetup::\n\n >>> from hydpy import pub\n >>> del pub.timegrids\n >>> from hydpy.core.parametertools import Parameter\n >>> Parameter.simulationstep.delete()\n Period()\n\n Method |Parameter.get_timefactor| raises the following error\n when time information is not available:\n\n >>> from hydpy.core.parametertools import Parameter\n >>> Parameter.get_timefactor()\n Traceback (most recent call last):\n ...\n RuntimeError: To calculate the conversion factor for adapting the \\\nvalues of the time-dependent parameters, you need to define both a \\\nparameter and a simulation time step size first.\n\n One can define both time step sizes directly:\n\n >>> _ = Parameter.parameterstep('1d')\n >>> _ = Parameter.simulationstep('6h')\n >>> Parameter.get_timefactor()\n 0.25\n\n As usual, the \"global\" simulation step size of the |Timegrids|\n object of module |pub| is prefered:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2000-01-01', '2001-01-01', '12h'\n >>> Parameter.get_timefactor()\n 0.5\n \"\"\"\n try:\n parfactor = hydpy.pub.timegrids.parfactor\n except RuntimeError:\n if not (cls.parameterstep and cls.simulationstep):\n raise RuntimeError(\n f'To calculate the conversion factor for adapting '\n f'the values of the time-dependent parameters, '\n f'you need to define both a parameter and a simulation '\n f'time step size first.')\n else:\n date1 = timetools.Date('2000.01.01')\n date2 = date1 + cls.simulationstep\n parfactor = timetools.Timegrids(timetools.Timegrid(\n date1, date2, cls.simulationstep)).parfactor\n return parfactor(cls.parameterstep)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchanges and return the given value in accordance with the type of time - dependentence and the type of time - dependentence .", "response": "def apply_timefactor(cls, values):\n \"\"\"Change and return the given value(s) in accordance with\n |Parameter.get_timefactor| and the type of time-dependence\n of the actual parameter subclass.\n\n .. testsetup::\n\n >>> from hydpy import pub\n >>> del pub.timegrids\n\n For the same conversion factor returned by method\n |Parameter.get_timefactor|, method |Parameter.apply_timefactor|\n behaves differently depending on the `TIME` attribute of the\n respective |Parameter| subclass. We first prepare a parameter\n test class and define both the parameter and simulation step size:\n\n >>> from hydpy.core.parametertools import Parameter\n >>> class Par(Parameter):\n ... TIME = None\n >>> Par.parameterstep = '1d'\n >>> Par.simulationstep = '6h'\n\n |None| means the value(s) of the parameter are not time-dependent\n (e.g. maximum storage capacity). Hence, |Parameter.apply_timefactor|\n returns the original value(s):\n\n >>> Par.apply_timefactor(4.0)\n 4.0\n\n |True| means the effective parameter value is proportional to\n the simulation step size (e.g. travel time). Hence,\n |Parameter.apply_timefactor| returns a reduced value in the\n next example (where the simulation step size is smaller than\n the parameter step size):\n\n >>> Par.TIME = True\n >>> Par.apply_timefactor(4.0)\n 1.0\n\n |False| means the effective parameter value is inversely\n proportional to the simulation step size (e.g. storage\n coefficient). Hence, |Parameter.apply_timefactor| returns\n an increased value in the next example:\n\n >>> Par.TIME = False\n >>> Par.apply_timefactor(4.0)\n 16.0\n \"\"\"\n if cls.TIME is True:\n return values * cls.get_timefactor()\n if cls.TIME is False:\n return values / cls.get_timefactor()\n return values"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef revert_timefactor(cls, values):\n if cls.TIME is True:\n return values / cls.get_timefactor()\n if cls.TIME is False:\n return values * cls.get_timefactor()\n return values", "response": "This method reverts the timefactor of a parameter to a new value."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef compress_repr(self) -> Optional[str]:\n if not hasattr(self, 'value'):\n return '?'\n if not self:\n return f\"{self.NDIM * '['}{self.NDIM * ']'}\"\n unique = numpy.unique(self[self.mask])\n if sum(numpy.isnan(unique)) == len(unique.flatten()):\n unique = numpy.array([numpy.nan])\n else:\n unique = self.revert_timefactor(unique)\n if len(unique) == 1:\n return objecttools.repr_(unique[0])\n return None", "response": "Return the compressed representation of the current parameter value."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nworks as |Parameter. compress_repr| but returns a string with constant names instead of constant values.", "response": "def compress_repr(self) -> str:\n \"\"\"Works as |Parameter.compress_repr|, but returns a\n string with constant names instead of constant values.\n\n See the main documentation on class |NameParameter| for\n further information.\n \"\"\"\n string = super().compress_repr()\n if string in ('?', '[]'):\n return string\n if string is None:\n values = self.values\n else:\n values = [int(string)]\n invmap = {value: key for key, value in\n self.CONSTANTS.items()}\n result = ', '.join(\n invmap.get(value, repr(value)) for value in values)\n if len(self) > 255:\n result = f'[{result}]'\n return result"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef compress_repr(self) -> Optional[str]:\n string = super().compress_repr()\n if string is not None:\n return string\n results = []\n mask = self.mask\n refindices = mask.refindices.values\n for (key, value) in self.MODEL_CONSTANTS.items():\n if value in mask.RELEVANT_VALUES:\n unique = numpy.unique(self.values[refindices == value])\n unique = self.revert_timefactor(unique)\n length = len(unique)\n if length == 1:\n results.append(\n f'{key.lower()}={objecttools.repr_(unique[0])}')\n elif length > 1:\n return None\n return ', '.join(sorted(results))", "response": "Returns a string representation of the object."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nrefreshes the actual simulation values based on the toy - value pairs.", "response": "def refresh(self) -> None:\n \"\"\"Update the actual simulation values based on the toy-value pairs.\n\n Usually, one does not need to call refresh explicitly. The\n \"magic\" methods __call__, __setattr__, and __delattr__ invoke\n it automatically, when required.\n\n Instantiate a 1-dimensional |SeasonalParameter| object:\n\n >>> from hydpy.core.parametertools import SeasonalParameter\n >>> class Par(SeasonalParameter):\n ... NDIM = 1\n ... TYPE = float\n ... TIME = None\n >>> par = Par(None)\n >>> par.simulationstep = '1d'\n >>> par.shape = (None,)\n\n When a |SeasonalParameter| object does not contain any toy-value\n pairs yet, the method |SeasonalParameter.refresh| sets all actual\n simulation values to zero:\n\n >>> par.values = 1.\n >>> par.refresh()\n >>> par.values[0]\n 0.0\n\n When there is only one toy-value pair, its values are relevant\n for all actual simulation values:\n\n >>> par.toy_1 = 2. # calls refresh automatically\n >>> par.values[0]\n 2.0\n\n Method |SeasonalParameter.refresh| performs a linear interpolation\n for the central time points of each simulation time step. Hence,\n in the following example, the original values of the toy-value\n pairs do not show up:\n\n >>> par.toy_12_31 = 4.\n >>> from hydpy import round_\n >>> round_(par.values[0])\n 2.00274\n >>> round_(par.values[-2])\n 3.99726\n >>> par.values[-1]\n 3.0\n\n If one wants to preserve the original values in this example, one\n would have to set the corresponding toy instances in the middle of\n some simulation step intervals:\n\n >>> del par.toy_1\n >>> del par.toy_12_31\n >>> par.toy_1_1_12 = 2\n >>> par.toy_12_31_12 = 4.\n >>> par.values[0]\n 2.0\n >>> round_(par.values[1])\n 2.005479\n >>> round_(par.values[-2])\n 3.994521\n >>> par.values[-1]\n 4.0\n \"\"\"\n if not self:\n self.values[:] = 0.\n elif len(self) == 1:\n values = list(self._toy2values.values())[0]\n self.values[:] = self.apply_timefactor(values)\n else:\n for idx, date in enumerate(\n timetools.TOY.centred_timegrid(self.simulationstep)):\n values = self.interp(date)\n self.values[idx] = self.apply_timefactor(values)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef interp(self, date: timetools.Date) -> float:\n xnew = timetools.TOY(date)\n xys = list(self)\n for idx, (x_1, y_1) in enumerate(xys):\n if x_1 > xnew:\n x_0, y_0 = xys[idx-1]\n break\n else:\n x_0, y_0 = xys[-1]\n x_1, y_1 = xys[0]\n return y_0+(y_1-y_0)/(x_1-x_0)*(xnew-x_0)", "response": "Perform a linear value interpolation for the given date and return the result."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update(self) -> None:\n mask = self.mask\n weights = self.refweights[mask]\n self[~mask] = numpy.nan\n self[mask] = weights/numpy.sum(weights)", "response": "Update subclass of |RelSubweightsMixin| based on refweights."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef alternative_initvalue(self) -> Union[bool, int, float]:\n if self._alternative_initvalue is None:\n raise AttributeError(\n f'No alternative initial value for solver parameter '\n f'{objecttools.elementphrase(self)} has been defined so far.')\n else:\n return self._alternative_initvalue", "response": "A user - defined value to be used instead of the value of class\n constant INIT."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef update(self) -> None:\n indexarray = hydpy.pub.indexer.timeofyear\n self.shape = indexarray.shape\n self.values = indexarray", "response": "Update |Indexer. timeofyear| array of the |Indexer| object available in module |pub|."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the premises model that is installed with the given app_label. model_name.", "response": "def get_premises_model():\n \"\"\"\n Support for custom company premises model\n with developer friendly validation.\n \"\"\"\n try:\n app_label, model_name = PREMISES_MODEL.split('.')\n except ValueError:\n raise ImproperlyConfigured(\"OPENINGHOURS_PREMISES_MODEL must be of the\"\n \" form 'app_label.model_name'\")\n premises_model = get_model(app_label=app_label, model_name=model_name)\n if premises_model is None:\n raise ImproperlyConfigured(\"OPENINGHOURS_PREMISES_MODEL refers to\"\n \" model '%s' that has not been installed\"\n % PREMISES_MODEL)\n return premises_model"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a datetime. datetime object for the current request.", "response": "def get_now():\n \"\"\"\n Allows to access global request and read a timestamp from query.\n \"\"\"\n if not get_current_request:\n return datetime.datetime.now()\n request = get_current_request()\n if request:\n openinghours_now = request.GET.get('openinghours-now')\n if openinghours_now:\n return datetime.datetime.strptime(openinghours_now, '%Y%m%d%H%M%S')\n return datetime.datetime.now()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_closing_rule_for_now(location):\n now = get_now()\n\n if location:\n return ClosingRules.objects.filter(company=location,\n start__lte=now, end__gte=now)\n\n return Company.objects.first().closingrules_set.filter(start__lte=now,\n end__gte=now)", "response": "Returns QuerySet of ClosingRules that are currently valid for the given location."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning True if company is currently open.", "response": "def is_open(location, now=None):\n \"\"\"\n Is the company currently open? Pass \"now\" to test with a specific\n timestamp. Can be used stand-alone or as a helper.\n \"\"\"\n if now is None:\n now = get_now()\n\n if has_closing_rule_for_now(location):\n return False\n\n now_time = datetime.time(now.hour, now.minute, now.second)\n\n if location:\n ohs = OpeningHours.objects.filter(company=location)\n else:\n ohs = Company.objects.first().openinghours_set.all()\n for oh in ohs:\n is_open = False\n # start and end is on the same day\n if (oh.weekday == now.isoweekday() and\n oh.from_hour <= now_time and\n now_time <= oh.to_hour):\n is_open = oh\n\n # start and end are not on the same day and we test on the start day\n if (oh.weekday == now.isoweekday() and\n oh.from_hour <= now_time and\n ((oh.to_hour < oh.from_hour) and\n (now_time < datetime.time(23, 59, 59)))):\n is_open = oh\n\n # start and end are not on the same day and we test on the end day\n if (oh.weekday == (now.isoweekday() - 1) % 7 and\n oh.from_hour >= now_time and\n oh.to_hour >= now_time and\n oh.to_hour < oh.from_hour):\n is_open = oh\n # print \" 'Special' case after midnight\", oh\n\n if is_open is not False:\n return oh\n return False"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef next_time_open(location):\n if not is_open(location):\n now = get_now()\n now_time = datetime.time(now.hour, now.minute, now.second)\n found_opening_hours = False\n for i in range(8):\n l_weekday = (now.isoweekday() + i) % 7\n ohs = OpeningHours.objects.filter(company=location,\n weekday=l_weekday\n ).order_by('weekday',\n 'from_hour')\n\n if ohs.count():\n for oh in ohs:\n future_now = now + datetime.timedelta(days=i)\n # same day issue\n tmp_now = datetime.datetime(future_now.year,\n future_now.month,\n future_now.day,\n oh.from_hour.hour,\n oh.from_hour.minute,\n oh.from_hour.second)\n if tmp_now < now:\n tmp_now = now # be sure to set the bound correctly...\n if is_open(location, now=tmp_now):\n found_opening_hours = oh\n break\n if found_opening_hours is not False:\n return found_opening_hours, tmp_now\n return False, None", "response": "Returns the next possible opening hours object for the given company location."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef refweights(self):\n # pylint: disable=unsubscriptable-object\n # due to a pylint bug (see https://github.com/PyCQA/pylint/issues/870)\n return numpy.full(self.shape, 1./self.shape[0], dtype=float)", "response": "A |numpy| |numpy. ndarray| with equal weights for all segment\n junctions.."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a directory and optionally its path.", "response": "def add(self, directory, path=None) -> None:\n \"\"\"Add a directory and optionally its path.\"\"\"\n objecttools.valid_variable_identifier(directory)\n if path is None:\n path = directory\n setattr(self, directory, path)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef basepath(self) -> str:\n return os.path.abspath(\n os.path.join(self.projectdir, self.BASEDIR))", "response": "Absolute path pointing to the available working directories."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef availabledirs(self) -> Folder2Path:\n directories = Folder2Path()\n for directory in os.listdir(self.basepath):\n if not directory.startswith('_'):\n path = os.path.join(self.basepath, directory)\n if os.path.isdir(path):\n directories.add(directory, path)\n elif directory.endswith('.zip'):\n directories.add(directory[:-4], path)\n return directories", "response": "Returns a Folder2Path object containing the names and paths of the available working directories."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef currentdir(self) -> str:\n if self._currentdir is None:\n directories = self.availabledirs.folders\n if len(directories) == 1:\n self.currentdir = directories[0]\n elif self.DEFAULTDIR in directories:\n self.currentdir = self.DEFAULTDIR\n else:\n prefix = (f'The current working directory of the '\n f'{objecttools.classname(self)} object '\n f'has not been defined manually and cannot '\n f'be determined automatically:')\n if not directories:\n raise RuntimeError(\n f'{prefix} `{objecttools.repr_(self.basepath)}` '\n f'does not contain any available directories.')\n if self.DEFAULTDIR is None:\n raise RuntimeError(\n f'{prefix} `{objecttools.repr_(self.basepath)}` '\n f'does contain multiple available directories '\n f'({objecttools.enumeration(directories)}).')\n raise RuntimeError(\n f'{prefix} The default directory ({self.DEFAULTDIR}) '\n f'is not among the available directories '\n f'({objecttools.enumeration(directories)}).')\n return self._currentdir", "response": "Return the name of the current working directory of the file manager object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a list of the filenames contained in the current working directory.", "response": "def filenames(self) -> List[str]:\n \"\"\"Names of the files contained in the the current working directory.\n\n Files names starting with underscores are ignored:\n\n >>> from hydpy.core.filetools import FileManager\n >>> filemanager = FileManager()\n >>> filemanager.BASEDIR = 'basename'\n >>> filemanager.projectdir = 'projectname'\n >>> from hydpy import TestIO\n >>> with TestIO():\n ... filemanager.currentdir = 'testdir'\n ... open('projectname/basename/testdir/file1.txt', 'w').close()\n ... open('projectname/basename/testdir/file2.npy', 'w').close()\n ... open('projectname/basename/testdir/_file1.nc', 'w').close()\n ... filemanager.filenames\n ['file1.txt', 'file2.npy']\n \"\"\"\n return sorted(\n fn for fn in os.listdir(self.currentpath)\n if not fn.startswith('_'))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef filepaths(self) -> List[str]:\n path = self.currentpath\n return [os.path.join(path, name) for name in self.filenames]", "response": "Absolute path names of the files contained in the current\n working directory."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef zip_currentdir(self) -> None:\n with zipfile.ZipFile(f'{self.currentpath}.zip', 'w') as zipfile_:\n for filepath, filename in zip(self.filepaths, self.filenames):\n zipfile_.write(filename=filepath, arcname=filename)\n del self.currentdir", "response": "Pack the current working directory in a zip file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_files(self) -> selectiontools.Selections:\n devicetools.Node.clear_all()\n devicetools.Element.clear_all()\n selections = selectiontools.Selections()\n for (filename, path) in zip(self.filenames, self.filepaths):\n # Ensure both `Node` and `Element`start with a `fresh` memory.\n devicetools.Node.extract_new()\n devicetools.Element.extract_new()\n try:\n info = runpy.run_path(path)\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to load the network file `{path}`')\n try:\n node: devicetools.Node = info['Node']\n element: devicetools.Element = info['Element']\n selections += selectiontools.Selection(\n filename.split('.')[0],\n node.extract_new(),\n element.extract_new())\n except KeyError as exc:\n raise RuntimeError(\n f'The class {exc.args[0]} cannot be loaded from the '\n f'network file `{path}`.')\n\n selections += selectiontools.Selection(\n 'complete',\n info['Node'].query_all(),\n info['Element'].query_all())\n return selections", "response": "Load all network files of the current working directory and return a |selectiontools. Selection| object."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsave the |Selections| objects contained in the given |Selections| instance to separate network files.", "response": "def save_files(self, selections) -> None:\n \"\"\"Save the |Selection| objects contained in the given |Selections|\n instance to separate network files.\"\"\"\n try:\n currentpath = self.currentpath\n selections = selectiontools.Selections(selections)\n for selection in selections:\n if selection.name == 'complete':\n continue\n path = os.path.join(currentpath, selection.name+'.py')\n selection.save_networkfile(filepath=path)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to save selections `%s` into network files'\n % selections)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ndeleting the files corresponding to the given selections.", "response": "def delete_files(self, selections) -> None:\n \"\"\"Delete the network files corresponding to the given selections\n (e.g. a |list| of |str| objects or a |Selections| object).\"\"\"\n try:\n currentpath = self.currentpath\n for selection in selections:\n name = str(selection)\n if name == 'complete':\n continue\n if not name.endswith('.py'):\n name += '.py'\n path = os.path.join(currentpath, name)\n os.remove(path)\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to remove the network files of '\n f'selections `{selections}`')"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads the given file into the internal registry.", "response": "def load_file(self, element=None, filename=None, clear_registry=True):\n \"\"\"Return the namespace of the given file (and eventually of its\n corresponding auxiliary subfiles) as a |dict|.\n\n By default, the internal registry is cleared when a control file and\n all its corresponding auxiliary files have been loaded. You can\n change this behaviour by passing `False` for the `clear_registry`\n argument. This might decrease model initialization times\n significantly. But then it is your own responsibility to call\n method |ControlManager.clear_registry| when necessary (before\n reloading a changed control file).\n \"\"\"\n if not filename:\n filename = element.name\n type(self)._workingpath = self.currentpath\n info = {}\n if element:\n info['element'] = element\n try:\n self.read2dict(filename, info)\n finally:\n type(self)._workingpath = '.'\n if clear_registry:\n self._registry.clear()\n return info"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef read2dict(cls, filename, info):\n if not filename.endswith('.py'):\n filename += '.py'\n path = os.path.join(cls._workingpath, filename)\n try:\n if path not in cls._registry:\n with open(path) as file_:\n cls._registry[path] = file_.read()\n exec(cls._registry[path], {}, info)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to load the control file `%s`'\n % path)\n if 'model' not in info:\n raise IOError(\n 'Model parameters cannot be loaded from control file `%s`. '\n 'Please refer to the HydPy documentation on how to prepare '\n 'control files properly.'\n % path)", "response": "Read the control parameters from the given file and store them in the given |dict| object info."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsaves the given text under the given control filename and the the current path.", "response": "def save_file(self, filename, text):\n \"\"\"Save the given text under the given control filename and the\n current path.\"\"\"\n if not filename.endswith('.py'):\n filename += '.py'\n path = os.path.join(self.currentpath, filename)\n with open(path, 'w', encoding=\"utf-8\") as file_:\n file_.write(text)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_file(self, filename):\n _defaultdir = self.DEFAULTDIR\n try:\n if not filename.endswith('.py'):\n filename += '.py'\n try:\n self.DEFAULTDIR = (\n 'init_' + hydpy.pub.timegrids.sim.firstdate.to_string('os'))\n except KeyError:\n pass\n filepath = os.path.join(self.currentpath, filename)\n with open(filepath) as file_:\n return file_.read()\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to read the conditions file `%s`'\n % filename)\n finally:\n self.DEFAULTDIR = _defaultdir", "response": "Read and return the content of the given file."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef save_file(self, filename, text):\n _defaultdir = self.DEFAULTDIR\n try:\n if not filename.endswith('.py'):\n filename += '.py'\n try:\n self.DEFAULTDIR = (\n 'init_' + hydpy.pub.timegrids.sim.lastdate.to_string('os'))\n except AttributeError:\n pass\n path = os.path.join(self.currentpath, filename)\n with open(path, 'w', encoding=\"utf-8\") as file_:\n file_.write(text)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to write the conditions file `%s`'\n % filename)\n finally:\n self.DEFAULTDIR = _defaultdir", "response": "Save the given text under the given filename and the current path."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading data from an external data file an pass it to the given |IOSequence|.", "response": "def load_file(self, sequence):\n \"\"\"Load data from an \"external\" data file an pass it to\n the given |IOSequence|.\"\"\"\n try:\n if sequence.filetype_ext == 'npy':\n sequence.series = sequence.adjust_series(\n *self._load_npy(sequence))\n elif sequence.filetype_ext == 'asc':\n sequence.series = sequence.adjust_series(\n *self._load_asc(sequence))\n elif sequence.filetype_ext == 'nc':\n self._load_nc(sequence)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to load the external data of sequence %s'\n % objecttools.devicephrase(sequence))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_file(self, sequence, array=None):\n if array is None:\n array = sequence.aggregate_series()\n try:\n if sequence.filetype_ext == 'nc':\n self._save_nc(sequence, array)\n else:\n filepath = sequence.filepath_ext\n if ((array is not None) and\n (array.info['type'] != 'unmodified')):\n filepath = (f'{filepath[:-4]}_{array.info[\"type\"]}'\n f'{filepath[-4:]}')\n if not sequence.overwrite_ext and os.path.exists(filepath):\n raise OSError(\n f'Sequence {objecttools.devicephrase(sequence)} '\n f'is not allowed to overwrite the existing file '\n f'`{sequence.filepath_ext}`.')\n if sequence.filetype_ext == 'npy':\n self._save_npy(array, filepath)\n elif sequence.filetype_ext == 'asc':\n self._save_asc(array, filepath)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to save the external data of sequence %s'\n % objecttools.devicephrase(sequence))", "response": "Write the date stored in |IOSequence. series| into an external data file."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprepare a new |NetCDFInterface| object for reading data.", "response": "def open_netcdf_reader(self, flatten=False, isolate=False, timeaxis=1):\n \"\"\"Prepare a new |NetCDFInterface| object for reading data.\"\"\"\n self._netcdf_reader = netcdftools.NetCDFInterface(\n flatten=bool(flatten),\n isolate=bool(isolate),\n timeaxis=int(timeaxis))"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nopens a new |NetCDFInterface| object for writing data.", "response": "def open_netcdf_writer(self, flatten=False, isolate=False, timeaxis=1):\n \"\"\"Prepare a new |NetCDFInterface| object for writing data.\"\"\"\n self._netcdf_writer = netcdftools.NetCDFInterface(\n flatten=bool(flatten),\n isolate=bool(isolate),\n timeaxis=int(timeaxis))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_nkor_v1(self):\n con = self.parameters.control.fastaccess\n inp = self.sequences.inputs.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nhru):\n flu.nkor[k] = con.kg[k] * inp.nied", "response": "Adjust the precipitation values of the next non - zero precipitation."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadjusts the given air temperature values.", "response": "def calc_tkor_v1(self):\n \"\"\"Adjust the given air temperature values.\n\n Required control parameters:\n |NHRU|\n |KT|\n\n Required input sequence:\n |TemL|\n\n Calculated flux sequence:\n |TKor|\n\n Basic equation:\n :math:`TKor = KT + TemL`\n\n Example:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(3)\n >>> kt(-2.0, 0.0, 2.0)\n >>> inputs.teml(1.)\n >>> model.calc_tkor_v1()\n >>> fluxes.tkor\n tkor(-1.0, 1.0, 3.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n inp = self.sequences.inputs.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nhru):\n flu.tkor[k] = con.kt[k] + inp.teml"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the reference evapotranspiration after Turc - Wendling.", "response": "def calc_et0_v1(self):\n \"\"\"Calculate reference evapotranspiration after Turc-Wendling.\n\n Required control parameters:\n |NHRU|\n |KE|\n |KF|\n |HNN|\n\n Required input sequence:\n |Glob|\n\n Required flux sequence:\n |TKor|\n\n Calculated flux sequence:\n |ET0|\n\n Basic equation:\n :math:`ET0 = KE \\\\cdot\n \\\\frac{(8.64 \\\\cdot Glob+93 \\\\cdot KF) \\\\cdot (TKor+22)}\n {165 \\\\cdot (TKor+123) \\\\cdot (1 + 0.00019 \\\\cdot min(HNN, 600))}`\n\n Example:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> nhru(3)\n >>> ke(1.1)\n >>> kf(0.6)\n >>> hnn(200.0, 600.0, 1000.0)\n >>> inputs.glob = 200.0\n >>> fluxes.tkor = 15.0\n >>> model.calc_et0_v1()\n >>> fluxes.et0\n et0(3.07171, 2.86215, 2.86215)\n \"\"\"\n con = self.parameters.control.fastaccess\n inp = self.sequences.inputs.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nhru):\n flu.et0[k] = (con.ke[k]*(((8.64*inp.glob+93.*con.kf[k]) *\n (flu.tkor[k]+22.)) /\n (165.*(flu.tkor[k]+123.) *\n (1.+0.00019*min(con.hnn[k], 600.)))))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncorrect the given reference evapotranspiration and update the corresponding log sequence.", "response": "def calc_et0_wet0_v1(self):\n \"\"\"Correct the given reference evapotranspiration and update the\n corresponding log sequence.\n\n Required control parameters:\n |NHRU|\n |KE|\n |WfET0|\n\n Required input sequence:\n |PET|\n\n Calculated flux sequence:\n |ET0|\n\n Updated log sequence:\n |WET0|\n\n Basic equations:\n :math:`ET0_{new} = WfET0 \\\\cdot KE \\\\cdot PET +\n (1-WfET0) \\\\cdot ET0_{alt}`\n\n Example:\n\n Prepare four hydrological response units with different value\n combinations of parameters |KE| and |WfET0|:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> nhru(4)\n >>> ke(0.8, 1.2, 0.8, 1.2)\n >>> wfet0(2.0, 2.0, 0.2, 0.2)\n\n Note that the actual value of time dependend parameter |WfET0|\n is reduced due the difference between the given parameter and\n simulation time steps:\n\n >>> from hydpy import round_\n >>> round_(wfet0.values)\n 1.0, 1.0, 0.1, 0.1\n\n For the first two hydrological response units, the given |PET|\n value is modified by -0.4 mm and +0.4 mm, respectively. For the\n other two response units, which weight the \"new\" evaporation\n value with 10 %, |ET0| does deviate from the old value of |WET0|\n by -0.04 mm and +0.04 mm only:\n\n >>> inputs.pet = 2.0\n >>> logs.wet0 = 2.0\n >>> model.calc_et0_wet0_v1()\n >>> fluxes.et0\n et0(1.6, 2.4, 1.96, 2.04)\n >>> logs.wet0\n wet0([[1.6, 2.4, 1.96, 2.04]])\n \"\"\"\n con = self.parameters.control.fastaccess\n inp = self.sequences.inputs.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n for k in range(con.nhru):\n flu.et0[k] = (con.wfet0[k]*con.ke[k]*inp.pet +\n (1.-con.wfet0[k])*log.wet0[0, k])\n log.wet0[0, k] = flu.et0[k]"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate the stand precipitation and update the interception storage with the new state sequence.", "response": "def calc_nbes_inzp_v1(self):\n \"\"\"Calculate stand precipitation and update the interception storage\n accordingly.\n\n Required control parameters:\n |NHRU|\n |Lnk|\n\n Required derived parameter:\n |KInz|\n\n Required flux sequence:\n |NKor|\n\n Calculated flux sequence:\n |NBes|\n\n Updated state sequence:\n |Inzp|\n\n Additional requirements:\n |Model.idx_sim|\n\n Basic equation:\n :math:`NBes = \\\\Bigl \\\\lbrace\n {\n {PKor \\\\ | \\\\ Inzp = KInz}\n \\\\atop\n {0 \\\\ | \\\\ Inzp < KInz}\n }`\n\n Examples:\n\n Initialize five HRUs with different land usages:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(5)\n >>> lnk(SIED_D, FEUCHT, GLETS, FLUSS, SEE)\n\n Define |KInz| values for July the selected land usages directly:\n\n >>> derived.kinz.sied_d_jul = 2.0\n >>> derived.kinz.feucht_jul = 1.0\n >>> derived.kinz.glets_jul = 0.0\n >>> derived.kinz.fluss_jul = 1.0\n >>> derived.kinz.see_jul = 1.0\n\n Now we prepare a |MOY| object, that assumes that the first, second,\n and third simulation time steps are in June, July, and August\n respectively (we make use of the value defined above for July, but\n setting the values of parameter |MOY| this way allows for a more\n rigorous testing of proper indexing):\n\n >>> derived.moy.shape = 3\n >>> derived.moy = 5, 6, 7\n >>> model.idx_sim = 1\n\n The dense settlement (|SIED_D|), the wetland area (|FEUCHT|), and\n both water areas (|FLUSS| and |SEE|) start with a initial interception\n storage of 1/2 mm, the glacier (|GLETS|) and water areas (|FLUSS| and\n |SEE|) start with 0 mm. In the first example, actual precipition\n is 1 mm:\n\n >>> states.inzp = 0.5, 0.5, 0.0, 1.0, 1.0\n >>> fluxes.nkor = 1.0\n >>> model.calc_nbes_inzp_v1()\n >>> states.inzp\n inzp(1.5, 1.0, 0.0, 0.0, 0.0)\n >>> fluxes.nbes\n nbes(0.0, 0.5, 1.0, 0.0, 0.0)\n\n Only for the settled area, interception capacity is not exceeded,\n meaning no stand precipitation occurs. Note that it is common in\n define zero interception capacities for glacier areas, but not\n mandatory. Also note that the |KInz|, |Inzp| and |NKor| values\n given for both water areas are ignored completely, and |Inzp|\n and |NBes| are simply set to zero.\n\n If there is no precipitation, there is of course also no stand\n precipitation and interception storage remains unchanged:\n\n >>> states.inzp = 0.5, 0.5, 0.0, 0.0, 0.0\n >>> fluxes.nkor = 0.\n >>> model.calc_nbes_inzp_v1()\n >>> states.inzp\n inzp(0.5, 0.5, 0.0, 0.0, 0.0)\n >>> fluxes.nbes\n nbes(0.0, 0.0, 0.0, 0.0, 0.0)\n\n Interception capacities change discontinuously between consecutive\n months. This can result in little stand precipitation events in\n periods without precipitation:\n\n >>> states.inzp = 1.0, 0.0, 0.0, 0.0, 0.0\n >>> derived.kinz.sied_d_jul = 0.6\n >>> fluxes.nkor = 0.0\n >>> model.calc_nbes_inzp_v1()\n >>> states.inzp\n inzp(0.6, 0.0, 0.0, 0.0, 0.0)\n >>> fluxes.nbes\n nbes(0.4, 0.0, 0.0, 0.0, 0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if con.lnk[k] in (WASSER, FLUSS, SEE):\n flu.nbes[k] = 0.\n sta.inzp[k] = 0.\n else:\n flu.nbes[k] = \\\n max(flu.nkor[k]+sta.inzp[k] -\n der.kinz[con.lnk[k]-1, der.moy[self.idx_sim]], 0.)\n sta.inzp[k] += flu.nkor[k]-flu.nbes[k]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_evi_inzp_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if con.lnk[k] in (WASSER, FLUSS, SEE):\n flu.evi[k] = flu.evpo[k]\n sta.inzp[k] = 0.\n else:\n flu.evi[k] = min(flu.evpo[k], sta.inzp[k])\n sta.inzp[k] -= flu.evi[k]", "response": "Calculates interception evaporation and update the interception\n storage accordingly."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the frozen part of stand precipitation.", "response": "def calc_sbes_v1(self):\n \"\"\"Calculate the frozen part of stand precipitation.\n\n Required control parameters:\n |NHRU|\n |TGr|\n |TSp|\n\n Required flux sequences:\n |TKor|\n |NBes|\n\n Calculated flux sequence:\n |SBes|\n\n Examples:\n\n In the first example, the threshold temperature of seven hydrological\n response units is 0 \u00b0C and the corresponding temperature interval of\n mixed precipitation 2 \u00b0C:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(7)\n >>> tgr(0.0)\n >>> tsp(2.0)\n\n The value of |NBes| is zero above 1 \u00b0C and equal to the value of\n |NBes| below -1 \u00b0C. Between these temperature values, |NBes|\n decreases linearly:\n\n >>> fluxes.nbes = 4.0\n >>> fluxes.tkor = -10.0, -1.0, -0.5, 0.0, 0.5, 1.0, 10.0\n >>> model.calc_sbes_v1()\n >>> fluxes.sbes\n sbes(4.0, 4.0, 3.0, 2.0, 1.0, 0.0, 0.0)\n\n Note the special case of a zero temperature interval. With the\n actual temperature being equal to the threshold temperature, the\n the value of `sbes` is zero:\n\n >>> tsp(0.)\n >>> model.calc_sbes_v1()\n >>> fluxes.sbes\n sbes(4.0, 4.0, 4.0, 0.0, 0.0, 0.0, 0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nhru):\n if flu.nbes[k] <= 0.:\n flu.sbes[k] = 0.\n elif flu.tkor[k] >= (con.tgr[k]+con.tsp[k]/2.):\n flu.sbes[k] = 0.\n elif flu.tkor[k] <= (con.tgr[k]-con.tsp[k]/2.):\n flu.sbes[k] = flu.nbes[k]\n else:\n flu.sbes[k] = ((((con.tgr[k]+con.tsp[k]/2.)-flu.tkor[k]) /\n con.tsp[k])*flu.nbes[k])"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_wgtf_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nhru):\n if con.lnk[k] in (WASSER, FLUSS, SEE):\n flu.wgtf[k] = 0.\n else:\n flu.wgtf[k] = (\n max(con.gtf[k]*(flu.tkor[k]-con.treft[k]), 0) +\n max(con.cpwasser/con.rschmelz*(flu.tkor[k]-con.trefn[k]), 0.))", "response": "Calculates the potential snowmelt."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_schm_wats_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if con.lnk[k] in (WASSER, FLUSS, SEE):\n sta.wats[k] = 0.\n flu.schm[k] = 0.\n else:\n sta.wats[k] += flu.sbes[k]\n flu.schm[k] = min(flu.wgtf[k], sta.wats[k])\n sta.wats[k] -= flu.schm[k]", "response": "Calculates the actual amount of water melting within the snow cover."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the actual water release from the snow cover.", "response": "def calc_wada_waes_v1(self):\n \"\"\"Calculate the actual water release from the snow cover.\n\n Required control parameters:\n |NHRU|\n |Lnk|\n |PWMax|\n\n Required flux sequences:\n |NBes|\n\n Calculated flux sequence:\n |WaDa|\n\n Updated state sequence:\n |WAeS|\n\n Basic equations:\n :math:`\\\\frac{dWAeS}{dt} = NBes - WaDa`\n :math:`WAeS \\\\leq PWMax \\\\cdot WATS`\n\n Examples:\n\n For simplicity, the threshold parameter |PWMax| is set to a value\n of two for each of the six initialized HRUs. Thus, snow cover can\n hold as much liquid water as it contains frozen water. Stand\n precipitation is also always set to the same value, but the initial\n conditions of the snow cover are varied:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(6)\n >>> lnk(FLUSS, SEE, ACKER, ACKER, ACKER, ACKER)\n >>> pwmax(2.0)\n >>> fluxes.nbes = 1.0\n >>> states.wats = 0.0, 0.0, 0.0, 1.0, 1.0, 1.0\n >>> states.waes = 1.0, 1.0, 0.0, 1.0, 1.5, 2.0\n >>> model.calc_wada_waes_v1()\n >>> states.waes\n waes(0.0, 0.0, 0.0, 2.0, 2.0, 2.0)\n >>> fluxes.wada\n wada(1.0, 1.0, 1.0, 0.0, 0.5, 1.0)\n\n Note the special cases of the first two HRUs of type |FLUSS| and\n |SEE|. For water areas, stand precipitaton |NBes| is generally\n passed to |WaDa| and |WAeS| is set to zero. For all other land\n use classes (of which only |ACKER| is selected), only the amount\n of |NBes| exceeding the actual snow holding capacity is passed\n to |WaDa|.\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if con.lnk[k] in (WASSER, FLUSS, SEE):\n sta.waes[k] = 0.\n flu.wada[k] = flu.nbes[k]\n else:\n sta.waes[k] += flu.nbes[k]\n flu.wada[k] = max(sta.waes[k]-con.pwmax[k]*sta.wats[k], 0.)\n sta.waes[k] -= flu.wada[k]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef calc_evb_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if (con.lnk[k] in (VERS, WASSER, FLUSS, SEE)) or (con.nfk[k] <= 0.):\n flu.evb[k] = 0.\n else:\n d_temp = modelutils.exp(-con.grasref_r *\n sta.bowa[k]/con.nfk[k])\n flu.evb[k] = ((flu.evpo[k]-flu.evi[k]) * (1.-d_temp) /\n (1.+d_temp-2.*modelutils.exp(-con.grasref_r)))", "response": "Calculates the actual evaporation from the snow cover."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_qbb_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if ((con.lnk[k] in (VERS, WASSER, FLUSS, SEE)) or\n (sta.bowa[k] <= der.wb[k]) or (con.nfk[k] <= 0.)):\n flu.qbb[k] = 0.\n elif sta.bowa[k] <= der.wz[k]:\n flu.qbb[k] = con.beta[k]*(sta.bowa[k]-der.wb[k])\n else:\n flu.qbb[k] = (con.beta[k]*(sta.bowa[k]-der.wb[k]) *\n (1.+(con.fbeta[k]-1.)*((sta.bowa[k]-der.wz[k]) /\n (con.nfk[k]-der.wz[k]))))", "response": "Calculates the amount of base flow released from the soil."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate the first inflow component released from the soil.", "response": "def calc_qib1_v1(self):\n \"\"\"Calculate the first inflow component released from the soil.\n\n Required control parameters:\n |NHRU|\n |Lnk|\n |NFk|\n |DMin|\n\n Required derived parameter:\n |WB|\n\n Required state sequence:\n |BoWa|\n\n Calculated flux sequence:\n |QIB1|\n\n Basic equation:\n :math:`QIB1 = DMin \\\\cdot \\\\frac{BoWa}{NFk}`\n\n Examples:\n\n For water and sealed areas, no interflow is calculated (the first\n three HRUs are of type |FLUSS|, |SEE|, and |VERS|, respectively).\n No principal distinction is made between the remaining land use\n classes (arable land |ACKER| has been selected for the last five\n HRUs arbitrarily):\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> nhru(8)\n >>> lnk(FLUSS, SEE, VERS, ACKER, ACKER, ACKER, ACKER, ACKER)\n >>> dmax(10.0)\n >>> dmin(4.0)\n >>> nfk(101.0, 101.0, 101.0, 0.0, 101.0, 101.0, 101.0, 202.0)\n >>> derived.wb(10.0)\n >>> states.bowa = 10.1, 10.1, 10.1, 0.0, 0.0, 10.0, 10.1, 10.1\n\n Note the time dependence of parameter |DMin|:\n\n >>> dmin\n dmin(4.0)\n >>> dmin.values\n array([ 2., 2., 2., 2., 2., 2., 2., 2.])\n\n Compared to the calculation of |QBB|, the following results show\n some relevant differences:\n\n >>> model.calc_qib1_v1()\n >>> fluxes.qib1\n qib1(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2, 0.1)\n\n Firstly, as demonstrated with the help of the seventh and the\n eight HRU, the generation of the first interflow component |QIB1|\n depends on relative soil moisture. Secondly, as demonstrated with\n the help the sixth and seventh HRU, it starts abruptly whenever\n the slightest exceedance of the threshold parameter |WB| occurs.\n Such sharp discontinuouties are a potential source of trouble.\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if ((con.lnk[k] in (VERS, WASSER, FLUSS, SEE)) or\n (sta.bowa[k] <= der.wb[k])):\n flu.qib1[k] = 0.\n else:\n flu.qib1[k] = con.dmin[k]*(sta.bowa[k]/con.nfk[k])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_qib2_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nhru):\n if ((con.lnk[k] in (VERS, WASSER, FLUSS, SEE)) or\n (sta.bowa[k] <= der.wz[k]) or (con.nfk[k] <= der.wz[k])):\n flu.qib2[k] = 0.\n else:\n flu.qib2[k] = ((con.dmax[k]-con.dmin[k]) *\n ((sta.bowa[k]-der.wz[k]) /\n (con.nfk[k]-der.wz[k]))**1.5)", "response": "Calculates the first inflow component released from the soil."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef calc_qdb_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n aid = self.sequences.aides.fastaccess\n for k in range(con.nhru):\n if con.lnk[k] == WASSER:\n flu.qdb[k] = 0.\n elif ((con.lnk[k] in (VERS, FLUSS, SEE)) or\n (con.nfk[k] <= 0.)):\n flu.qdb[k] = flu.wada[k]\n else:\n if sta.bowa[k] < con.nfk[k]:\n aid.sfa[k] = (\n (1.-sta.bowa[k]/con.nfk[k])**(1./(con.bsf[k]+1.)) -\n (flu.wada[k]/((con.bsf[k]+1.)*con.nfk[k])))\n else:\n aid.sfa[k] = 0.\n aid.exz[k] = sta.bowa[k]+flu.wada[k]-con.nfk[k]\n flu.qdb[k] = aid.exz[k]\n if aid.sfa[k] > 0.:\n flu.qdb[k] += aid.sfa[k]**(con.bsf[k]+1.)*con.nfk[k]\n flu.qdb[k] = max(flu.qdb[k], 0.)", "response": "Calculates the QDB for the first four HRUs of the current state sequence and flux."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_bowa_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n aid = self.sequences.aides.fastaccess\n for k in range(con.nhru):\n if con.lnk[k] in (VERS, WASSER, FLUSS, SEE):\n sta.bowa[k] = 0.\n else:\n aid.bvl[k] = (\n flu.evb[k]+flu.qbb[k]+flu.qib1[k]+flu.qib2[k]+flu.qdb[k])\n aid.mvl[k] = sta.bowa[k]+flu.wada[k]\n if aid.bvl[k] > aid.mvl[k]:\n aid.rvl[k] = aid.mvl[k]/aid.bvl[k]\n flu.evb[k] *= aid.rvl[k]\n flu.qbb[k] *= aid.rvl[k]\n flu.qib1[k] *= aid.rvl[k]\n flu.qib2[k] *= aid.rvl[k]\n flu.qdb[k] *= aid.rvl[k]\n sta.bowa[k] = 0.\n else:\n sta.bowa[k] = aid.mvl[k]-aid.bvl[k]", "response": "Calculates the Bowa state sequence and correct fluxes if necessary."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the sum of the base flow released by all soil types and the stream network.", "response": "def calc_qbgz_v1(self):\n \"\"\"Aggregate the amount of base flow released by all \"soil type\" HRUs\n and the \"net precipitation\" above water areas of type |SEE|.\n\n Water areas of type |SEE| are assumed to be directly connected with\n groundwater, but not with the stream network. This is modelled by\n adding their (positive or negative) \"net input\" (|NKor|-|EvI|) to the\n \"percolation output\" of the soil containing HRUs.\n\n Required control parameters:\n |Lnk|\n |NHRU|\n |FHRU|\n\n Required flux sequences:\n |QBB|\n |NKor|\n |EvI|\n\n Calculated state sequence:\n |QBGZ|\n\n Basic equation:\n :math:`QBGZ = \\\\Sigma(FHRU \\\\cdot QBB) +\n \\\\Sigma(FHRU \\\\cdot (NKor_{SEE}-EvI_{SEE}))`\n\n Examples:\n\n The first example shows that |QBGZ| is the area weighted sum of\n |QBB| from \"soil type\" HRUs like arable land (|ACKER|) and of\n |NKor|-|EvI| from water areas of type |SEE|. All other water\n areas (|WASSER| and |FLUSS|) and also sealed surfaces (|VERS|)\n have no impact on |QBGZ|:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep()\n >>> nhru(6)\n >>> lnk(ACKER, ACKER, VERS, WASSER, FLUSS, SEE)\n >>> fhru(0.1, 0.2, 0.1, 0.1, 0.1, 0.4)\n >>> fluxes.qbb = 2., 4.0, 300.0, 300.0, 300.0, 300.0\n >>> fluxes.nkor = 200.0, 200.0, 200.0, 200.0, 200.0, 20.0\n >>> fluxes.evi = 100.0, 100.0, 100.0, 100.0, 100.0, 10.0\n >>> model.calc_qbgz_v1()\n >>> states.qbgz\n qbgz(5.0)\n\n The second example shows that large evaporation values above a\n HRU of type |SEE| can result in negative values of |QBGZ|:\n\n >>> fluxes.evi[5] = 30\n >>> model.calc_qbgz_v1()\n >>> states.qbgz\n qbgz(-3.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n sta.qbgz = 0.\n for k in range(con.nhru):\n if con.lnk[k] == SEE:\n sta.qbgz += con.fhru[k]*(flu.nkor[k]-flu.evi[k])\n elif con.lnk[k] not in (WASSER, FLUSS, VERS):\n sta.qbgz += con.fhru[k]*flu.qbb[k]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calc_qigz1_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n sta.qigz1 = 0.\n for k in range(con.nhru):\n sta.qigz1 += con.fhru[k]*flu.qib1[k]", "response": "Calculates the sum of the amount of the first interflow component released by all HRUs."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the sum of the second interflow component released by all HRUs.", "response": "def calc_qigz2_v1(self):\n \"\"\"Aggregate the amount of the second interflow component released\n by all HRUs.\n\n Required control parameters:\n |NHRU|\n |FHRU|\n\n Required flux sequence:\n |QIB2|\n\n Calculated state sequence:\n |QIGZ2|\n\n Basic equation:\n :math:`QIGZ2 = \\\\Sigma(FHRU \\\\cdot QIB2)`\n\n Example:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep()\n >>> nhru(2)\n >>> fhru(0.75, 0.25)\n >>> fluxes.qib2 = 1.0, 5.0\n >>> model.calc_qigz2_v1()\n >>> states.qigz2\n qigz2(2.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n sta.qigz2 = 0.\n for k in range(con.nhru):\n sta.qigz2 += con.fhru[k]*flu.qib2[k]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef calc_qdgz1_qdgz2_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n if flu.qdgz > con.a2:\n sta.qdgz2 = (flu.qdgz-con.a2)**2/(flu.qdgz+con.a1-con.a2)\n sta.qdgz1 = flu.qdgz-sta.qdgz2\n else:\n sta.qdgz2 = 0.\n sta.qdgz1 = flu.qdgz", "response": "Calculates the total direct flow into a small and a fast component."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nperform the runoff concentration calculation for base flow.", "response": "def calc_qbga_v1(self):\n \"\"\"Perform the runoff concentration calculation for base flow.\n\n The working equation is the analytical solution of the linear storage\n equation under the assumption of constant change in inflow during\n the simulation time step.\n\n Required derived parameter:\n |KB|\n\n Required flux sequence:\n |QBGZ|\n\n Calculated state sequence:\n |QBGA|\n\n Basic equation:\n :math:`QBGA_{neu} = QBGA_{alt} +\n (QBGZ_{alt}-QBGA_{alt}) \\\\cdot (1-exp(-KB^{-1})) +\n (QBGZ_{neu}-QBGZ_{alt}) \\\\cdot (1-KB\\\\cdot(1-exp(-KB^{-1})))`\n\n Examples:\n\n A normal test case:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep()\n >>> derived.kb(0.1)\n >>> states.qbgz.old = 2.0\n >>> states.qbgz.new = 4.0\n >>> states.qbga.old = 3.0\n >>> model.calc_qbga_v1()\n >>> states.qbga\n qbga(3.800054)\n\n First extreme test case (zero division is circumvented):\n\n >>> derived.kb(0.0)\n >>> model.calc_qbga_v1()\n >>> states.qbga\n qbga(4.0)\n\n Second extreme test case (numerical overflow is circumvented):\n\n >>> derived.kb(1e500)\n >>> model.calc_qbga_v1()\n >>> states.qbga\n qbga(5.0)\n \"\"\"\n der = self.parameters.derived.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n if der.kb <= 0.:\n new.qbga = new.qbgz\n elif der.kb > 1e200:\n new.qbga = old.qbga+new.qbgz-old.qbgz\n else:\n d_temp = (1.-modelutils.exp(-1./der.kb))\n new.qbga = (old.qbga +\n (old.qbgz-old.qbga)*d_temp +\n (new.qbgz-old.qbgz)*(1.-der.kb*d_temp))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calc_qiga1_v1(self):\n der = self.parameters.derived.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n if der.ki1 <= 0.:\n new.qiga1 = new.qigz1\n elif der.ki1 > 1e200:\n new.qiga1 = old.qiga1+new.qigz1-old.qigz1\n else:\n d_temp = (1.-modelutils.exp(-1./der.ki1))\n new.qiga1 = (old.qiga1 +\n (old.qigz1-old.qiga1)*d_temp +\n (new.qigz1-old.qigz1)*(1.-der.ki1*d_temp))", "response": "Perform the runoff concentration calculation for the first interflow component."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_qiga2_v1(self):\n der = self.parameters.derived.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n if der.ki2 <= 0.:\n new.qiga2 = new.qigz2\n elif der.ki2 > 1e200:\n new.qiga2 = old.qiga2+new.qigz2-old.qigz2\n else:\n d_temp = (1.-modelutils.exp(-1./der.ki2))\n new.qiga2 = (old.qiga2 +\n (old.qigz2-old.qiga2)*d_temp +\n (new.qigz2-old.qigz2)*(1.-der.ki2*d_temp))", "response": "Perform the runoff concentration calculation for the second\n interflow component."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nperform the runoff concentration calculation for slow direct runoff.", "response": "def calc_qdga1_v1(self):\n \"\"\"Perform the runoff concentration calculation for \"slow\" direct runoff.\n\n The working equation is the analytical solution of the linear storage\n equation under the assumption of constant change in inflow during\n the simulation time step.\n\n Required derived parameter:\n |KD1|\n\n Required state sequence:\n |QDGZ1|\n\n Calculated state sequence:\n |QDGA1|\n\n Basic equation:\n :math:`QDGA1_{neu} = QDGA1_{alt} +\n (QDGZ1_{alt}-QDGA1_{alt}) \\\\cdot (1-exp(-KD1^{-1})) +\n (QDGZ1_{neu}-QDGZ1_{alt}) \\\\cdot (1-KD1\\\\cdot(1-exp(-KD1^{-1})))`\n\n Examples:\n\n A normal test case:\n\n >>> from hydpy.models.lland import *\n >>> parameterstep()\n >>> derived.kd1(0.1)\n >>> states.qdgz1.old = 2.0\n >>> states.qdgz1.new = 4.0\n >>> states.qdga1.old = 3.0\n >>> model.calc_qdga1_v1()\n >>> states.qdga1\n qdga1(3.800054)\n\n First extreme test case (zero division is circumvented):\n\n >>> derived.kd1(0.0)\n >>> model.calc_qdga1_v1()\n >>> states.qdga1\n qdga1(4.0)\n\n Second extreme test case (numerical overflow is circumvented):\n\n >>> derived.kd1(1e500)\n >>> model.calc_qdga1_v1()\n >>> states.qdga1\n qdga1(5.0)\n \"\"\"\n der = self.parameters.derived.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n if der.kd1 <= 0.:\n new.qdga1 = new.qdgz1\n elif der.kd1 > 1e200:\n new.qdga1 = old.qdga1+new.qdgz1-old.qdgz1\n else:\n d_temp = (1.-modelutils.exp(-1./der.kd1))\n new.qdga1 = (old.qdga1 +\n (old.qdgz1-old.qdga1)*d_temp +\n (new.qdgz1-old.qdgz1)*(1.-der.kd1*d_temp))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_qdga2_v1(self):\n der = self.parameters.derived.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n if der.kd2 <= 0.:\n new.qdga2 = new.qdgz2\n elif der.kd2 > 1e200:\n new.qdga2 = old.qdga2+new.qdgz2-old.qdgz2\n else:\n d_temp = (1.-modelutils.exp(-1./der.kd2))\n new.qdga2 = (old.qdga2 +\n (old.qdgz2-old.qdga2)*d_temp +\n (new.qdgz2-old.qdgz2)*(1.-der.kd2*d_temp))", "response": "Perform the runoff concentration calculation for fast direct runoff."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef calc_q_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n aid = self.sequences.aides.fastaccess\n flu.q = sta.qbga+sta.qiga1+sta.qiga2+sta.qdga1+sta.qdga2\n if (not con.negq) and (flu.q < 0.):\n d_area = 0.\n for k in range(con.nhru):\n if con.lnk[k] in (FLUSS, SEE):\n d_area += con.fhru[k]\n if d_area > 0.:\n for k in range(con.nhru):\n if con.lnk[k] in (FLUSS, SEE):\n flu.evi[k] += flu.q/d_area\n flu.q = 0.\n aid.epw = 0.\n for k in range(con.nhru):\n if con.lnk[k] == WASSER:\n flu.q += con.fhru[k]*flu.nkor[k]\n aid.epw += con.fhru[k]*flu.evi[k]\n if (flu.q > aid.epw) or con.negq:\n flu.q -= aid.epw\n elif aid.epw > 0.:\n for k in range(con.nhru):\n if con.lnk[k] == WASSER:\n flu.evi[k] *= flu.q/aid.epw\n flu.q = 0.", "response": "Calculates the final runoff of the water areas in the current runoff tree."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pass_q_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n out = self.sequences.outlets.fastaccess\n out.q[0] += der.qfactor*flu.q", "response": "Update the outlet link sequence."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the actual interpolation or extrapolation of the flux sequence.", "response": "def calc_outputs_v1(self):\n \"\"\"Performs the actual interpolation or extrapolation.\n\n Required control parameters:\n |XPoints|\n |YPoints|\n\n Required derived parameter:\n |NmbPoints|\n |NmbBranches|\n\n Required flux sequence:\n |Input|\n\n Calculated flux sequence:\n |Outputs|\n\n Examples:\n\n As a simple example, assume a weir directing all discharge into\n `branch1` until the capacity limit of 2 m\u00b3/s is reached. The\n discharge exceeding this threshold is directed into `branch2`:\n\n >>> from hydpy.models.hbranch import *\n >>> parameterstep()\n >>> xpoints(0., 2., 4.)\n >>> ypoints(branch1=[0., 2., 2.],\n ... branch2=[0., 0., 2.])\n >>> model.parameters.update()\n\n Low discharge example (linear interpolation between the first two\n supporting point pairs):\n\n >>> fluxes.input = 1.\n >>> model.calc_outputs_v1()\n >>> fluxes.outputs\n outputs(branch1=1.0,\n branch2=0.0)\n\n Medium discharge example (linear interpolation between the second\n two supporting point pairs):\n\n >>> fluxes.input = 3.\n >>> model.calc_outputs_v1()\n >>> print(fluxes.outputs)\n outputs(branch1=2.0,\n branch2=1.0)\n\n High discharge example (linear extrapolation beyond the second two\n supporting point pairs):\n\n >>> fluxes.input = 5.\n >>> model.calc_outputs_v1()\n >>> fluxes.outputs\n outputs(branch1=2.0,\n branch2=3.0)\n\n Non-monotonous relationships and balance violations are allowed,\n e.g.:\n\n >>> xpoints(0., 2., 4., 6.)\n >>> ypoints(branch1=[0., 2., 0., 0.],\n ... branch2=[0., 0., 2., 4.])\n >>> model.parameters.update()\n >>> fluxes.input = 7.\n >>> model.calc_outputs_v1()\n >>> fluxes.outputs\n outputs(branch1=0.0,\n branch2=5.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n # Search for the index of the two relevant x points...\n for pdx in range(1, der.nmbpoints):\n if con.xpoints[pdx] > flu.input:\n break\n # ...and use it for linear interpolation (or extrapolation).\n for bdx in range(der.nmbbranches):\n flu.outputs[bdx] = (\n (flu.input-con.xpoints[pdx-1]) *\n (con.ypoints[bdx, pdx]-con.ypoints[bdx, pdx-1]) /\n (con.xpoints[pdx]-con.xpoints[pdx-1]) +\n con.ypoints[bdx, pdx-1])"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates |Input| based on |Total|.", "response": "def pick_input_v1(self):\n \"\"\"Updates |Input| based on |Total|.\"\"\"\n flu = self.sequences.fluxes.fastaccess\n inl = self.sequences.inlets.fastaccess\n flu.input = 0.\n for idx in range(inl.len_total):\n flu.input += inl.total[idx][0]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pass_outputs_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n out = self.sequences.outlets.fastaccess\n for bdx in range(der.nmbbranches):\n out.branched[bdx][0] += flu.outputs[bdx]", "response": "Update |Branched| based on |Outputs|."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef connect(self):\n nodes = self.element.inlets\n total = self.sequences.inlets.total\n if total.shape != (len(nodes),):\n total.shape = len(nodes)\n for idx, node in enumerate(nodes):\n double = node.get_double('inlets')\n total.set_pointer(double, idx)\n for (idx, name) in enumerate(self.nodenames):\n try:\n outlet = getattr(self.element.outlets, name)\n double = outlet.get_double('outlets')\n except AttributeError:\n raise RuntimeError(\n f'Model {objecttools.elementphrase(self)} tried '\n f'to connect to an outlet node named `{name}`, '\n f'which is not an available outlet node of element '\n f'`{self.element.name}`.')\n self.sequences.outlets.branched.set_pointer(double, idx)", "response": "Connect the |LinkSequence| instances handled by one inlet node and the actual model\n multiple oulet nodes."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self):\n responses = self.subpars.pars.control.responses\n self.shape = len(responses)\n self(responses.ar_orders)", "response": "Update the internal structure of the object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self):\n pars = self.subpars.pars\n coefs = pars.control.responses.ar_coefs\n self.shape = coefs.shape\n self(coefs)\n pars.model.sequences.logs.logout.shape = self.shape", "response": "Update the log - related AR coefficients."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update(self):\n pars = self.subpars.pars\n coefs = pars.control.responses.ma_coefs\n self.shape = coefs.shape\n self(coefs)\n pars.model.sequences.logs.login.shape = self.shape", "response": "Update the log - in and log - in terms of the MA coefficients."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _prepare_docstrings():\n if config.USEAUTODOC:\n source = inspect.getsource(Options)\n docstrings = source.split('\"\"\"')[3::2]\n attributes = [line.strip().split()[0] for line in source.split('\\n')\n if '_Option(' in line]\n for attribute, docstring in zip(attributes, docstrings):\n Options.__dict__[attribute].__doc__ = docstring", "response": "Assign docstrings to the corresponding attributes of class Options to make them available in the interactive mode of Python."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntry to convert the given argument to a list of articles and return it.", "response": "def __getiterable(value): # ToDo: refactor\n \"\"\"Try to convert the given argument to a |list| of |Selection|\n objects and return it.\n \"\"\"\n if isinstance(value, Selection):\n return [value]\n try:\n for selection in value:\n if not isinstance(selection, Selection):\n raise TypeError\n return list(value)\n except TypeError:\n raise TypeError(\n f'Binary operations on Selections objects are defined for '\n f'other Selections objects, single Selection objects, or '\n f'iterables containing `Selection` objects, but the type of '\n f'the given argument is `{objecttools.classname(value)}`.')"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef assignrepr(self, prefix='') -> str:\n with objecttools.repr_.preserve_strings(True):\n with hydpy.pub.options.ellipsis(2, optional=True):\n prefix += '%s(' % objecttools.classname(self)\n repr_ = objecttools.assignrepr_values(\n sorted(self.names), prefix, 70)\n return repr_ + ')'", "response": "Return a |repr| string with a prefixed assignment."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the upstream network of the given starting point.", "response": "def search_upstream(self, device: devicetools.Device,\n name: str = 'upstream') -> 'Selection':\n \"\"\"Return the network upstream of the given starting point, including\n the starting point itself.\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, _ = prepare_full_example_2()\n\n You can pass both |Node| and |Element| objects and, optionally,\n the name of the newly created |Selection| object:\n\n >>> test = pub.selections.complete.copy('test')\n >>> test.search_upstream(hp.nodes.lahn_2)\n Selection(\"upstream\",\n nodes=(\"dill\", \"lahn_1\", \"lahn_2\"),\n elements=(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\",\n \"stream_dill_lahn_2\", \"stream_lahn_1_lahn_2\"))\n >>> test.search_upstream(\n ... hp.elements.stream_lahn_1_lahn_2, 'UPSTREAM')\n Selection(\"UPSTREAM\",\n nodes=\"lahn_1\",\n elements=(\"land_lahn_1\", \"stream_lahn_1_lahn_2\"))\n\n Wrong device specifications result in errors like the following:\n\n >>> test.search_upstream(1)\n Traceback (most recent call last):\n ...\n TypeError: While trying to determine the upstream network of \\\nselection `test`, the following error occurred: Either a `Node` or \\\nan `Element` object is required as the \"outlet device\", but the given \\\n`device` value is of type `int`.\n\n >>> pub.selections.headwaters.search_upstream(hp.nodes.lahn_3)\n Traceback (most recent call last):\n ...\n KeyError: \"While trying to determine the upstream network of \\\nselection `headwaters`, the following error occurred: 'No node named \\\n`lahn_3` available.'\"\n\n Method |Selection.select_upstream| restricts the current selection\n to the one determined with the method |Selection.search_upstream|:\n\n >>> test.select_upstream(hp.nodes.lahn_2)\n Selection(\"test\",\n nodes=(\"dill\", \"lahn_1\", \"lahn_2\"),\n elements=(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\",\n \"stream_dill_lahn_2\", \"stream_lahn_1_lahn_2\"))\n\n On the contrary, the method |Selection.deselect_upstream| restricts\n the current selection to all devices not determined by method\n |Selection.search_upstream|:\n\n >>> complete = pub.selections.complete.deselect_upstream(\n ... hp.nodes.lahn_2)\n >>> complete\n Selection(\"complete\",\n nodes=\"lahn_3\",\n elements=(\"land_lahn_3\", \"stream_lahn_2_lahn_3\"))\n\n If necessary, include the \"outlet device\" manually afterwards:\n\n >>> complete.nodes += hp.nodes.lahn_2\n >>> complete\n Selection(\"complete\",\n nodes=(\"lahn_2\", \"lahn_3\"),\n elements=(\"land_lahn_3\", \"stream_lahn_2_lahn_3\"))\n \"\"\"\n try:\n selection = Selection(name)\n if isinstance(device, devicetools.Node):\n node = self.nodes[device.name]\n return self.__get_nextnode(node, selection)\n if isinstance(device, devicetools.Element):\n element = self.elements[device.name]\n return self.__get_nextelement(element, selection)\n raise TypeError(\n f'Either a `Node` or an `Element` object is required '\n f'as the \"outlet device\", but the given `device` value '\n f'is of type `{objecttools.classname(device)}`.')\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to determine the upstream network of '\n f'selection `{self.name}`')"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef select_upstream(self, device: devicetools.Device) -> 'Selection':\n upstream = self.search_upstream(device)\n self.nodes = upstream.nodes\n self.elements = upstream.elements\n return self", "response": "Restrict the current selection to the network upstream of the given devicetools. Device."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a new |Selection| object containing only the elements of the given models of the given types.", "response": "def search_modeltypes(self, *models: ModelTypesArg,\n name: str = 'modeltypes') -> 'Selection':\n \"\"\"Return a |Selection| object containing only the elements\n currently handling models of the given types.\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, _ = prepare_full_example_2()\n\n You can pass both |Model| objects and names and, as a keyword\n argument, the name of the newly created |Selection| object:\n\n >>> test = pub.selections.complete.copy('test')\n >>> from hydpy import prepare_model\n >>> hland_v1 = prepare_model('hland_v1')\n\n >>> test.search_modeltypes(hland_v1)\n Selection(\"modeltypes\",\n nodes=(),\n elements=(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\",\n \"land_lahn_3\"))\n >>> test.search_modeltypes(\n ... hland_v1, 'hstream_v1', 'lland_v1', name='MODELTYPES')\n Selection(\"MODELTYPES\",\n nodes=(),\n elements=(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\",\n \"land_lahn_3\", \"stream_dill_lahn_2\",\n \"stream_lahn_1_lahn_2\", \"stream_lahn_2_lahn_3\"))\n\n Wrong model specifications result in errors like the following:\n\n >>> test.search_modeltypes('wrong')\n Traceback (most recent call last):\n ...\n ModuleNotFoundError: While trying to determine the elements of \\\nselection `test` handling the model defined by the argument(s) `wrong` \\\nof type(s) `str`, the following error occurred: \\\nNo module named 'hydpy.models.wrong'\n\n Method |Selection.select_modeltypes| restricts the current selection to\n the one determined with the method the |Selection.search_modeltypes|:\n\n >>> test.select_modeltypes(hland_v1)\n Selection(\"test\",\n nodes=(),\n elements=(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\",\n \"land_lahn_3\"))\n\n On the contrary, the method |Selection.deselect_upstream| restricts\n the current selection to all devices not determined by method the\n |Selection.search_upstream|:\n\n >>> pub.selections.complete.deselect_modeltypes(hland_v1)\n Selection(\"complete\",\n nodes=(),\n elements=(\"stream_dill_lahn_2\", \"stream_lahn_1_lahn_2\",\n \"stream_lahn_2_lahn_3\"))\n \"\"\"\n try:\n typelist = []\n for model in models:\n if not isinstance(model, modeltools.Model):\n model = importtools.prepare_model(model)\n typelist.append(type(model))\n typetuple = tuple(typelist)\n selection = Selection(name)\n for element in self.elements:\n if isinstance(element.model, typetuple):\n selection.elements += element\n return selection\n except BaseException:\n values = objecttools.enumeration(models)\n classes = objecttools.enumeration(\n objecttools.classname(model) for model in models)\n objecttools.augment_excmessage(\n f'While trying to determine the elements of selection '\n f'`{self.name}` handling the model defined by the '\n f'argument(s) `{values}` of type(s) `{classes}`')"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef select_modeltypes(self, *models: ModelTypesArg) -> 'Selection':\n self.nodes = devicetools.Nodes()\n self.elements = self.search_modeltypes(*models).elements\n return self", "response": "Restrict the current |Selection| object to all elements containing the given model types."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef search_nodenames(self, *substrings: str, name: str = 'nodenames') -> \\\n 'Selection':\n \"\"\"Return a new selection containing all nodes of the current\n selection with a name containing at least one of the given substrings.\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, _ = prepare_full_example_2()\n\n Pass the (sub)strings as positional arguments and, optionally, the\n name of the newly created |Selection| object as a keyword argument:\n\n >>> test = pub.selections.complete.copy('test')\n >>> from hydpy import prepare_model\n >>> test.search_nodenames('dill', 'lahn_1')\n Selection(\"nodenames\",\n nodes=(\"dill\", \"lahn_1\"),\n elements=())\n\n Wrong string specifications result in errors like the following:\n\n >>> test.search_nodenames(['dill', 'lahn_1'])\n Traceback (most recent call last):\n ...\n TypeError: While trying to determine the nodes of selection \\\n`test` with names containing at least one of the given substrings \\\n`['dill', 'lahn_1']`, the following error occurred: 'in ' \\\nrequires string as left operand, not list\n\n Method |Selection.select_nodenames| restricts the current selection\n to the one determined with the the method |Selection.search_nodenames|:\n\n >>> test.select_nodenames('dill', 'lahn_1')\n Selection(\"test\",\n nodes=(\"dill\", \"lahn_1\"),\n elements=(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\",\n \"land_lahn_3\", \"stream_dill_lahn_2\",\n \"stream_lahn_1_lahn_2\", \"stream_lahn_2_lahn_3\"))\n\n On the contrary, the method |Selection.deselect_nodenames| restricts\n the current selection to all devices not determined by the method\n |Selection.search_nodenames|:\n\n >>> pub.selections.complete.deselect_nodenames('dill', 'lahn_1')\n Selection(\"complete\",\n nodes=(\"lahn_2\", \"lahn_3\"),\n elements=(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\",\n \"land_lahn_3\", \"stream_dill_lahn_2\",\n \"stream_lahn_1_lahn_2\", \"stream_lahn_2_lahn_3\"))\n \"\"\"\n try:\n selection = Selection(name)\n for node in self.nodes:\n for substring in substrings:\n if substring in node.name:\n selection.nodes += node\n break\n return selection\n except BaseException:\n values = objecttools.enumeration(substrings)\n objecttools.augment_excmessage(\n f'While trying to determine the nodes of selection '\n f'`{self.name}` with names containing at least one '\n f'of the given substrings `{values}`')", "response": "Return a new selection containing all nodes of the current selection with a name containing at least one of the given substrings."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrestricting the current selection to all nodes with a name containing at least one of the given substrings.", "response": "def select_nodenames(self, *substrings: str) -> 'Selection':\n \"\"\"Restrict the current selection to all nodes with a name\n containing at least one of the given substrings (does not\n affect any elements).\n\n See the documentation on method |Selection.search_nodenames| for\n additional information.\n \"\"\"\n self.nodes = self.search_nodenames(*substrings).nodes\n return self"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef deselect_nodenames(self, *substrings: str) -> 'Selection':\n self.nodes -= self.search_nodenames(*substrings).nodes\n return self", "response": "Restrict the current selection to all nodes with a name\n not containing at least one of the given substrings."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef search_elementnames(self, *substrings: str,\n name: str = 'elementnames') -> 'Selection':\n \"\"\"Return a new selection containing all elements of the current\n selection with a name containing at least one of the given substrings.\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, _ = prepare_full_example_2()\n\n Pass the (sub)strings as positional arguments and, optionally, the\n name of the newly created |Selection| object as a keyword argument:\n\n >>> test = pub.selections.complete.copy('test')\n >>> from hydpy import prepare_model\n >>> test.search_elementnames('dill', 'lahn_1')\n Selection(\"elementnames\",\n nodes=(),\n elements=(\"land_dill\", \"land_lahn_1\", \"stream_dill_lahn_2\",\n \"stream_lahn_1_lahn_2\"))\n\n Wrong string specifications result in errors like the following:\n\n >>> test.search_elementnames(['dill', 'lahn_1'])\n Traceback (most recent call last):\n ...\n TypeError: While trying to determine the elements of selection \\\n`test` with names containing at least one of the given substrings \\\n`['dill', 'lahn_1']`, the following error occurred: 'in ' \\\nrequires string as left operand, not list\n\n Method |Selection.select_elementnames| restricts the current selection\n to the one determined with the method |Selection.search_elementnames|:\n\n >>> test.select_elementnames('dill', 'lahn_1')\n Selection(\"test\",\n nodes=(\"dill\", \"lahn_1\", \"lahn_2\", \"lahn_3\"),\n elements=(\"land_dill\", \"land_lahn_1\", \"stream_dill_lahn_2\",\n \"stream_lahn_1_lahn_2\"))\n\n On the contrary, the method |Selection.deselect_elementnames|\n restricts the current selection to all devices not determined\n by the method |Selection.search_elementnames|:\n\n >>> pub.selections.complete.deselect_elementnames('dill', 'lahn_1')\n Selection(\"complete\",\n nodes=(\"dill\", \"lahn_1\", \"lahn_2\", \"lahn_3\"),\n elements=(\"land_lahn_2\", \"land_lahn_3\",\n \"stream_lahn_2_lahn_3\"))\n \"\"\"\n try:\n selection = Selection(name)\n for element in self.elements:\n for substring in substrings:\n if substring in element.name:\n selection.elements += element\n break\n return selection\n except BaseException:\n values = objecttools.enumeration(substrings)\n objecttools.augment_excmessage(\n f'While trying to determine the elements of selection '\n f'`{self.name}` with names containing at least one '\n f'of the given substrings `{values}`')", "response": "Return a new selection containing all elements of the current selection with a name containing at least one of the given substrings."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrestrict the current selection to all elements with a name containing at least one of the given substrings.", "response": "def select_elementnames(self, *substrings: str) -> 'Selection':\n \"\"\"Restrict the current selection to all elements with a name\n containing at least one of the given substrings (does not\n affect any nodes).\n\n See the documentation on method |Selection.search_elementnames| for\n additional information.\n \"\"\"\n self.elements = self.search_elementnames(*substrings).elements\n return self"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrestricting the current selection to all elements with a name not containing at least one of the given substrings.", "response": "def deselect_elementnames(self, *substrings: str) -> 'Selection':\n \"\"\"Restrict the current selection to all elements with a name\n not containing at least one of the given substrings. (does\n not affect any nodes).\n\n See the documentation on method |Selection.search_elementnames| for\n additional information.\n \"\"\"\n self.elements -= self.search_elementnames(*substrings).elements\n return self"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef copy(self, name: str) -> 'Selection':\n return type(self)(name, copy.copy(self.nodes), copy.copy(self.elements))", "response": "Return a new |Selection| object with the given name and copies\n of the handles |Nodes| and |Elements| objects based on method\n |Devices. copy|."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsaving the selection as a network file.", "response": "def save_networkfile(self, filepath: Union[str, None] = None,\n write_nodes: bool = True) -> None:\n \"\"\"Save the selection as a network file.\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> _, pub, TestIO = prepare_full_example_2()\n\n In most cases, one should conveniently write network files via method\n |NetworkManager.save_files| of class |NetworkManager|. However,\n using the method |Selection.save_networkfile| allows for additional\n configuration via the arguments `filepath` and `write_nodes`:\n\n >>> with TestIO():\n ... pub.selections.headwaters.save_networkfile()\n ... with open('headwaters.py') as networkfile:\n ... print(networkfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy import Node, Element\n \n \n Node(\"dill\", variable=\"Q\",\n keywords=\"gauge\")\n \n Node(\"lahn_1\", variable=\"Q\",\n keywords=\"gauge\")\n \n \n Element(\"land_dill\",\n outlets=\"dill\",\n keywords=\"catchment\")\n \n Element(\"land_lahn_1\",\n outlets=\"lahn_1\",\n keywords=\"catchment\")\n \n\n >>> with TestIO():\n ... pub.selections.headwaters.save_networkfile('test.py', False)\n ... with open('test.py') as networkfile:\n ... print(networkfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy import Node, Element\n \n \n Element(\"land_dill\",\n outlets=\"dill\",\n keywords=\"catchment\")\n \n Element(\"land_lahn_1\",\n outlets=\"lahn_1\",\n keywords=\"catchment\")\n \n \"\"\"\n if filepath is None:\n filepath = self.name + '.py'\n with open(filepath, 'w', encoding=\"utf-8\") as file_:\n file_.write('# -*- coding: utf-8 -*-\\n')\n file_.write('\\nfrom hydpy import Node, Element\\n\\n')\n if write_nodes:\n for node in self.nodes:\n file_.write('\\n' + repr(node) + '\\n')\n file_.write('\\n')\n for element in self.elements:\n file_.write('\\n' + repr(element) + '\\n')"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef assignrepr(self, prefix: str) -> str:\n with objecttools.repr_.preserve_strings(True):\n with hydpy.pub.options.ellipsis(2, optional=True):\n with objecttools.assignrepr_tuple.always_bracketed(False):\n classname = objecttools.classname(self)\n blanks = ' ' * (len(prefix+classname) + 1)\n nodestr = objecttools.assignrepr_tuple(\n self.nodes.names, blanks+'nodes=', 70)\n elementstr = objecttools.assignrepr_tuple(\n self.elements.names, blanks + 'elements=', 70)\n return (f'{prefix}{classname}(\"{self.name}\",\\n'\n f'{nodestr},\\n'\n f'{elementstr})')", "response": "Return a |repr| string with a prefixed assignment."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalculates the input discharge portions of the different response functions and the corresponding flux sequences.", "response": "def calc_qpin_v1(self):\n \"\"\"Calculate the input discharge portions of the different response\n functions.\n\n Required derived parameters:\n |Nmb|\n |MaxQ|\n |DiffQ|\n\n Required flux sequence:\n |QIn|\n\n Calculated flux sequences:\n |QPIn|\n\n Examples:\n\n Initialize an arma model with three different response functions:\n\n >>> from hydpy.models.arma import *\n >>> parameterstep()\n >>> derived.nmb = 3\n >>> derived.maxq.shape = 3\n >>> derived.diffq.shape = 2\n >>> fluxes.qpin.shape = 3\n\n Define the maximum discharge value of the respective response\n functions and their successive differences:\n\n >>> derived.maxq(0.0, 2.0, 6.0)\n >>> derived.diffq(2., 4.)\n\n The first six examples are performed for inflow values ranging from\n 0 to 12 m\u00b3/s:\n\n >>> from hydpy import UnitTest\n >>> test = UnitTest(\n ... model, model.calc_qpin_v1,\n ... last_example=6,\n ... parseqs=(fluxes.qin, fluxes.qpin))\n >>> test.nexts.qin = 0., 1., 2., 4., 6., 12.\n >>> test()\n | ex. | qin | qpin |\n -------------------------------\n | 1 | 0.0 | 0.0 0.0 0.0 |\n | 2 | 1.0 | 1.0 0.0 0.0 |\n | 3 | 2.0 | 2.0 0.0 0.0 |\n | 4 | 4.0 | 2.0 2.0 0.0 |\n | 5 | 6.0 | 2.0 4.0 0.0 |\n | 6 | 12.0 | 2.0 4.0 6.0 |\n\n\n The following two additional examples are just supposed to\n demonstrate method |calc_qpin_v1| also functions properly if\n there is only one response function, wherefore total discharge\n does not need to be divided:\n\n >>> derived.nmb = 1\n >>> derived.maxq.shape = 1\n >>> derived.diffq.shape = 0\n >>> fluxes.qpin.shape = 1\n >>> derived.maxq(0.)\n\n >>> test = UnitTest(\n ... model, model.calc_qpin_v1,\n ... first_example=7, last_example=8,\n ... parseqs=(fluxes.qin,\n ... fluxes.qpin))\n >>> test.nexts.qin = 0., 12.\n >>> test()\n | ex. | qin | qpin |\n ---------------------\n | 7 | 0.0 | 0.0 |\n | 8 | 12.0 | 12.0 |\n\n \"\"\"\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for idx in range(der.nmb-1):\n if flu.qin < der.maxq[idx]:\n flu.qpin[idx] = 0.\n elif flu.qin < der.maxq[idx+1]:\n flu.qpin[idx] = flu.qin-der.maxq[idx]\n else:\n flu.qpin[idx] = der.diffq[idx]\n flu.qpin[der.nmb-1] = max(flu.qin-der.maxq[der.nmb-1], 0.)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_login_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n for idx in range(der.nmb):\n for jdx in range(der.ma_order[idx]-2, -1, -1):\n log.login[idx, jdx+1] = log.login[idx, jdx]\n for idx in range(der.nmb):\n log.login[idx, 0] = flu.qpin[idx]", "response": "Refresh the input log sequence for the different MA processes."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calc_qma_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n for idx in range(der.nmb):\n flu.qma[idx] = 0.\n for jdx in range(der.ma_order[idx]):\n flu.qma[idx] += der.ma_coefs[idx, jdx] * log.login[idx, jdx]", "response": "Calculate the discharge responses of the different MA processes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the discharge responses of the different AR processes and the fluxes of the different AR processes.", "response": "def calc_qar_v1(self):\n \"\"\"Calculate the discharge responses of the different AR processes.\n\n Required derived parameters:\n |Nmb|\n |AR_Order|\n |AR_Coefs|\n\n Required log sequence:\n |LogOut|\n\n Calculated flux sequence:\n |QAR|\n\n Examples:\n\n Assume there are four response functions, involving zero, one, two,\n and three AR coefficients respectively:\n\n >>> from hydpy.models.arma import *\n >>> parameterstep()\n >>> derived.nmb(4)\n >>> derived.ar_order.shape = 4\n >>> derived.ar_order = 0, 1, 2, 3\n >>> derived.ar_coefs.shape = (4, 3)\n >>> logs.logout.shape = (4, 3)\n >>> fluxes.qar.shape = 4\n\n The coefficients of the different AR processes are stored in\n separate rows of the 2-dimensional parameter `ma_coefs`.\n Note the special case of the first AR process of zero order\n (first row), which involves no autoregressive memory at all:\n\n >>> derived.ar_coefs = ((nan, nan, nan),\n ... (1.0, nan, nan),\n ... (0.8, 0.2, nan),\n ... (0.5, 0.3, 0.2))\n\n The \"memory values\" of the different AR processes are defined as\n follows (one row for each process). The values of the last time\n step are stored in first column, the values of the last time step\n in the second column, and so on:\n\n >>> logs.logout = ((nan, nan, nan),\n ... (1.0, nan, nan),\n ... (2.0, 3.0, nan),\n ... (4.0, 5.0, 6.0))\n\n Applying method |calc_qar_v1| is equivalent to calculating the\n inner product of the different rows of both matrices:\n\n >>> model.calc_qar_v1()\n >>> fluxes.qar\n qar(0.0, 1.0, 2.2, 4.7)\n\n \"\"\"\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n for idx in range(der.nmb):\n flu.qar[idx] = 0.\n for jdx in range(der.ar_order[idx]):\n flu.qar[idx] += der.ar_coefs[idx, jdx] * log.logout[idx, jdx]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calc_qpout_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for idx in range(der.nmb):\n flu.qpout[idx] = flu.qma[idx]+flu.qar[idx]", "response": "Calculate the ARMA results for the different response functions."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_logout_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n for idx in range(der.nmb):\n for jdx in range(der.ar_order[idx]-2, -1, -1):\n log.logout[idx, jdx+1] = log.logout[idx, jdx]\n for idx in range(der.nmb):\n if der.ar_order[idx] > 0:\n log.logout[idx, 0] = flu.qpout[idx]", "response": "Refresh the log sequence for the different AR processes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the sum of the results of the different response functions and add them to the fluxes.", "response": "def calc_qout_v1(self):\n \"\"\"Sum up the results of the different response functions.\n\n Required derived parameter:\n |Nmb|\n\n Required flux sequences:\n |QPOut|\n\n Calculated flux sequence:\n |QOut|\n\n Examples:\n\n Initialize an arma model with three different response functions:\n\n >>> from hydpy.models.arma import *\n >>> parameterstep()\n >>> derived.nmb(3)\n >>> fluxes.qpout.shape = 3\n\n Define the output values of the three response functions and\n apply method |calc_qout_v1|:\n\n >>> fluxes.qpout = 1.0, 2.0, 3.0\n >>> model.calc_qout_v1()\n >>> fluxes.qout\n qout(6.0)\n \"\"\"\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.qout = 0.\n for idx in range(der.nmb):\n flu.qout += flu.qpout[idx]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pick_q_v1(self):\n flu = self.sequences.fluxes.fastaccess\n inl = self.sequences.inlets.fastaccess\n flu.qin = 0.\n for idx in range(inl.len_q):\n flu.qin += inl.q[idx][0]", "response": "Update inflow based on the number of inlets."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update(self):\n con = self.subpars.pars.control\n self(con.ypoints.shape[0])", "response": "Update the internal variable self based on the number of branches in the control set."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update(self):\n con = self.subpars.pars.control\n self(0.)\n for idx in range(2):\n if (con.bbv[idx] > 0.) and (con.bnv[idx] > 0.):\n self.values[idx] = con.bbv[idx]/con.bnv[idx]", "response": "Update value based on : math : HV = BBV / BNV."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update(self):\n mod = self.subpars.pars.model\n con = mod.parameters.control\n flu = mod.sequences.fluxes\n flu.h = con.hm\n mod.calc_qg()\n self(flu.qg)", "response": "Update the value of the internal object with the current value of the internal object."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update(self):\n pars = self.subpars.pars\n self(int(round(pars.control.lag)))\n pars.model.sequences.states.qjoints.shape = self+1", "response": "Updates the internal state of the object with the current state of the current state."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update(self):\n damp = self.subpars.pars.control.damp\n self(numpy.clip(damp/(1.+damp), 0., .5))", "response": "Update |C1| based on |Damp}{1 + Damp|."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update(self):\n der = self.subpars\n self(numpy.clip(1. - der.c1 - der.c3, 0., 1.))", "response": "Update |C2| based on |1. - c_1 - c_3|."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef view(data, enc=None, start_pos=None, delimiter=None, hdr_rows=None,\n idx_cols=None, sheet_index=0, transpose=False, wait=None,\n recycle=None, detach=None, metavar=None, title=None):\n \"\"\"View the supplied data in an interactive, graphical table widget.\n\n data: When a valid path or IO object, read it as a tabular text file. When\n a valid URI, a Blaze object is constructed and visualized. Any other\n supported datatype is visualized directly and incrementally *without\n copying*.\n\n enc: File encoding (such as \"utf-8\", normally autodetected).\n\n delimiter: Text file delimiter (normally autodetected).\n\n hdr_rows: For files or lists of lists, specify the number of header rows.\n For files only, a default of one header line is assumed.\n\n idx_cols: For files or lists of lists, specify the number of index columns.\n By default, no index is assumed.\n\n sheet_index: For multi-table files (such as xls[x]), specify the sheet\n index to read, starting from 0. Defaults to the first.\n\n start_pos: A tuple of the form (y, x) specifying the initial cursor\n position. Negative offsets count from the end of the dataset.\n\n transpose: Transpose the resulting view.\n\n metavar: name of the variable being shown for display purposes (inferred\n automatically when possible).\n\n title: title of the data window.\n\n wait: Wait for the user to close the view before returning. By default, try\n to match the behavior of ``matplotlib.is_interactive()``. If\n matplotlib is not loaded, wait only if ``detach`` is also False. The\n default value can also be set through ``gtabview.WAIT``.\n\n recycle: Recycle the previous window instead of creating a new one. The\n default is True, and can also be set through ``gtabview.RECYCLE``.\n\n detach: Create a fully detached GUI thread for interactive use (note: this\n is *not* necessary if matplotlib is loaded). The default is False,\n and can also be set through ``gtabview.DETACH``.\n \"\"\"\n global WAIT, RECYCLE, DETACH, VIEW\n\n model = read_model(data, enc=enc, delimiter=delimiter, hdr_rows=hdr_rows,\n idx_cols=idx_cols, sheet_index=sheet_index,\n transpose=transpose)\n if model is None:\n warnings.warn(\"cannot visualize the supplied data type: {}\".format(type(data)),\n category=RuntimeWarning)\n return None\n\n # setup defaults\n if wait is None: wait = WAIT\n if recycle is None: recycle = RECYCLE\n if detach is None: detach = DETACH\n if wait is None:\n if 'matplotlib' not in sys.modules:\n wait = not bool(detach)\n else:\n import matplotlib.pyplot as plt\n wait = not plt.isinteractive()\n\n # try to fetch the variable name in the upper stack\n if metavar is None:\n if isinstance(data, basestring):\n metavar = data\n else:\n metavar = _varname_in_stack(data, 1)\n\n # create a view controller\n if VIEW is None:\n if not detach:\n VIEW = ViewController()\n else:\n VIEW = DetachedViewController()\n VIEW.setDaemon(True)\n VIEW.start()\n if VIEW.is_detached():\n atexit.register(VIEW.exit)\n else:\n VIEW = None\n return None\n\n # actually show the data\n view_kwargs = {'hdr_rows': hdr_rows, 'idx_cols': idx_cols,\n 'start_pos': start_pos, 'metavar': metavar, 'title': title}\n VIEW.view(model, view_kwargs, wait=wait, recycle=recycle)\n return VIEW", "response": "View the supplied data in an interactive graphical table widget."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef gather_registries() -> Tuple[Dict, Mapping, Mapping]:\n id2devices = copy.copy(_id2devices)\n registry = copy.copy(_registry)\n selection = copy.copy(_selection)\n dict_ = globals()\n dict_['_id2devices'] = {}\n dict_['_registry'] = {Node: {}, Element: {}}\n dict_['_selection'] = {Node: {}, Element: {}}\n return id2devices, registry, selection", "response": "Get and clear the current node and element registries."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreset the current |Node| and |Element| registries.", "response": "def reset_registries(dicts: Tuple[Dict, Mapping, Mapping]):\n \"\"\"Reset the current |Node| and |Element| registries.\n\n Function |reset_registries| is thought to be used by class |Tester| only.\n \"\"\"\n dict_ = globals()\n dict_['_id2devices'] = dicts[0]\n dict_['_registry'] = dicts[1]\n dict_['_selection'] = dicts[2]"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a pandas index for the current date range.", "response": "def _get_pandasindex():\n \"\"\"\n >>> from hydpy import pub\n >>> pub.timegrids = '2004.01.01', '2005.01.01', '1d'\n >>> from hydpy.core.devicetools import _get_pandasindex\n >>> _get_pandasindex() # doctest: +ELLIPSIS\n DatetimeIndex(['2004-01-01 12:00:00', '2004-01-02 12:00:00',\n ...\n '2004-12-30 12:00:00', '2004-12-31 12:00:00'],\n dtype='datetime64[ns]', length=366, freq=None)\n \"\"\"\n tg = hydpy.pub.timegrids.init\n shift = tg.stepsize / 2\n index = pandas.date_range(\n (tg.firstdate + shift).datetime,\n (tg.lastdate - shift).datetime,\n (tg.lastdate - tg.firstdate - tg.stepsize) / tg.stepsize + 1)\n return index"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a list of all keywords starting with the given string.", "response": "def startswith(self, name: str) -> List[str]:\n \"\"\"Return a list of all keywords starting with the given string.\n\n >>> from hydpy.core.devicetools import Keywords\n >>> keywords = Keywords('first_keyword', 'second_keyword',\n ... 'keyword_3', 'keyword_4',\n ... 'keyboard')\n >>> keywords.startswith('keyword')\n ['keyword_3', 'keyword_4']\n \"\"\"\n return sorted(keyword for keyword in self if keyword.startswith(name))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a list of all keywords ending with the given string.", "response": "def endswith(self, name: str) -> List[str]:\n \"\"\"Return a list of all keywords ending with the given string.\n\n >>> from hydpy.core.devicetools import Keywords\n >>> keywords = Keywords('first_keyword', 'second_keyword',\n ... 'keyword_3', 'keyword_4',\n ... 'keyboard')\n >>> keywords.endswith('keyword')\n ['first_keyword', 'second_keyword']\n \"\"\"\n return sorted(keyword for keyword in self if keyword.endswith(name))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of all keywords containing the given string.", "response": "def contains(self, name: str) -> List[str]:\n \"\"\"Return a list of all keywords containing the given string.\n\n >>> from hydpy.core.devicetools import Keywords\n >>> keywords = Keywords('first_keyword', 'second_keyword',\n ... 'keyword_3', 'keyword_4',\n ... 'keyboard')\n >>> keywords.contains('keyword')\n ['first_keyword', 'keyword_3', 'keyword_4', 'second_keyword']\n \"\"\"\n return sorted(keyword for keyword in self if name in keyword)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nupdates the set of variable identifiers.", "response": "def update(self, *names: Any) -> None:\n \"\"\"Before updating, the given names are checked to be valid\n variable identifiers.\n\n >>> from hydpy.core.devicetools import Keywords\n >>> keywords = Keywords('first_keyword', 'second_keyword',\n ... 'keyword_3', 'keyword_4',\n ... 'keyboard')\n >>> keywords.update('test_1', 'test 2') # doctest: +ELLIPSIS\n Traceback (most recent call last):\n ...\n ValueError: While trying to add the keyword `test 2` to device ?, \\\nthe following error occurred: The given name string `test 2` does not \\\ndefine a valid variable identifier. ...\n\n Note that even the first string (`test1`) is not added due to the\n second one (`test 2`) being invalid.\n\n >>> keywords\n Keywords(\"first_keyword\", \"keyboard\", \"keyword_3\", \"keyword_4\",\n \"second_keyword\")\n\n After correcting the second string, everything works fine:\n\n >>> keywords.update('test_1', 'test_2')\n >>> keywords\n Keywords(\"first_keyword\", \"keyboard\", \"keyword_3\", \"keyword_4\",\n \"second_keyword\", \"test_1\", \"test_2\")\n \"\"\"\n _names = [str(name) for name in names]\n self._check_keywords(_names)\n super().update(_names)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add(self, name: Any) -> None:\n self._check_keywords([str(name)])\n super().add(str(name))", "response": "Add a new name to the set."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadding the given device to the actual nodes or elements.", "response": "def add_device(self, device: Union[DeviceType, str]) -> None:\n \"\"\"Add the given |Node| or |Element| object to the actual\n |Nodes| or |Elements| object.\n\n You can pass either a string or a device:\n\n >>> from hydpy import Nodes\n >>> nodes = Nodes()\n >>> nodes.add_device('old_node')\n >>> nodes\n Nodes(\"old_node\")\n >>> nodes.add_device('new_node')\n >>> nodes\n Nodes(\"new_node\", \"old_node\")\n\n Method |Devices.add_device| is disabled for immutable |Nodes|\n and |Elements| objects:\n\n >>> nodes.mutable = False\n >>> nodes.add_device('newest_node')\n Traceback (most recent call last):\n ...\n RuntimeError: While trying to add the device `newest_node` to a \\\nNodes object, the following error occurred: Adding devices to immutable \\\nNodes objects is not allowed.\n \"\"\"\n try:\n if self.mutable:\n _device = self.get_contentclass()(device)\n self._name2device[_device.name] = _device\n _id2devices[_device][id(self)] = self\n else:\n raise RuntimeError(\n f'Adding devices to immutable '\n f'{objecttools.classname(self)} objects is not allowed.')\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to add the device `{device}` to a '\n f'{objecttools.classname(self)} object')"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef remove_device(self, device: Union[DeviceType, str]) -> None:\n try:\n if self.mutable:\n _device = self.get_contentclass()(device)\n try:\n del self._name2device[_device.name]\n except KeyError:\n raise ValueError(\n f'The actual {objecttools.classname(self)} '\n f'object does not handle such a device.')\n del _id2devices[_device][id(self)]\n else:\n raise RuntimeError(\n f'Removing devices from immutable '\n f'{objecttools.classname(self)} objects is not allowed.')\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to remove the device `{device}` from a '\n f'{objecttools.classname(self)} object')", "response": "Remove the given device from the actual nodes or elements."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef copy(self: DevicesTypeBound) -> DevicesTypeBound:\n new = type(self)()\n vars(new).update(vars(self))\n vars(new)['_name2device'] = copy.copy(self._name2device)\n vars(new)['_shadowed_keywords'].clear()\n for device in self:\n _id2devices[device][id(new)] = new\n return new", "response": "Return a shallow copy of the actual |Nodes| or |Elements| object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall methods |Node. prepare_simseries| and |Node. prepare_obsseries| and |Node. prepare_simseries| and |Node. prepare_obsseries|.", "response": "def prepare_allseries(self, ramflag: bool = True) -> None:\n \"\"\"Call methods |Node.prepare_simseries| and\n |Node.prepare_obsseries|.\"\"\"\n self.prepare_simseries(ramflag)\n self.prepare_obsseries(ramflag)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalling method |Node. prepare_simseries| of all handled objects.", "response": "def prepare_simseries(self, ramflag: bool = True) -> None:\n \"\"\"Call method |Node.prepare_simseries| of all handled\n |Node| objects.\"\"\"\n for node in printtools.progressbar(self):\n node.prepare_simseries(ramflag)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef prepare_obsseries(self, ramflag: bool = True) -> None:\n for node in printtools.progressbar(self):\n node.prepare_obsseries(ramflag)", "response": "Call method |Node. prepare_obsseries| of all handled\n objects."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef init_models(self) -> None:\n try:\n for element in printtools.progressbar(self):\n element.init_model(clear_registry=False)\n finally:\n hydpy.pub.controlmanager.clear_registry()", "response": "Initializes the model objects of all elements of all handle."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsave the control parameters of the model object handled by the Arc with the given parameterstep and simulationstep.", "response": "def save_controls(self, parameterstep: 'timetools.PeriodConstrArg' = None,\n simulationstep: 'timetools.PeriodConstrArg' = None,\n auxfiler: 'Optional[auxfiletools.Auxfiler]' = None):\n \"\"\"Save the control parameters of the |Model| object handled by\n each |Element| object and eventually the ones handled by the\n given |Auxfiler| object.\"\"\"\n if auxfiler:\n auxfiler.save(parameterstep, simulationstep)\n for element in printtools.progressbar(self):\n element.model.parameters.save_controls(\n parameterstep=parameterstep,\n simulationstep=simulationstep,\n auxfiler=auxfiler)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsaving the initial conditions of the model object handled by each |Element| object.", "response": "def load_conditions(self) -> None:\n \"\"\"Save the initial conditions of the |Model| object handled by\n each |Element| object.\"\"\"\n for element in printtools.progressbar(self):\n element.model.sequences.load_conditions()"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsaves the calculated conditions of the model objects handled by each |Element| object.", "response": "def save_conditions(self) -> None:\n \"\"\"Save the calculated conditions of the |Model| object handled by\n each |Element| object.\"\"\"\n for element in printtools.progressbar(self):\n element.model.sequences.save_conditions()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef conditions(self) -> \\\n Dict[str, Dict[str, Dict[str, Union[float, numpy.ndarray]]]]:\n \"\"\"A nested dictionary containing the values of all\n |ConditionSequence| objects of all currently handled models.\n\n See the documentation on property |HydPy.conditions| for further\n information.\n \"\"\"\n return {element.name: element.model.sequences.conditions\n for element in self}", "response": "A nested dictionary containing the values of all currently handled models."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef prepare_allseries(self, ramflag: bool = True) -> None:\n for element in printtools.progressbar(self):\n element.prepare_allseries(ramflag)", "response": "Call method |Element. prepare_allseries| of all handled\n objects."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncall method |Element. prepare_inputseries| of all handled objects.", "response": "def prepare_inputseries(self, ramflag: bool = True) -> None:\n \"\"\"Call method |Element.prepare_inputseries| of all handled\n |Element| objects.\"\"\"\n for element in printtools.progressbar(self):\n element.prepare_inputseries(ramflag)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall method |Element. prepare_fluxseries| of all handled objects.", "response": "def prepare_fluxseries(self, ramflag: bool = True) -> None:\n \"\"\"Call method |Element.prepare_fluxseries| of all handled\n |Element| objects.\"\"\"\n for element in printtools.progressbar(self):\n element.prepare_fluxseries(ramflag)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef prepare_stateseries(self, ramflag: bool = True) -> None:\n for element in printtools.progressbar(self):\n element.prepare_stateseries(ramflag)", "response": "Call method |Element. prepare_stateseries| of all handled\n objects."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nextracts all new |Node| or |Element| objects.", "response": "def extract_new(cls) -> DevicesTypeUnbound:\n \"\"\"Gather all \"new\" |Node| or |Element| objects.\n\n See the main documentation on module |devicetools| for further\n information.\n \"\"\"\n devices = cls.get_handlerclass()(*_selection[cls])\n _selection[cls].clear()\n return devices"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_double(self, group: str) -> pointerutils.Double:\n if group in ('inlets', 'receivers'):\n if self.deploymode != 'obs':\n return self.sequences.fastaccess.sim\n return self.sequences.fastaccess.obs\n if group in ('outlets', 'senders'):\n if self.deploymode != 'oldsim':\n return self.sequences.fastaccess.sim\n return self.__blackhole\n raise ValueError(\n f'Function `get_double` of class `Node` does not '\n f'support the given group name `{group}`.')", "response": "Return the |Double| object appropriate for the given |Element| input or output group and the actual |Node. deploymode|."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_simseries(self, **kwargs: Any) -> None:\n self.__plot_series([self.sequences.sim], kwargs)", "response": "Plot the |IOSequence. series| of the Sim object."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_obsseries(self, **kwargs: Any) -> None:\n self.__plot_series([self.sequences.obs], kwargs)", "response": "Plot the |IOSequence. series| of the |Obs| sequence object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef assignrepr(self, prefix: str = '') -> str:\n lines = ['%sNode(\"%s\", variable=\"%s\",'\n % (prefix, self.name, self.variable)]\n if self.keywords:\n subprefix = '%skeywords=' % (' '*(len(prefix)+5))\n with objecttools.repr_.preserve_strings(True):\n with objecttools.assignrepr_tuple.always_bracketed(False):\n line = objecttools.assignrepr_list(\n sorted(self.keywords), subprefix, width=70)\n lines.append(line + ',')\n lines[-1] = lines[-1][:-1]+')'\n return '\\n'.join(lines)", "response": "Return a |repr| string with a prefixed assignment."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef model(self) -> 'modeltools.Model':\n model = vars(self).get('model')\n if model:\n return model\n raise AttributeError(\n f'The model object of element `{self.name}` has '\n f'been requested but not been prepared so far.')", "response": "The model object of the actual |Element| object."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef init_model(self, clear_registry: bool = True) -> None:\n try:\n with hydpy.pub.options.warnsimulationstep(False):\n info = hydpy.pub.controlmanager.load_file(\n element=self, clear_registry=clear_registry)\n self.model = info['model']\n self.model.parameters.update()\n except OSError:\n if hydpy.pub.options.warnmissingcontrolfile:\n warnings.warn(\n f'Due to a missing or no accessible control file, no '\n f'model could be initialised for element `{self.name}`')\n else:\n objecttools.augment_excmessage(\n f'While trying to initialise the model '\n f'object of element `{self.name}`')\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to initialise the model '\n f'object of element `{self.name}`')", "response": "Initialise the internal |Model| object of the actual |Element| object."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef variables(self) -> Set[str]:\n variables: Set[str] = set()\n for connection in self.__connections:\n variables.update(connection.variables)\n return variables", "response": "A set of all different variable values of the actual |Node| object."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprepare the |IOSequence. series| objects of all input flux and state sequences of the current element.", "response": "def prepare_allseries(self, ramflag: bool = True) -> None:\n \"\"\"Prepare the |IOSequence.series| objects of all `input`, `flux` and\n `state` sequences of the model handled by this element.\n\n Call this method before a simulation run, if you need access to\n (nearly) all simulated series of the handled model after the\n simulation run is finished.\n\n By default, the time series are stored in RAM, which is the faster\n option. If your RAM is limited, pass |False| to function argument\n `ramflag` to store the series on disk.\n \"\"\"\n self.prepare_inputseries(ramflag)\n self.prepare_fluxseries(ramflag)\n self.prepare_stateseries(ramflag)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_inputseries(\n self, names: Optional[Iterable[str]] = None,\n average: bool = False, **kwargs: Any) \\\n -> None:\n \"\"\"Plot (the selected) |InputSequence| |IOSequence.series| values.\n\n We demonstrate the functionalities of method |Element.plot_inputseries|\n based on the `Lahn` example project:\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, _, _ = prepare_full_example_2(lastdate='1997-01-01')\n\n Without any arguments, |Element.plot_inputseries| prints the\n time series of all input sequences handled by its |Model| object\n directly to the screen (in the given example, |hland_inputs.P|,\n |hland_inputs.T|, |hland_inputs.TN|, and |hland_inputs.EPN| of\n application model |hland_v1|):\n\n >>> land = hp.elements.land_dill\n >>> land.plot_inputseries()\n\n You can use the `pyplot` API of `matplotlib` to modify the figure\n or to save it to disk (or print it to the screen, in case the\n interactive mode of `matplotlib` is disabled):\n\n >>> from matplotlib import pyplot\n >>> from hydpy.docs import figs\n >>> pyplot.savefig(figs.__path__[0] + '/Element_plot_inputseries.png')\n >>> pyplot.close()\n\n .. image:: Element_plot_inputseries.png\n\n Methods |Element.plot_fluxseries| and |Element.plot_stateseries|\n work in the same manner. Before applying them, one has at first\n to calculate the time series of the |FluxSequence| and\n |StateSequence| objects:\n\n >>> hp.doit()\n\n All three methods allow to select certain sequences by passing their\n names (here, flux sequences |hland_fluxes.Q0| and |hland_fluxes.Q1|\n of |hland_v1|). Additionally, you can pass the keyword arguments\n supported by `matplotlib` for modifying the line style:\n\n >>> land.plot_fluxseries(['q0', 'q1'], linewidth=2)\n\n >>> pyplot.savefig(figs.__path__[0] + '/Element_plot_fluxseries.png')\n >>> pyplot.close()\n\n .. image:: Element_plot_fluxseries.png\n\n For 1-dimensional |IOSequence| objects, all three methods plot the\n individual time series in the same colour (here, from the state\n sequences |hland_states.SP| and |hland_states.WC| of |hland_v1|):\n\n >>> land.plot_stateseries(['sp', 'wc'])\n\n >>> pyplot.savefig(figs.__path__[0] + '/Element_plot_stateseries1.png')\n >>> pyplot.close()\n\n .. image:: Element_plot_stateseries1.png\n\n Alternatively, you can print the averaged time series through\n passing |True| to the method `average` argument (demonstrated\n for the state sequence |hland_states.SM|):\n\n >>> land.plot_stateseries(['sm'], color='grey')\n >>> land.plot_stateseries(\n ... ['sm'], average=True, color='black', linewidth=3)\n\n >>> pyplot.savefig(figs.__path__[0] + '/Element_plot_stateseries2.png')\n >>> pyplot.close()\n\n .. image:: Element_plot_stateseries2.png\n \"\"\"\n self.__plot(self.model.sequences.inputs, names, average, kwargs)", "response": "Plot the time series of all input sequences handled by the object."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nplotting the flux series of the handled model.", "response": "def plot_fluxseries(\n self, names: Optional[Iterable[str]] = None,\n average: bool = False, **kwargs: Any) \\\n -> None:\n \"\"\"Plot the `flux` series of the handled model.\n\n See the documentation on method |Element.plot_inputseries| for\n additional information.\n \"\"\"\n self.__plot(self.model.sequences.fluxes, names, average, kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nplot the state series of the handled model.", "response": "def plot_stateseries(\n self, names: Optional[Iterable[str]] = None,\n average: bool = False, **kwargs: Any) \\\n -> None:\n \"\"\"Plot the `state` series of the handled model.\n\n See the documentation on method |Element.plot_inputseries| for\n additional information.\n \"\"\"\n self.__plot(self.model.sequences.states, names, average, kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a |repr| string with a prefixed assignment.", "response": "def assignrepr(self, prefix: str) -> str:\n \"\"\"Return a |repr| string with a prefixed assignment.\"\"\"\n with objecttools.repr_.preserve_strings(True):\n with objecttools.assignrepr_tuple.always_bracketed(False):\n blanks = ' ' * (len(prefix) + 8)\n lines = ['%sElement(\"%s\",' % (prefix, self.name)]\n for groupname in ('inlets', 'outlets', 'receivers', 'senders'):\n group = getattr(self, groupname, Node)\n if group:\n subprefix = '%s%s=' % (blanks, groupname)\n # pylint: disable=not-an-iterable\n # because pylint is wrong\n nodes = [str(node) for node in group]\n # pylint: enable=not-an-iterable\n line = objecttools.assignrepr_list(\n nodes, subprefix, width=70)\n lines.append(line + ',')\n if self.keywords:\n subprefix = '%skeywords=' % blanks\n line = objecttools.assignrepr_list(\n sorted(self.keywords), subprefix, width=70)\n lines.append(line + ',')\n lines[-1] = lines[-1][:-1]+')'\n return '\\n'.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _init_methods(self):\n for name_group in self._METHOD_GROUPS:\n functions = getattr(self, name_group, ())\n uniques = {}\n for func in functions:\n name_func = func.__name__\n method = types.MethodType(func, self)\n setattr(self, name_func, method)\n shortname = '_'.join(name_func.split('_')[:-1])\n if shortname in uniques:\n uniques[shortname] = None\n else:\n uniques[shortname] = method\n for (shortname, method) in uniques.items():\n if method is not None:\n setattr(self, shortname, method)", "response": "Convert all pure Python calculation functions of the model class to\n methods and assign them to the model instance."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nnaming of the model type.", "response": "def name(self):\n \"\"\"Name of the model type.\n\n For base models, |Model.name| corresponds to the package name:\n\n >>> from hydpy import prepare_model\n >>> hland = prepare_model('hland')\n >>> hland.name\n 'hland'\n\n For application models, |Model.name| corresponds the module name:\n\n >>> hland_v1 = prepare_model('hland_v1')\n >>> hland_v1.name\n 'hland_v1'\n\n This last example has only technical reasons:\n\n >>> hland.name\n 'hland'\n \"\"\"\n name = self.__name\n if name:\n return name\n subs = self.__module__.split('.')\n if len(subs) == 2:\n type(self).__name = subs[1]\n else:\n type(self).__name = subs[2]\n return self.__name"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconnecting the link sequences of the actual model to the actual node.", "response": "def connect(self):\n \"\"\"Connect the link sequences of the actual model.\"\"\"\n try:\n for group in ('inlets', 'receivers', 'outlets', 'senders'):\n self._connect_subgroup(group)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to build the node connection of the `%s` '\n 'sequences of the model handled by element `%s`'\n % (group[:-1], objecttools.devicename(self)))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\napply all methods stored in the hidden attribute PART_ODE_METHODS.", "response": "def calculate_single_terms(self):\n \"\"\"Apply all methods stored in the hidden attribute\n `PART_ODE_METHODS`.\n\n >>> from hydpy.models.test_v1 import *\n >>> parameterstep()\n >>> k(0.25)\n >>> states.s = 1.0\n >>> model.calculate_single_terms()\n >>> fluxes.q\n q(0.25)\n \"\"\"\n self.numvars.nmb_calls = self.numvars.nmb_calls+1\n for method in self.PART_ODE_METHODS:\n method(self)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the sum of the fluxes calculated so far.", "response": "def get_sum_fluxes(self):\n \"\"\"Get the sum of the fluxes calculated so far.\n\n >>> from hydpy.models.test_v1 import *\n >>> parameterstep()\n >>> fluxes.q = 0.0\n >>> fluxes.fastaccess._q_sum = 1.0\n >>> model.get_sum_fluxes()\n >>> fluxes.q\n q(1.0)\n \"\"\"\n fluxes = self.sequences.fluxes\n for flux in fluxes.numerics:\n flux(getattr(fluxes.fastaccess, '_%s_sum' % flux.name))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nintegrating the fluxes between the two sequences.", "response": "def integrate_fluxes(self):\n \"\"\"Perform a dot multiplication between the fluxes and the\n A coefficients associated with the different stages of the\n actual method.\n\n >>> from hydpy.models.test_v1 import *\n >>> parameterstep()\n >>> model.numvars.idx_method = 2\n >>> model.numvars.idx_stage = 1\n >>> model.numvars.dt = 0.5\n >>> points = numpy.asarray(fluxes.fastaccess._q_points)\n >>> points[:4] = 15., 2., -999., 0.\n >>> model.integrate_fluxes()\n >>> from hydpy import round_\n >>> from hydpy import pub\n >>> round_(numpy.asarray(model.numconsts.a_coefs)[1, 1, :2])\n 0.375, 0.125\n >>> fluxes.q\n q(2.9375)\n \"\"\"\n fluxes = self.sequences.fluxes\n for flux in fluxes.numerics:\n points = getattr(fluxes.fastaccess, '_%s_points' % flux.name)\n coefs = self.numconsts.a_coefs[self.numvars.idx_method-1,\n self.numvars.idx_stage,\n :self.numvars.idx_method]\n flux(self.numvars.dt *\n numpy.dot(coefs, points[:self.numvars.idx_method]))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreset the sum of the fluxes calculated so far to zero.", "response": "def reset_sum_fluxes(self):\n \"\"\"Set the sum of the fluxes calculated so far to zero.\n\n >>> from hydpy.models.test_v1 import *\n >>> parameterstep()\n >>> fluxes.fastaccess._q_sum = 5.\n >>> model.reset_sum_fluxes()\n >>> fluxes.fastaccess._q_sum\n 0.0\n \"\"\"\n fluxes = self.sequences.fluxes\n for flux in fluxes.numerics:\n if flux.NDIM == 0:\n setattr(fluxes.fastaccess, '_%s_sum' % flux.name, 0.)\n else:\n getattr(fluxes.fastaccess, '_%s_sum' % flux.name)[:] = 0."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef addup_fluxes(self):\n fluxes = self.sequences.fluxes\n for flux in fluxes.numerics:\n sum_ = getattr(fluxes.fastaccess, '_%s_sum' % flux.name)\n sum_ += flux\n if flux.NDIM == 0:\n setattr(fluxes.fastaccess, '_%s_sum' % flux.name, sum_)", "response": "Add up the sum of the fluxes calculated so far."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calculate_error(self):\n self.numvars.error = 0.\n fluxes = self.sequences.fluxes\n for flux in fluxes.numerics:\n results = getattr(fluxes.fastaccess, '_%s_results' % flux.name)\n diff = (results[self.numvars.idx_method] -\n results[self.numvars.idx_method-1])\n self.numvars.error = max(self.numvars.error,\n numpy.max(numpy.abs(diff)))", "response": "Estimate the numerical error based on the fluxes calculated\n by the current and the last method."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nestimating the numerical error to be expected when applying all methods available.", "response": "def extrapolate_error(self):\n \"\"\"Estimate the numerical error to be expected when applying all\n methods available based on the results of the current and the\n last method.\n\n Note that this expolation strategy cannot be applied on the first\n method. If the current method is the first one, `-999.9` is returned.\n\n >>> from hydpy.models.test_v1 import *\n >>> parameterstep()\n >>> model.numvars.error = 1e-2\n >>> model.numvars.last_error = 1e-1\n >>> model.numvars.idx_method = 10\n >>> model.extrapolate_error()\n >>> from hydpy import round_\n >>> round_(model.numvars.extrapolated_error)\n 0.01\n >>> model.numvars.idx_method = 9\n >>> model.extrapolate_error()\n >>> round_(model.numvars.extrapolated_error)\n 0.001\n \"\"\"\n if self.numvars.idx_method > 2:\n self.numvars.extrapolated_error = modelutils.exp(\n modelutils.log(self.numvars.error) +\n (modelutils.log(self.numvars.error) -\n modelutils.log(self.numvars.last_error)) *\n (self.numconsts.nmb_methods-self.numvars.idx_method))\n else:\n self.numvars.extrapolated_error = -999.9"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform a HydPy simulation in agreement with the given XML configuration file.", "response": "def run_simulation(projectname: str, xmlfile: str):\n \"\"\"Perform a HydPy workflow in agreement with the given XML configuration\n file available in the directory of the given project. ToDo\n\n Function |run_simulation| is a \"script function\" and is normally used as\n explained in the main documentation on module |xmltools|.\n \"\"\"\n write = commandtools.print_textandtime\n hydpy.pub.options.printprogress = False\n write(f'Start HydPy project `{projectname}`')\n hp = hydpytools.HydPy(projectname)\n write(f'Read configuration file `{xmlfile}`')\n interface = XMLInterface(xmlfile)\n write('Interpret the defined options')\n interface.update_options()\n hydpy.pub.options.printprogress = False\n write('Interpret the defined period')\n interface.update_timegrids()\n write('Read all network files')\n hp.prepare_network()\n write('Activate the selected network')\n hp.update_devices(interface.fullselection)\n write('Read the required control files')\n hp.init_models()\n write('Read the required condition files')\n interface.conditions_io.load_conditions()\n write('Read the required time series files')\n interface.series_io.prepare_series()\n interface.series_io.load_series()\n write('Perform the simulation run')\n hp.doit()\n write('Write the desired condition files')\n interface.conditions_io.save_conditions()\n write('Write the desired time series files')\n interface.series_io.save_series()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef validate_xml(self) -> None:\n try:\n filenames = ('HydPyConfigSingleRun.xsd',\n 'HydPyConfigMultipleRuns.xsd')\n for name in filenames:\n if name in self.root.tag:\n schemafile = name\n break\n else:\n raise RuntimeError(\n f'Configuration file `{os.path.split(self.filepath)[-1]}` '\n f'does not correctly refer to one of the available XML '\n f'schema files ({objecttools.enumeration(filenames)}).')\n schemapath = os.path.join(conf.__path__[0], schemafile)\n schema = xmlschema.XMLSchema(schemapath)\n schema.validate(self.filepath)\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to validate XML file `{self.filepath}`')", "response": "Raise an error if the actual XML does not agree with one of the available schema files."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_options(self) -> None:\n options = hydpy.pub.options\n for option in self.find('options'):\n value = option.text\n if value in ('true', 'false'):\n value = value == 'true'\n setattr(options, strip(option.tag), value)\n options.printprogress = False\n options.printincolor = False", "response": "Update the |Options| object available in module |pub| with the values defined in the options XML element."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_timegrids(self) -> None:\n timegrid_xml = self.find('timegrid')\n try:\n timegrid = timetools.Timegrid(\n *(timegrid_xml[idx].text for idx in range(3)))\n hydpy.pub.timegrids = timetools.Timegrids(timegrid)\n except IndexError:\n seriesfile = find(timegrid_xml, 'seriesfile').text\n with netcdf4.Dataset(seriesfile) as ncfile:\n hydpy.pub.timegrids = timetools.Timegrids(\n netcdftools.query_timegrid(ncfile))", "response": "Update the |Timegrids| object with the values defined in the timegrid XML element."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef elements(self) -> Iterator[devicetools.Element]:\n selections = copy.copy(self.selections)\n selections += self.devices\n elements = set()\n for selection in selections:\n for element in selection.elements:\n if element not in elements:\n elements.add(element)\n yield element", "response": "Yield all |Element| objects returned by |XMLInterface. selections| and |XMLInterface. devices| without duplicates."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_conditions(self) -> None:\n hydpy.pub.conditionmanager.currentdir = strip(\n self.find('inputdir').text)\n for element in self.master.elements:\n element.model.sequences.load_conditions()", "response": "Load the condition files of all |Element| objects of all |Element| objects of all |XMLInterface. elements|."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef save_conditions(self) -> None:\n hydpy.pub.conditionmanager.currentdir = strip(\n self.find('outputdir').text)\n for element in self.master.elements:\n element.model.sequences.save_conditions()\n if strip(self.find('zip').text) == 'true':\n hydpy.pub.conditionmanager.zip_currentdir()", "response": "Save the conditions of the |Model| objects of all |Element| objects returned by |XMLInterface. elements|."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef prepare_series(self) -> None:\n # noinspection PyUnresolvedReferences\n \"\"\"Call |XMLSubseries.prepare_series| of all |XMLSubseries|\n objects with the same memory |set| object.\n\n >>> from hydpy.auxs.xmltools import XMLInterface, XMLSubseries\n >>> from hydpy import data\n >>> interface = XMLInterface('single_run.xml', data.get_path('LahnH'))\n >>> series_io = interface.series_io\n >>> from unittest import mock\n >>> prepare_series = XMLSubseries.prepare_series\n >>> XMLSubseries.prepare_series = mock.MagicMock()\n >>> series_io.prepare_series()\n >>> args = XMLSubseries.prepare_series.call_args_list\n >>> len(args) == len(series_io.readers) + len(series_io.writers)\n True\n >>> args[0][0][0]\n set()\n >>> args[0][0][0] is args[-1][0][0]\n True\n >>> XMLSubseries.prepare_series = prepare_series\n \"\"\"\n memory = set()\n for output in itertools.chain(self.readers, self.writers):\n output.prepare_series(memory)", "response": "Call |XMLSubseries. prepare_series| of all |XMLSubseries| objects with the same memory |set| object."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef selections(self) -> selectiontools.Selections:\n selections = self.find('selections')\n master = self\n while selections is None:\n master = master.master\n selections = master.find('selections')\n return _query_selections(selections)", "response": "The |Selections| object defined for the respective reader or writer element of the actual XML file. ToDo\n is used."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef prepare_sequencemanager(self) -> None:\n for config, convert in (\n ('filetype', lambda x: x),\n ('aggregation', lambda x: x),\n ('overwrite', lambda x: x.lower() == 'true'),\n ('dirpath', lambda x: x)):\n xml_special = self.find(config)\n xml_general = self.master.find(config)\n for name_manager, name_xml in zip(\n ('input', 'flux', 'state', 'node'),\n ('inputs', 'fluxes', 'states', 'nodes')):\n value = None\n for xml, attr_xml in zip(\n (xml_special, xml_special, xml_general, xml_general),\n (name_xml, 'general', name_xml, 'general')):\n try:\n value = find(xml, attr_xml).text\n except AttributeError:\n continue\n break\n setattr(hydpy.pub.sequencemanager,\n f'{name_manager}{config}',\n convert(value))", "response": "Configure the |SequenceManager| object available in module\n ."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef prepare_series(self, memory: set) -> None:\n for sequence in self._iterate_sequences():\n if sequence not in memory:\n memory.add(sequence)\n sequence.activate_ram()", "response": "Call |IOSequence. activate_ram| of all sequences selected by\n |IOSequence. deactivate_ram| of all sequences selected by\n |IOSequence. prepare_series| of all sequences selected by\n |IOSequence. prepare_full_example_1| of the full XML file."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_series(self) -> None:\n kwargs = {}\n for keyword in ('flattennetcdf', 'isolatenetcdf', 'timeaxisnetcdf'):\n argument = getattr(hydpy.pub.options, keyword, None)\n if argument is not None:\n kwargs[keyword[:-6]] = argument\n hydpy.pub.sequencemanager.open_netcdf_reader(**kwargs)\n self.prepare_sequencemanager()\n for sequence in self._iterate_sequences():\n sequence.load_ext()\n hydpy.pub.sequencemanager.close_netcdf_reader()", "response": "Load time series data as defined by the actual XML reader."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsaves time series data as defined by the actual XML writer.", "response": "def save_series(self) -> None:\n \"\"\"Save time series data as defined by the actual XML `writer`\n element.\n\n >>> from hydpy.core.examples import prepare_full_example_1\n >>> prepare_full_example_1()\n\n >>> from hydpy import HydPy, TestIO, XMLInterface\n >>> hp = HydPy('LahnH')\n >>> with TestIO():\n ... hp.prepare_network()\n ... hp.init_models()\n ... interface = XMLInterface('single_run.xml')\n ... interface.update_options()\n >>> interface.update_timegrids()\n >>> series_io = interface.series_io\n >>> series_io.prepare_series()\n >>> hp.elements.land_dill.model.sequences.fluxes.pc.series[2, 3] = 9.0\n >>> hp.nodes.lahn_2.sequences.sim.series[4] = 7.0\n >>> with TestIO():\n ... series_io.save_series()\n >>> import numpy\n >>> with TestIO():\n ... os.path.exists(\n ... 'LahnH/series/output/land_lahn_2_flux_pc.npy')\n ... os.path.exists(\n ... 'LahnH/series/output/land_lahn_3_flux_pc.npy')\n ... numpy.load(\n ... 'LahnH/series/output/land_dill_flux_pc.npy')[13+2, 3]\n ... numpy.load(\n ... 'LahnH/series/output/lahn_2_sim_q_mean.npy')[13+4]\n True\n False\n 9.0\n 7.0\n \"\"\"\n hydpy.pub.sequencemanager.open_netcdf_writer(\n flatten=hydpy.pub.options.flattennetcdf,\n isolate=hydpy.pub.options.isolatenetcdf)\n self.prepare_sequencemanager()\n for sequence in self._iterate_sequences():\n sequence.save_ext()\n hydpy.pub.sequencemanager.close_netcdf_writer()"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef item(self):\n target = f'{self.master.name}.{self.name}'\n if self.master.name == 'nodes':\n master = self.master.name\n itemgroup = self.master.master.name\n else:\n master = self.master.master.name\n itemgroup = self.master.master.master.name\n itemclass = _ITEMGROUP2ITEMCLASS[itemgroup]\n if itemgroup == 'getitems':\n return self._get_getitem(target, master, itemclass)\n return self._get_changeitem(target, master, itemclass, itemgroup)", "response": "ToDo\n is a simple example of how to get the current item."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite the complete base schema file for hydpy.", "response": "def write_xsd(cls) -> None:\n \"\"\"Write the complete base schema file `HydPyConfigBase.xsd` based\n on the template file `HydPyConfigBase.xsdt`.\n\n Method |XSDWriter.write_xsd| adds model specific information to the\n general information of template file `HydPyConfigBase.xsdt` regarding\n reading and writing of time series data and exchanging parameter\n and sequence values e.g. during calibration.\n\n The following example shows that after writing a new schema file,\n method |XMLInterface.validate_xml| does not raise an error when\n either applied on the XML configuration files `single_run.xml` or\n `multiple_runs.xml` of the `LahnH` example project:\n\n >>> import os\n >>> from hydpy.auxs.xmltools import XSDWriter, XMLInterface\n >>> if os.path.exists(XSDWriter.filepath_target):\n ... os.remove(XSDWriter.filepath_target)\n >>> os.path.exists(XSDWriter.filepath_target)\n False\n >>> XSDWriter.write_xsd()\n >>> os.path.exists(XSDWriter.filepath_target)\n True\n\n >>> from hydpy import data\n >>> for configfile in ('single_run.xml', 'multiple_runs.xml'):\n ... XMLInterface(configfile, data.get_path('LahnH')).validate_xml()\n \"\"\"\n with open(cls.filepath_source) as file_:\n template = file_.read()\n template = template.replace(\n '', cls.get_insertion())\n template = template.replace(\n '', cls.get_exchangeinsertion())\n with open(cls.filepath_target, 'w') as file_:\n file_.write(template)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a sorted |list| containing all application model names.", "response": "def get_modelnames() -> List[str]:\n \"\"\"Return a sorted |list| containing all application model names.\n\n >>> from hydpy.auxs.xmltools import XSDWriter\n >>> print(XSDWriter.get_modelnames()) # doctest: +ELLIPSIS\n [...'dam_v001', 'dam_v002', 'dam_v003', 'dam_v004', 'dam_v005',...]\n \"\"\"\n return sorted(str(fn.split('.')[0])\n for fn in os.listdir(models.__path__[0])\n if (fn.endswith('.py') and (fn != '__init__.py')))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the insertion string required for the given application model.", "response": "def get_modelinsertion(cls, model, indent) -> str:\n \"\"\"Return the insertion string required for the given application model.\n\n >>> from hydpy.auxs.xmltools import XSDWriter\n >>> from hydpy import prepare_model\n >>> model = prepare_model('hland_v1')\n >>> print(XSDWriter.get_modelinsertion(model, 1)) # doctest: +ELLIPSIS\n \n \n \n \n ...\n \n \n ...\n \n \n ...\n \n \"\"\"\n texts = []\n for name in ('inputs', 'fluxes', 'states'):\n subsequences = getattr(model.sequences, name, None)\n if subsequences:\n texts.append(\n cls.get_subsequencesinsertion(subsequences, indent))\n return '\\n'.join(texts)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the insertion string required for the given group of sequences.", "response": "def get_subsequencesinsertion(cls, subsequences, indent) -> str:\n \"\"\"Return the insertion string required for the given group of\n sequences.\n\n >>> from hydpy.auxs.xmltools import XSDWriter\n >>> from hydpy import prepare_model\n >>> model = prepare_model('hland_v1')\n >>> print(XSDWriter.get_subsequencesinsertion(\n ... model.sequences.fluxes, 1)) # doctest: +ELLIPSIS\n \n \n \n \n \n ...\n \n \n \n \n\n \"\"\"\n blanks = ' ' * (indent*4)\n lines = [f'{blanks}',\n f'{blanks} ',\n f'{blanks} ']\n for sequence in subsequences:\n lines.append(cls.get_sequenceinsertion(sequence, indent + 3))\n lines.extend([f'{blanks} ',\n f'{blanks} ',\n f'{blanks}'])\n return '\\n'.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the complete string related to the definition of exchange items to be inserted into the string of the template file.", "response": "def get_exchangeinsertion(cls):\n \"\"\"Return the complete string related to the definition of exchange\n items to be inserted into the string of the template file.\n\n >>> from hydpy.auxs.xmltools import XSDWriter\n >>> print(XSDWriter.get_exchangeinsertion()) # doctest: +ELLIPSIS\n \n ...\n \n ...\n \n ...\n \n ...\n \n ...\n \"\"\"\n indent = 1\n subs = [cls.get_mathitemsinsertion(indent)]\n for groupname in ('setitems', 'additems', 'getitems'):\n subs.append(cls.get_itemsinsertion(groupname, indent))\n subs.append(cls.get_itemtypesinsertion(groupname, indent))\n return '\\n'.join(subs)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_mathitemsinsertion(cls, indent) -> str:\n blanks = ' ' * (indent*4)\n subs = []\n for modelname in cls.get_modelnames():\n model = importtools.prepare_model(modelname)\n subs.extend([\n f'{blanks}',\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks} '])\n for subvars in cls._get_subvars(model):\n for var in subvars:\n subs.append(\n f'{blanks} '\n f'')\n subs.extend([\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks}',\n f''])\n return '\\n'.join(subs)", "response": "Return a string defining a model specific XML type extending\n ."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_itemsinsertion(cls, itemgroup, indent) -> str:\n blanks = ' ' * (indent*4)\n subs = []\n subs.extend([\n f'{blanks}',\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks} '])\n for modelname in cls.get_modelnames():\n type_ = cls._get_itemstype(modelname, itemgroup)\n subs.append(f'{blanks} ')\n if itemgroup in ('setitems', 'getitems'):\n type_ = f'nodes_{itemgroup}Type'\n subs.append(f'{blanks} ')\n subs.extend([\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks}',\n f''])\n return '\\n'.join(subs)", "response": "Return a string defining the XML element for the given item group."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a string defining the required types for the given item group.", "response": "def get_itemtypesinsertion(cls, itemgroup, indent) -> str:\n \"\"\"Return a string defining the required types for the given\n exchange item group.\n\n >>> from hydpy.auxs.xmltools import XSDWriter\n >>> print(XSDWriter.get_itemtypesinsertion(\n ... 'setitems', 1)) # doctest: +ELLIPSIS\n \n ...\n \n \n \n ...\n \n ...\n \"\"\"\n subs = []\n for modelname in cls.get_modelnames():\n subs.append(cls.get_itemtypeinsertion(itemgroup, modelname, indent))\n subs.append(cls.get_nodesitemtypeinsertion(itemgroup, indent))\n return '\\n'.join(subs)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a string defining the required types for the given itemgroup and an application model.", "response": "def get_itemtypeinsertion(cls, itemgroup, modelname, indent) -> str:\n \"\"\"Return a string defining the required types for the given\n combination of an exchange item group and an application model.\n\n >>> from hydpy.auxs.xmltools import XSDWriter\n >>> print(XSDWriter.get_itemtypeinsertion(\n ... 'setitems', 'hland_v1', 1)) # doctest: +ELLIPSIS\n \n \n \n \n \n ...\n \n \n \n \"\"\"\n blanks = ' ' * (indent * 4)\n type_ = cls._get_itemstype(modelname, itemgroup)\n subs = [\n f'{blanks}',\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks} ',\n cls.get_subgroupsiteminsertion(itemgroup, modelname, indent+2),\n f'{blanks} ',\n f'{blanks}',\n f'']\n return '\\n'.join(subs)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_nodesitemtypeinsertion(cls, itemgroup, indent) -> str:\n blanks = ' ' * (indent * 4)\n subs = [\n f'{blanks}',\n f'{blanks} ',\n f'{blanks} ',\n f'{blanks} ']\n type_ = 'getitemType' if itemgroup == 'getitems' else 'setitemType'\n for name in ('sim', 'obs', 'sim.series', 'obs.series'):\n subs.extend([\n f'{blanks} '])\n subs.extend([\n f'{blanks} ',\n f'{blanks}',\n f''])\n return '\\n'.join(subs)", "response": "Return a string defining the required types for the given item group and |Node| objects."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_subgroupsiteminsertion(cls, itemgroup, modelname, indent) -> str:\n subs = []\n model = importtools.prepare_model(modelname)\n for subvars in cls._get_subvars(model):\n subs.append(cls.get_subgroupiteminsertion(\n itemgroup, model, subvars, indent))\n return '\\n'.join(subs)", "response": "Return a string defining the required types for the given item group and an application model."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_subgroupiteminsertion(\n cls, itemgroup, model, subgroup, indent) -> str:\n \"\"\"Return a string defining the required types for the given\n combination of an exchange item group and a specific variable\n subgroup of an application model or class |Node|.\n\n Note that for `setitems` and `getitems` `setitemType` and\n `getitemType` are referenced, respectively, and for all others\n the model specific `mathitemType`:\n\n >>> from hydpy import prepare_model\n >>> model = prepare_model('hland_v1')\n >>> from hydpy.auxs.xmltools import XSDWriter\n >>> print(XSDWriter.get_subgroupiteminsertion( # doctest: +ELLIPSIS\n ... 'setitems', model, model.parameters.control, 1))\n \n \n \n \n \n \n \n \n \n\n >>> print(XSDWriter.get_subgroupiteminsertion( # doctest: +ELLIPSIS\n ... 'getitems', model, model.parameters.control, 1))\n \n ...\n\n >>> print(XSDWriter.get_subgroupiteminsertion( # doctest: +ELLIPSIS\n ... 'additems', model, model.parameters.control, 1))\n \n ...\n\n For sequence classes, additional \"series\" elements are added:\n\n >>> print(XSDWriter.get_subgroupiteminsertion( # doctest: +ELLIPSIS\n ... 'setitems', model, model.sequences.fluxes, 1))\n \n \n \n \n \n \"\"\"\n blanks1 = ' ' * (indent * 4)\n blanks2 = ' ' * ((indent+5) * 4 + 1)\n subs = [\n f'{blanks1}',\n f'{blanks1} ',\n f'{blanks1} ',\n f'{blanks1} ',\n f'{blanks1} ']\n seriesflags = (False,) if subgroup.name == 'control' else (False, True)\n for variable in subgroup:\n for series in seriesflags:\n name = f'{variable.name}.series' if series else variable.name\n subs.append(f'{blanks1} ')\n subs.extend([\n f'{blanks1} ',\n f'{blanks1} ',\n f'{blanks1}'])\n return '\\n'.join(subs)", "response": "Return a string defining the required types for the given item group and a specific variable\n subgroup of an application model or class |Node|."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef array2mask(cls, array=None, **kwargs):\n kwargs['dtype'] = bool\n if array is None:\n return numpy.ndarray.__new__(cls, 0, **kwargs)\n return numpy.asarray(array, **kwargs).view(cls)", "response": "Create a new mask object based on the given |numpy. ndarray| and return it."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef new(cls, variable, **kwargs):\n return cls.array2mask(numpy.full(variable.shape, True))", "response": "Return a new |DefaultMask| object associated with the\n given |Variable| object."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef new(cls, variable, **kwargs):\n indices = cls.get_refindices(variable)\n if numpy.min(getattr(indices, 'values', 0)) < 1:\n raise RuntimeError(\n f'The mask of parameter {objecttools.elementphrase(variable)} '\n f'cannot be determined, as long as parameter `{indices.name}` '\n f'is not prepared properly.')\n mask = numpy.full(indices.shape, False, dtype=bool)\n refvalues = indices.values\n for relvalue in cls.RELEVANT_VALUES:\n mask[refvalues == relvalue] = True\n return cls.array2mask(mask, **kwargs)", "response": "Return a |IndexMask| object of the same shape as the\n parameter referenced by |Property|. The |IndexMask| object is only |True| if the respective entries of the referenced parameter are contained in the respective entries of the respective parameter."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef relevantindices(self) -> List[int]:\n return [idx for idx in numpy.unique(self.refindices.values)\n if idx in self.RELEVANT_VALUES]", "response": "A |list| of all currently relevant indices calculated as an\n intercection of the class attribute RELEVANT_VALUES and the property |IndexMask. refindices|."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_qref_v1(self):\n new = self.sequences.states.fastaccess_new\n old = self.sequences.states.fastaccess_old\n flu = self.sequences.fluxes.fastaccess\n flu.qref = (new.qz+old.qz+old.qa)/3.", "response": "Determine the reference discharge within the given space - time interval."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndetermine the actual traveling time of the water.", "response": "def calc_rk_v1(self):\n \"\"\"Determine the actual traveling time of the water (not of the wave!).\n\n Required derived parameter:\n |Sek|\n\n Required flux sequences:\n |AG|\n |QRef|\n\n Calculated flux sequence:\n |RK|\n\n Basic equation:\n :math:`RK = \\\\frac{Laen \\\\cdot A}{QRef}`\n\n Examples:\n\n First, note that the traveling time is determined in the unit of the\n actual simulation step size:\n\n >>> from hydpy.models.lstream import *\n >>> parameterstep()\n >>> laen(25.0)\n >>> derived.sek(24*60*60)\n >>> fluxes.ag = 10.0\n >>> fluxes.qref = 1.0\n >>> model.calc_rk_v1()\n >>> fluxes.rk\n rk(2.893519)\n\n Second, for negative values or zero values of |AG| or |QRef|,\n the value of |RK| is set to zero:\n\n >>> fluxes.ag = 0.0\n >>> fluxes.qref = 1.0\n >>> model.calc_rk_v1()\n >>> fluxes.rk\n rk(0.0)\n\n >>> fluxes.ag = 0.0\n >>> fluxes.qref = 1.0\n >>> model.calc_rk_v1()\n >>> fluxes.rk\n rk(0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n if (flu.ag > 0.) and (flu.qref > 0.):\n flu.rk = (1000.*con.laen*flu.ag)/(der.sek*flu.qref)\n else:\n flu.rk = 0."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_am_um_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n if flu.h <= 0.:\n flu.am = 0.\n flu.um = 0.\n elif flu.h < con.hm:\n flu.am = flu.h*(con.bm+flu.h*con.bnm)\n flu.um = con.bm+2.*flu.h*(1.+con.bnm**2)**.5\n else:\n flu.am = (con.hm*(con.bm+con.hm*con.bnm) +\n ((flu.h-con.hm)*(con.bm+2.*con.hm*con.bnm)))\n flu.um = con.bm+(2.*con.hm*(1.+con.bnm**2)**.5)+(2*(flu.h-con.hm))", "response": "Calculates the flown through area and the wetted perimeter of the main channel."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_qm_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n if (flu.am > 0.) and (flu.um > 0.):\n flu.qm = con.ekm*con.skm*flu.am**(5./3.)/flu.um**(2./3.)*con.gef**.5\n else:\n flu.qm = 0.", "response": "Calculate the discharge of the main channel after Manning - Strickler."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the flown through area and wetted perimeter of the given forelands and the flown through the given area and the wetted perimeter of the given forelands.", "response": "def calc_av_uv_v1(self):\n \"\"\"Calculate the flown through area and the wetted perimeter of both\n forelands.\n\n Note that the each foreland lies between the main channel and one\n outer embankment and that water flowing exactly above the a foreland\n is contributing to |AV|. The theoretical surface seperating water\n above the main channel from water above the foreland is not\n contributing to |UV|, but the surface seperating water above the\n foreland from water above its outer embankment is contributing to |UV|.\n\n Required control parameters:\n |HM|\n |BV|\n |BNV|\n\n Required derived parameter:\n |HV|\n\n Required flux sequence:\n |H|\n\n Calculated flux sequence:\n |AV|\n |UV|\n\n Examples:\n\n Generally, right trapezoids are assumed. Here, for simplicity, both\n forelands are assumed to be symmetrical. Their smaller bases (bottoms)\n hava a length of 2 meters, their non-vertical legs show an inclination\n of 1 meter per 4 meters, and their height (depths) is 1 meter. Both\n forelands lie 1 meter above the main channels bottom.\n\n >>> from hydpy.models.lstream import *\n >>> parameterstep()\n >>> hm(1.0)\n >>> bv(2.0)\n >>> bnv(4.0)\n >>> derived.hv(1.0)\n\n The first example deals with normal flow conditions, where water flows\n within the main channel completely (|H| < |HM|):\n\n >>> fluxes.h = 0.5\n >>> model.calc_av_uv_v1()\n >>> fluxes.av\n av(0.0, 0.0)\n >>> fluxes.uv\n uv(0.0, 0.0)\n\n The second example deals with moderate high flow conditions, where\n water flows over both forelands, but not over their embankments\n (|HM| < |H| < (|HM| + |HV|)):\n\n >>> fluxes.h = 1.5\n >>> model.calc_av_uv_v1()\n >>> fluxes.av\n av(1.5, 1.5)\n >>> fluxes.uv\n uv(4.061553, 4.061553)\n\n The third example deals with extreme high flow conditions, where\n water flows over the both foreland and their outer embankments\n ((|HM| + |HV|) < |H|):\n\n >>> fluxes.h = 2.5\n >>> model.calc_av_uv_v1()\n >>> fluxes.av\n av(7.0, 7.0)\n >>> fluxes.uv\n uv(6.623106, 6.623106)\n\n The forth example assures that zero widths or hights of the forelands\n are handled properly:\n\n >>> bv.left = 0.0\n >>> derived.hv.right = 0.0\n >>> model.calc_av_uv_v1()\n >>> fluxes.av\n av(4.0, 3.0)\n >>> fluxes.uv\n uv(4.623106, 3.5)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for i in range(2):\n if flu.h <= con.hm:\n flu.av[i] = 0.\n flu.uv[i] = 0.\n elif flu.h <= (con.hm+der.hv[i]):\n flu.av[i] = (flu.h-con.hm)*(con.bv[i]+(flu.h-con.hm)*con.bnv[i]/2.)\n flu.uv[i] = con.bv[i]+(flu.h-con.hm)*(1.+con.bnv[i]**2)**.5\n else:\n flu.av[i] = (der.hv[i]*(con.bv[i]+der.hv[i]*con.bnv[i]/2.) +\n ((flu.h-(con.hm+der.hv[i])) *\n (con.bv[i]+der.hv[i]*con.bnv[i])))\n flu.uv[i] = ((con.bv[i])+(der.hv[i]*(1.+con.bnv[i]**2)**.5) +\n (flu.h-(con.hm+der.hv[i])))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the discharge of both forelands after Manning - Strickler.", "response": "def calc_qv_v1(self):\n \"\"\"Calculate the discharge of both forelands after Manning-Strickler.\n\n Required control parameters:\n |EKV|\n |SKV|\n |Gef|\n\n Required flux sequence:\n |AV|\n |UV|\n\n Calculated flux sequence:\n |lstream_fluxes.QV|\n\n Examples:\n\n For appropriate strictly positive values:\n\n >>> from hydpy.models.lstream import *\n >>> parameterstep()\n >>> ekv(2.0)\n >>> skv(50.0)\n >>> gef(0.01)\n >>> fluxes.av = 3.0\n >>> fluxes.uv = 7.0\n >>> model.calc_qv_v1()\n >>> fluxes.qv\n qv(17.053102, 17.053102)\n\n For zero or negative values of the flown through surface or\n the wetted perimeter:\n\n >>> fluxes.av = -1.0, 3.0\n >>> fluxes.uv = 7.0, 0.0\n >>> model.calc_qv_v1()\n >>> fluxes.qv\n qv(0.0, 0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for i in range(2):\n if (flu.av[i] > 0.) and (flu.uv[i] > 0.):\n flu.qv[i] = (con.ekv[i]*con.skv[i] *\n flu.av[i]**(5./3.)/flu.uv[i]**(2./3.)*con.gef**.5)\n else:\n flu.qv[i] = 0."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_avr_uvr_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for i in range(2):\n if flu.h <= (con.hm+der.hv[i]):\n flu.avr[i] = 0.\n flu.uvr[i] = 0.\n else:\n flu.avr[i] = (flu.h-(con.hm+der.hv[i]))**2*con.bnvr[i]/2.\n flu.uvr[i] = (flu.h-(con.hm+der.hv[i]))*(1.+con.bnvr[i]**2)**.5", "response": "Calculates the flown through area and the wetted perimeter of the all - water - related embankments."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_qvr_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for i in range(2):\n if (flu.avr[i] > 0.) and (flu.uvr[i] > 0.):\n flu.qvr[i] = (con.ekv[i]*con.skv[i] *\n flu.avr[i]**(5./3.)/flu.uvr[i]**(2./3.)*con.gef**.5)\n else:\n flu.qvr[i] = 0.", "response": "Calculate the discharge of both outer embankments after\n Manning - Strickler."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the total cross section area of the total cross section.", "response": "def calc_ag_v1(self):\n \"\"\"Sum the through flown area of the total cross section.\n\n Required flux sequences:\n |AM|\n |AV|\n |AVR|\n\n Calculated flux sequence:\n |AG|\n\n Example:\n\n >>> from hydpy.models.lstream import *\n >>> parameterstep()\n >>> fluxes.am = 1.0\n >>> fluxes.av= 2.0, 3.0\n >>> fluxes.avr = 4.0, 5.0\n >>> model.calc_ag_v1()\n >>> fluxes.ag\n ag(15.0)\n \"\"\"\n flu = self.sequences.fluxes.fastaccess\n flu.ag = flu.am+flu.av[0]+flu.av[1]+flu.avr[0]+flu.avr[1]"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_qg_v1(self):\n flu = self.sequences.fluxes.fastaccess\n self.calc_am_um()\n self.calc_qm()\n self.calc_av_uv()\n self.calc_qv()\n self.calc_avr_uvr()\n self.calc_qvr()\n flu.qg = flu.qm+flu.qv[0]+flu.qv[1]+flu.qvr[0]+flu.qvr[1]", "response": "Calculate the discharge of the total cross section."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ndetermining an starting interval for iteration methods as the one implemented in method |calc_h_v1|.", "response": "def calc_hmin_qmin_hmax_qmax_v1(self):\n \"\"\"Determine an starting interval for iteration methods as the one\n implemented in method |calc_h_v1|.\n\n The resulting interval is determined in a manner, that on the\n one hand :math:`Qmin \\\\leq QRef \\\\leq Qmax` is fulfilled and on the\n other hand the results of method |calc_qg_v1| are continuous\n for :math:`Hmin \\\\leq H \\\\leq Hmax`.\n\n Required control parameter:\n |HM|\n\n Required derived parameters:\n |HV|\n |lstream_derived.QM|\n |lstream_derived.QV|\n\n Required flux sequence:\n |QRef|\n\n Calculated aide sequences:\n |HMin|\n |HMax|\n |QMin|\n |QMax|\n\n Besides the mentioned required parameters and sequences, those of the\n actual method for calculating the discharge of the total cross section\n might be required. This is the case whenever water flows on both outer\n embankments. In such occasions no previously determined upper boundary\n values are available and method |calc_hmin_qmin_hmax_qmax_v1| needs\n to increase the value of :math:`HMax` successively until the condition\n :math:`QG \\\\leq QMax` is met.\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n if flu.qref <= der.qm:\n aid.hmin = 0.\n aid.qmin = 0.\n aid.hmax = con.hm\n aid.qmax = der.qm\n elif flu.qref <= min(der.qv[0], der.qv[1]):\n aid.hmin = con.hm\n aid.qmin = der.qm\n aid.hmax = con.hm+min(der.hv[0], der.hv[1])\n aid.qmax = min(der.qv[0], der.qv[1])\n elif flu.qref < max(der.qv[0], der.qv[1]):\n aid.hmin = con.hm+min(der.hv[0], der.hv[1])\n aid.qmin = min(der.qv[0], der.qv[1])\n aid.hmax = con.hm+max(der.hv[0], der.hv[1])\n aid.qmax = max(der.qv[0], der.qv[1])\n else:\n flu.h = con.hm+max(der.hv[0], der.hv[1])\n aid.hmin = flu.h\n aid.qmin = flu.qg\n while True:\n flu.h *= 2.\n self.calc_qg()\n if flu.qg < flu.qref:\n aid.hmin = flu.h\n aid.qmin = flu.qg\n else:\n aid.hmax = flu.h\n aid.qmax = flu.qg\n break"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the approximate water stage resulting in a certain reference discarge.", "response": "def calc_h_v1(self):\n \"\"\"Approximate the water stage resulting in a certain reference discarge\n with the Pegasus iteration method.\n\n Required control parameters:\n |QTol|\n |HTol|\n\n Required flux sequence:\n |QRef|\n\n Modified aide sequences:\n |HMin|\n |HMax|\n |QMin|\n |QMax|\n\n Calculated flux sequence:\n |H|\n\n Besides the parameters and sequences given above, those of the\n actual method for calculating the discharge of the total cross section\n are required.\n\n Examples:\n\n Essentially, the Pegasus method is a root finding algorithm which\n sequentially decreases its search radius (like the simple bisection\n algorithm) and shows superlinear convergence properties (like the\n Newton-Raphson algorithm). Ideally, its convergence should be proved\n for each application model to be derived from HydPy-L-Stream.\n The following examples focus on the methods\n |calc_hmin_qmin_hmax_qmax_v1| and |calc_qg_v1| (including their\n submethods) only:\n\n >>> from hydpy.models.lstream import *\n >>> parameterstep()\n >>> model.calc_hmin_qmin_hmax_qmax = model.calc_hmin_qmin_hmax_qmax_v1\n >>> model.calc_qg = model.calc_qg_v1\n >>> model.calc_qm = model.calc_qm_v1\n >>> model.calc_av_uv = model.calc_av_uv_v1\n >>> model.calc_qv = model.calc_qv_v1\n >>> model.calc_avr_uvr = model.calc_avr_uvr_v1\n >>> model.calc_qvr = model.calc_qvr_v1\n\n Define the geometry and roughness values for the first test channel:\n\n >>> bm(2.0)\n >>> bnm(4.0)\n >>> hm(1.0)\n >>> bv(0.5, 10.0)\n >>> bbv(1.0, 2.0)\n >>> bnv(1.0, 8.0)\n >>> bnvr(20.0)\n >>> ekm(1.0)\n >>> skm(20.0)\n >>> ekv(1.0)\n >>> skv(60.0, 80.0)\n >>> gef(0.01)\n\n Set the error tolerances of the iteration small enough to not\n compromise the shown first six decimal places of the following\n results:\n\n >>> qtol(1e-10)\n >>> htol(1e-10)\n\n Derive the required secondary parameters:\n\n >>> derived.hv.update()\n >>> derived.qm.update()\n >>> derived.qv.update()\n\n Define a test function, accepting a reference discharge and printing\n both the approximated water stage and the related discharge value:\n\n >>> def test(qref):\n ... fluxes.qref = qref\n ... model.calc_hmin_qmin_hmax_qmax()\n ... model.calc_h()\n ... print(repr(fluxes.h))\n ... print(repr(fluxes.qg))\n\n Zero discharge and the following discharge values are related to the\n only discontinuities of the given root finding problem:\n\n >>> derived.qm\n qm(8.399238)\n >>> derived.qv\n qv(left=154.463234, right=23.073584)\n\n The related water stages are the ones (directly or indirectly)\n defined above:\n\n >>> test(0.0)\n h(0.0)\n qg(0.0)\n >>> test(derived.qm)\n h(1.0)\n qg(8.399238)\n >>> test(derived.qv.left)\n h(2.0)\n qg(154.463234)\n >>> test(derived.qv.right)\n h(1.25)\n qg(23.073584)\n\n Test some intermediate water stages, inundating the only the main\n channel, the main channel along with the right foreland, and the\n main channel along with both forelands respectively:\n\n >>> test(6.0)\n h(0.859452)\n qg(6.0)\n >>> test(10.0)\n h(1.047546)\n qg(10.0)\n >>> test(100.0)\n h(1.77455)\n qg(100.0)\n\n Finally, test two extreme water stages, inundating both outer\n foreland embankments:\n\n >>> test(200.0)\n h(2.152893)\n qg(200.0)\n >>> test(2000.0)\n h(4.240063)\n qg(2000.0)\n\n There is a potential risk of the implemented iteration method to fail\n for special channel geometries. To test such cases in a more\n condensed manner, the following test methods evaluates different water\n stages automatically in accordance with the example above. An error\n message is printed only, the estimated discharge does not approximate\n the reference discharge with six decimal places:\n\n >>> def test():\n ... derived.hv.update()\n ... derived.qm.update()\n ... derived.qv.update()\n ... qm, qv = derived.qm, derived.qv\n ... for qref in [0.0, qm, qv.left, qv.right,\n ... 2.0/3.0*qm+1.0/3.0*min(qv),\n ... 2.0/3.0*min(qv)+1.0/3.0*max(qv),\n ... 3.0*max(qv), 30.0*max(qv)]:\n ... fluxes.qref = qref\n ... model.calc_hmin_qmin_hmax_qmax()\n ... model.calc_h()\n ... if abs(round(fluxes.qg-qref) > 0.0):\n ... print('Error!', 'qref:', qref, 'qg:', fluxes.qg)\n\n Check for a triangle main channel:\n\n >>> bm(0.0)\n >>> test()\n >>> bm(2.0)\n\n Check for a completely flat main channel:\n\n >>> hm(0.0)\n >>> test()\n\n Repeat the last example but with a decreased value of |QTol|\n allowing to trigger another stopping mechanisms if the\n iteration algorithm:\n\n >>> qtol(0.0)\n >>> test()\n >>> hm(1.0)\n >>> qtol(1e-10)\n\n Check for a nonexistend main channel:\n\n >>> bm(0.0)\n >>> bnm(0.0)\n >>> test()\n >>> bm(2.0)\n >>> bnm(4.0)\n\n Check for a nonexistend forelands:\n\n >>> bv(0.0)\n >>> bbv(0.0)\n >>> test()\n >>> bv(0.5, 10.0)\n >>> bbv(1., 2.0)\n\n Check for nonexistend outer foreland embankments:\n\n >>> bnvr(0.0)\n >>> test()\n\n To take the last test as an illustrative example, one can see that\n the given reference discharge is met by the estimated total discharge,\n which consists of components related to the main channel and the\n forelands only:\n\n >>> fluxes.qref\n qref(3932.452785)\n >>> fluxes.qg\n qg(3932.452785)\n >>> fluxes.qm\n qm(530.074621)\n >>> fluxes.qv\n qv(113.780226, 3288.597937)\n >>> fluxes.qvr\n qvr(0.0, 0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n aid.qmin -= flu.qref\n aid.qmax -= flu.qref\n if modelutils.fabs(aid.qmin) < con.qtol:\n flu.h = aid.hmin\n self.calc_qg()\n elif modelutils.fabs(aid.qmax) < con.qtol:\n flu.h = aid.hmax\n self.calc_qg()\n elif modelutils.fabs(aid.hmax-aid.hmin) < con.htol:\n flu.h = (aid.hmin+aid.hmax)/2.\n self.calc_qg()\n else:\n while True:\n flu.h = aid.hmin-aid.qmin*(aid.hmax-aid.hmin)/(aid.qmax-aid.qmin)\n self.calc_qg()\n aid.qtest = flu.qg-flu.qref\n if modelutils.fabs(aid.qtest) < con.qtol:\n return\n if (((aid.qmax < 0.) and (aid.qtest < 0.)) or\n ((aid.qmax > 0.) and (aid.qtest > 0.))):\n aid.qmin *= aid.qmax/(aid.qmax+aid.qtest)\n else:\n aid.hmin = aid.hmax\n aid.qmin = aid.qmax\n aid.hmax = flu.h\n aid.qmax = aid.qtest\n if modelutils.fabs(aid.hmax-aid.hmin) < con.htol:\n return"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating the outflow of the linear storage.", "response": "def calc_qa_v1(self):\n \"\"\"Calculate outflow.\n\n The working equation is the analytical solution of the linear storage\n equation under the assumption of constant change in inflow during\n the simulation time step.\n\n Required flux sequence:\n |RK|\n\n Required state sequence:\n |QZ|\n\n Updated state sequence:\n |QA|\n\n Basic equation:\n :math:`QA_{neu} = QA_{alt} +\n (QZ_{alt}-QA_{alt}) \\\\cdot (1-exp(-RK^{-1})) +\n (QZ_{neu}-QZ_{alt}) \\\\cdot (1-RK\\\\cdot(1-exp(-RK^{-1})))`\n\n Examples:\n\n A normal test case:\n\n >>> from hydpy.models.lstream import *\n >>> parameterstep()\n >>> fluxes.rk(0.1)\n >>> states.qz.old = 2.0\n >>> states.qz.new = 4.0\n >>> states.qa.old = 3.0\n >>> model.calc_qa_v1()\n >>> states.qa\n qa(3.800054)\n\n First extreme test case (zero division is circumvented):\n\n >>> fluxes.rk(0.0)\n >>> model.calc_qa_v1()\n >>> states.qa\n qa(4.0)\n\n Second extreme test case (numerical overflow is circumvented):\n\n >>> fluxes.rk(1e201)\n >>> model.calc_qa_v1()\n >>> states.qa\n qa(5.0)\n \"\"\"\n flu = self.sequences.fluxes.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n aid = self.sequences.aides.fastaccess\n if flu.rk <= 0.:\n new.qa = new.qz\n elif flu.rk > 1e200:\n new.qa = old.qa+new.qz-old.qz\n else:\n aid.temp = (1.-modelutils.exp(-1./flu.rk))\n new.qa = (old.qa +\n (old.qz-old.qa)*aid.temp +\n (new.qz-old.qz)*(1.-flu.rk*aid.temp))"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_tc_v1(self):\n con = self.parameters.control.fastaccess\n inp = self.sequences.inputs.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nmbzones):\n flu.tc[k] = inp.t-con.tcalt[k]*(con.zonez[k]-con.zrelt)", "response": "Adjust the measured air temperature to the altitude of the individual zones."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the areal mean temperature of the subbasin.", "response": "def calc_tmean_v1(self):\n \"\"\"Calculate the areal mean temperature of the subbasin.\n\n Required derived parameter:\n |RelZoneArea|\n\n Required flux sequence:\n |TC|\n\n Calculated flux sequences:\n |TMean|\n\n Examples:\n\n Prepare two zones, the first one being twice as large\n as the second one:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(2)\n >>> derived.relzonearea(2.0/3.0, 1.0/3.0)\n\n With temperature values of 5\u00b0C and 8\u00b0C of the respective zones,\n the mean temperature is 6\u00b0C:\n\n >>> fluxes.tc = 5.0, 8.0\n >>> model.calc_tmean_v1()\n >>> fluxes.tmean\n tmean(6.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.tmean = 0.\n for k in range(con.nmbzones):\n flu.tmean += der.relzonearea[k]*flu.tc[k]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the temperature - dependent fraction of the rainfall and the total precipitation.", "response": "def calc_fracrain_v1(self):\n \"\"\"Determine the temperature-dependent fraction of (liquid) rainfall\n and (total) precipitation.\n\n Required control parameters:\n |NmbZones|\n |TT|,\n |TTInt|\n\n Required flux sequence:\n |TC|\n\n Calculated flux sequences:\n |FracRain|\n\n Basic equation:\n :math:`FracRain = \\\\frac{TC-(TT-\\\\frac{TTInt}{2})}{TTInt}`\n\n Restriction:\n :math:`0 \\\\leq FracRain \\\\leq 1`\n\n Examples:\n\n The threshold temperature of seven zones is 0\u00b0C and the corresponding\n temperature interval of mixed precipitation 2\u00b0C:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(7)\n >>> tt(0.0)\n >>> ttint(2.0)\n\n The fraction of rainfall is zero below -1\u00b0C, is one above 1\u00b0C and\n increases linearly in between:\n\n >>> fluxes.tc = -10.0, -1.0, -0.5, 0.0, 0.5, 1.0, 10.0\n >>> model.calc_fracrain_v1()\n >>> fluxes.fracrain\n fracrain(0.0, 0.0, 0.25, 0.5, 0.75, 1.0, 1.0)\n\n Note the special case of a zero temperature interval. With a\n actual temperature being equal to the threshold temperature, the\n rainfall fraction is one:\n\n >>> ttint(0.0)\n >>> model.calc_fracrain_v1()\n >>> fluxes.fracrain\n fracrain(0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nmbzones):\n if flu.tc[k] >= (con.tt[k]+con.ttint[k]/2.):\n flu.fracrain[k] = 1.\n elif flu.tc[k] <= (con.tt[k]-con.ttint[k]/2.):\n flu.fracrain[k] = 0.\n else:\n flu.fracrain[k] = ((flu.tc[k]-(con.tt[k]-con.ttint[k]/2.)) /\n con.ttint[k])"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating the corrected fractions rainfall and snowfall precipitation for a single base - level RAInfall and total - base - Frac - Rain.", "response": "def calc_rfc_sfc_v1(self):\n \"\"\"Calculate the corrected fractions rainfall/snowfall and total\n precipitation.\n\n Required control parameters:\n |NmbZones|\n |RfCF|\n |SfCF|\n\n Calculated flux sequences:\n |RfC|\n |SfC|\n\n Basic equations:\n :math:`RfC = RfCF \\\\cdot FracRain` \\n\n :math:`SfC = SfCF \\\\cdot (1 - FracRain)`\n\n Examples:\n\n Assume five zones with different temperatures and hence\n different fractions of rainfall and total precipitation:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(5)\n >>> fluxes.fracrain = 0.0, 0.25, 0.5, 0.75, 1.0\n\n With no rainfall and no snowfall correction (implied by the\n respective factors being one), the corrected fraction related\n to rainfall is identical with the original fraction and the\n corrected fraction related to snowfall behaves opposite:\n\n >>> rfcf(1.0)\n >>> sfcf(1.0)\n >>> model.calc_rfc_sfc_v1()\n >>> fluxes.rfc\n rfc(0.0, 0.25, 0.5, 0.75, 1.0)\n >>> fluxes.sfc\n sfc(1.0, 0.75, 0.5, 0.25, 0.0)\n\n With a negative rainfall correction of 20% and a positive\n snowfall correction of 20 % the corrected fractions are:\n\n >>> rfcf(0.8)\n >>> sfcf(1.2)\n >>> model.calc_rfc_sfc_v1()\n >>> fluxes.rfc\n rfc(0.0, 0.2, 0.4, 0.6, 0.8)\n >>> fluxes.sfc\n sfc(1.2, 0.9, 0.6, 0.3, 0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nmbzones):\n flu.rfc[k] = flu.fracrain[k]*con.rfcf[k]\n flu.sfc[k] = (1.-flu.fracrain[k])*con.sfcf[k]"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the precipitation correction factors and adjusts precipitation to the altitude of the individual zones.", "response": "def calc_pc_v1(self):\n \"\"\"Apply the precipitation correction factors and adjust precipitation\n to the altitude of the individual zones.\n\n Required control parameters:\n |NmbZones|\n |PCorr|\n |PCAlt|\n |ZoneZ|\n |ZRelP|\n\n Required input sequence:\n |P|\n\n Required flux sequences:\n |RfC|\n |SfC|\n\n Calculated flux sequences:\n |PC|\n\n Basic equation:\n :math:`PC = P \\\\cdot PCorr\n \\\\cdot (1+PCAlt \\\\cdot (ZoneZ-ZRelP))\n \\\\cdot (RfC + SfC)`\n\n Examples:\n\n Five zones are at an elevation of 200 m. A precipitation value\n of 5 mm has been measured at a gauge at an elevation of 300 m:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(5)\n >>> zrelp(2.0)\n >>> zonez(3.0)\n >>> inputs.p = 5.0\n\n The first four zones illustrate the individual precipitation\n corrections due to the general precipitation correction factor\n (|PCorr|, first zone), the altitude correction factor (|PCAlt|,\n second zone), the rainfall related correction (|RfC|, third zone),\n and the snowfall related correction factor (|SfC|, fourth zone).\n The fifth zone illustrates the interaction between all corrections:\n\n >>> pcorr(1.3, 1.0, 1.0, 1.0, 1.3)\n >>> pcalt(0.0, 0.1, 0.0, 0.0, 0.1)\n >>> fluxes.rfc = 0.5, 0.5, 0.4, 0.5, 0.4\n >>> fluxes.sfc = 0.5, 0.5, 0.5, 0.7, 0.7\n >>> model.calc_pc_v1()\n >>> fluxes.pc\n pc(6.5, 5.5, 4.5, 6.0, 7.865)\n\n Usually, one would set zero or positive values for parameter |PCAlt|.\n But it is also allowed to set negative values, in order to reflect\n possible negative relationships between precipitation and altitude.\n To prevent from calculating negative precipitation when too large\n negative values are applied, a truncation is performed:\n\n >>> pcalt(-1.0)\n >>> model.calc_pc_v1()\n >>> fluxes.pc\n pc(0.0, 0.0, 0.0, 0.0, 0.0)\n\n \"\"\"\n con = self.parameters.control.fastaccess\n inp = self.sequences.inputs.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nmbzones):\n flu.pc[k] = inp.p*(1.+con.pcalt[k]*(con.zonez[k]-con.zrelp))\n if flu.pc[k] <= 0.:\n flu.pc[k] = 0.\n else:\n flu.pc[k] *= con.pcorr[k]*(flu.rfc[k]+flu.sfc[k])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the potential evaporation to the actual temperature.", "response": "def calc_ep_v1(self):\n \"\"\"Adjust potential norm evaporation to the actual temperature.\n\n Required control parameters:\n |NmbZones|\n |ETF|\n\n Required input sequence:\n |EPN|\n |TN|\n\n Required flux sequence:\n |TMean|\n\n Calculated flux sequences:\n |EP|\n\n Basic equation:\n :math:`EP = EPN \\\\cdot (1 + ETF \\\\cdot (TMean - TN))`\n\n Restriction:\n :math:`0 \\\\leq EP \\\\leq 2 \\\\cdot EPN`\n\n Examples:\n\n Assume four zones with different values of the temperature\n related factor for the adjustment of evaporation (the\n negative value of the first zone is not meaningful, but used\n for illustration purporses):\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(4)\n >>> etf(-0.5, 0.0, 0.1, 0.5)\n >>> inputs.tn = 20.0\n >>> inputs.epn = 2.0\n\n With mean temperature equal to norm temperature, actual\n (uncorrected) evaporation is equal to norm evaporation:\n\n >>> fluxes.tmean = 20.0\n >>> model.calc_ep_v1()\n >>> fluxes.ep\n ep(2.0, 2.0, 2.0, 2.0)\n\n With mean temperature 5\u00b0C higher than norm temperature, potential\n evaporation is increased by 1 mm for the third zone, which\n possesses a very common adjustment factor. For the first zone,\n potential evaporation is 0 mm (which is the smallest value\n allowed), and for the fourth zone it is the double value of the\n norm evaporation (which is the largest value allowed):\n\n >>> fluxes.tmean = 25.0\n >>> model.calc_ep_v1()\n >>> fluxes.ep\n ep(0.0, 2.0, 3.0, 4.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n inp = self.sequences.inputs.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nmbzones):\n flu.ep[k] = inp.epn*(1.+con.etf[k]*(flu.tmean-inp.tn))\n flu.ep[k] = min(max(flu.ep[k], 0.), 2.*inp.epn)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_epc_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n for k in range(con.nmbzones):\n flu.epc[k] = (flu.ep[k]*con.ecorr[k] *\n (1. - con.ecalt[k]*(con.zonez[k]-con.zrele)))\n if flu.epc[k] <= 0.:\n flu.epc[k] = 0.\n else:\n flu.epc[k] *= modelutils.exp(-con.epf[k]*flu.pc[k])", "response": "Calculates the evaporation correction factors and adjust evaporation\n to the altitude of the individual zones and perform some corrections between those two evaporation terms."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef calc_tf_ic_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] in (FIELD, FOREST):\n flu.tf[k] = max(flu.pc[k]-(con.icmax[k]-sta.ic[k]), 0.)\n sta.ic[k] += flu.pc[k]-flu.tf[k]\n else:\n flu.tf[k] = flu.pc[k]\n sta.ic[k] = 0.", "response": "Calculates throughfall and update the interception storage with the current state sequence."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef calc_ei_ic_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] in (FIELD, FOREST):\n flu.ei[k] = min(flu.epc[k], sta.ic[k])\n sta.ic[k] -= flu.ei[k]\n else:\n flu.ei[k] = 0.\n sta.ic[k] = 0.", "response": "Calculates interception evaporation and update the interception term storage accordingly."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calc_sp_wc_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] != ILAKE:\n if (flu.rfc[k]+flu.sfc[k]) > 0.:\n sta.wc[k] += flu.tf[k]*flu.rfc[k]/(flu.rfc[k]+flu.sfc[k])\n sta.sp[k] += flu.tf[k]*flu.sfc[k]/(flu.rfc[k]+flu.sfc[k])\n else:\n sta.wc[k] = 0.\n sta.sp[k] = 0.", "response": "Calculates the throughfall of the snow layer."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_refr_sp_wc_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] != ILAKE:\n if flu.tc[k] < der.ttm[k]:\n flu.refr[k] = min(con.cfr[k]*con.cfmax[k] *\n (der.ttm[k]-flu.tc[k]), sta.wc[k])\n sta.sp[k] += flu.refr[k]\n sta.wc[k] -= flu.refr[k]\n else:\n flu.refr[k] = 0.\n\n else:\n flu.refr[k] = 0.\n sta.wc[k] = 0.\n sta.sp[k] = 0.", "response": "Calculates the refreezing of the water content within the snow layer and update both the snow layer and the water content within the snow layer and the water content within the snow layer."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef calc_in_wc_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] != ILAKE:\n flu.in_[k] = max(sta.wc[k]-con.whc[k]*sta.sp[k], 0.)\n sta.wc[k] -= flu.in_[k]\n else:\n flu.in_[k] = flu.tf[k]\n sta.wc[k] = 0.", "response": "Calculates the actual water release from the snow layer."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calc_glmelt_in_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if ((con.zonetype[k] == GLACIER) and\n (sta.sp[k] <= 0.) and (flu.tc[k] > der.ttm[k])):\n flu.glmelt[k] = con.gmelt[k]*(flu.tc[k]-der.ttm[k])\n flu.in_[k] += flu.glmelt[k]\n else:\n flu.glmelt[k] = 0.", "response": "Calculates the melting from glaciers which are actually not covered by a snow layer and adds it to the water release of the snow module."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating the effective precipitation and update soil moisture.", "response": "def calc_r_sm_v1(self):\n \"\"\"Calculate effective precipitation and update soil moisture.\n\n Required control parameters:\n |NmbZones|\n |ZoneType|\n |FC|\n |Beta|\n\n Required fluxes sequence:\n |In_|\n\n Calculated flux sequence:\n |R|\n\n Updated state sequence:\n |SM|\n\n Basic equations:\n :math:`\\\\frac{dSM}{dt} = IN - R` \\n\n :math:`R = IN \\\\cdot \\\\left(\\\\frac{SM}{FC}\\\\right)^{Beta}`\n\n\n Examples:\n\n Initialize six zones of different types. The field\n capacity of all fields and forests is set to 200mm, the input\n of each zone is 10mm:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(6)\n >>> zonetype(ILAKE, GLACIER, FIELD, FOREST, FIELD, FIELD)\n >>> fc(200.0)\n >>> fluxes.in_ = 10.0\n\n With a common nonlinearity parameter value of 2, a relative\n soil moisture of 50% (zones three and four) results in a\n discharge coefficient of 25%. For a soil completely dried\n (zone five) or completely saturated (one six) the discharge\n coefficient does not depend on the nonlinearity parameter and\n is 0% and 100% respectively. Glaciers and internal lakes also\n always route 100% of their input as effective precipitation:\n\n >>> beta(2.0)\n >>> states.sm = 0.0, 0.0, 100.0, 100.0, 0.0, 200.0\n >>> model.calc_r_sm_v1()\n >>> fluxes.r\n r(10.0, 10.0, 2.5, 2.5, 0.0, 10.0)\n >>> states.sm\n sm(0.0, 0.0, 107.5, 107.5, 10.0, 200.0)\n\n Through decreasing the nonlinearity parameter, the discharge\n coefficient increases. A parameter value of zero leads to a\n discharge coefficient of 100% for any soil moisture:\n\n >>> beta(0.0)\n >>> states.sm = 0.0, 0.0, 100.0, 100.0, 0.0, 200.0\n >>> model.calc_r_sm_v1()\n >>> fluxes.r\n r(10.0, 10.0, 10.0, 10.0, 10.0, 10.0)\n >>> states.sm\n sm(0.0, 0.0, 100.0, 100.0, 0.0, 200.0)\n\n With zero field capacity, the discharge coefficient also always\n equates to 100%:\n\n >>> fc(0.0)\n >>> beta(2.0)\n >>> states.sm = 0.0\n >>> model.calc_r_sm_v1()\n >>> fluxes.r\n r(10.0, 10.0, 10.0, 10.0, 10.0, 10.0)\n >>> states.sm\n sm(0.0, 0.0, 0.0, 0.0, 0.0, 0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] in (FIELD, FOREST):\n if con.fc[k] > 0.:\n flu.r[k] = flu.in_[k]*(sta.sm[k]/con.fc[k])**con.beta[k]\n flu.r[k] = max(flu.r[k], sta.sm[k]+flu.in_[k]-con.fc[k])\n else:\n flu.r[k] = flu.in_[k]\n sta.sm[k] += flu.in_[k]-flu.r[k]\n else:\n flu.r[k] = flu.in_[k]\n sta.sm[k] = 0."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_cf_sm_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] in (FIELD, FOREST):\n if con.fc[k] > 0.:\n flu.cf[k] = con.cflux[k]*(1.-sta.sm[k]/con.fc[k])\n flu.cf[k] = min(flu.cf[k], sta.uz+flu.r[k])\n flu.cf[k] = min(flu.cf[k], con.fc[k]-sta.sm[k])\n else:\n flu.cf[k] = 0.\n sta.sm[k] += flu.cf[k]\n else:\n flu.cf[k] = 0.\n sta.sm[k] = 0.", "response": "Calculates the capillary flow and update soil moisture for a single field and forest."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating soil evaporation and update soil moisture.", "response": "def calc_ea_sm_v1(self):\n \"\"\"Calculate soil evaporation and update soil moisture.\n\n Required control parameters:\n |NmbZones|\n |ZoneType|\n |FC|\n |LP|\n |ERed|\n\n Required fluxes sequences:\n |EPC|\n |EI|\n\n Required state sequence:\n |SP|\n\n Calculated flux sequence:\n |EA|\n\n Updated state sequence:\n |SM|\n\n Basic equations:\n :math:`\\\\frac{dSM}{dt} = - EA` \\n\n :math:`EA_{temp} = \\\\biggl \\\\lbrace\n {\n {EPC \\\\cdot min\\\\left(\\\\frac{SM}{LP \\\\cdot FC}, 1\\\\right)\n \\\\ | \\\\ SP = 0}\n \\\\atop\n {0 \\\\ | \\\\ SP > 0}\n }` \\n\n :math:`EA = EA_{temp} - max(ERED \\\\cdot (EA_{temp} + EI - EPC), 0)`\n\n Examples:\n\n Initialize seven zones of different types. The field capacity\n of all fields and forests is set to 200mm, potential evaporation\n and interception evaporation are 2mm and 1mm respectively:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(7)\n >>> zonetype(ILAKE, GLACIER, FIELD, FOREST, FIELD, FIELD, FIELD)\n >>> fc(200.0)\n >>> lp(0.0, 0.0, 0.5, 0.5, 0.0, 0.8, 1.0)\n >>> ered(0.0)\n >>> fluxes.epc = 2.0\n >>> fluxes.ei = 1.0\n >>> states.sp = 0.0\n\n Only fields and forests include soils; for glaciers and zones (the\n first two zones) no soil evaporation is performed. For fields and\n forests, the underlying calculations are the same. In the following\n example, the relative soil moisture is 50% in all field and forest\n zones. Hence, differences in soil evaporation are related to the\n different soil evaporation parameter values only:\n\n >>> states.sm = 100.0\n >>> model.calc_ea_sm_v1()\n >>> fluxes.ea\n ea(0.0, 0.0, 2.0, 2.0, 2.0, 1.25, 1.0)\n >>> states.sm\n sm(0.0, 0.0, 98.0, 98.0, 98.0, 98.75, 99.0)\n\n In the last example, evaporation values of 2mm have been calculated\n for some zones despite the fact, that these 2mm added to the actual\n interception evaporation of 1mm exceed potential evaporation. This\n behaviour can be reduced...\n\n >>> states.sm = 100.0\n >>> ered(0.5)\n >>> model.calc_ea_sm_v1()\n >>> fluxes.ea\n ea(0.0, 0.0, 1.5, 1.5, 1.5, 1.125, 1.0)\n >>> states.sm\n sm(0.0, 0.0, 98.5, 98.5, 98.5, 98.875, 99.0)\n\n ...or be completely excluded:\n\n >>> states.sm = 100.0\n >>> ered(1.0)\n >>> model.calc_ea_sm_v1()\n >>> fluxes.ea\n ea(0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0)\n >>> states.sm\n sm(0.0, 0.0, 99.0, 99.0, 99.0, 99.0, 99.0)\n\n Any occurrence of a snow layer suppresses soil evaporation\n completely:\n\n >>> states.sp = 0.01\n >>> states.sm = 100.0\n >>> model.calc_ea_sm_v1()\n >>> fluxes.ea\n ea(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)\n >>> states.sm\n sm(0.0, 0.0, 100.0, 100.0, 100.0, 100.0, 100.0)\n\n For (unrealistic) soils with zero field capacity, soil evaporation\n is always zero:\n\n >>> fc(0.0)\n >>> states.sm = 0.0\n >>> model.calc_ea_sm_v1()\n >>> fluxes.ea\n ea(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)\n >>> states.sm\n sm(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if con.zonetype[k] in (FIELD, FOREST):\n if sta.sp[k] <= 0.:\n if (con.lp[k]*con.fc[k]) > 0.:\n flu.ea[k] = flu.epc[k]*sta.sm[k]/(con.lp[k]*con.fc[k])\n flu.ea[k] = min(flu.ea[k], flu.epc[k])\n else:\n flu.ea[k] = flu.epc[k]\n flu.ea[k] -= max(con.ered[k] *\n (flu.ea[k]+flu.ei[k]-flu.epc[k]), 0.)\n flu.ea[k] = min(flu.ea[k], sta.sm[k])\n else:\n flu.ea[k] = 0.\n sta.sm[k] -= flu.ea[k]\n else:\n flu.ea[k] = 0.\n sta.sm[k] = 0."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\naccumulate the total inflow into the upper zone layer.", "response": "def calc_inuz_v1(self):\n \"\"\"Accumulate the total inflow into the upper zone layer.\n\n Required control parameters:\n |NmbZones|\n |ZoneType|\n\n Required derived parameters:\n |RelLandZoneArea|\n\n Required fluxes sequences:\n |R|\n |CF|\n\n Calculated flux sequence:\n |InUZ|\n\n Basic equation:\n :math:`InUZ = R - CF`\n\n Examples:\n\n Initialize three zones of different relative `land sizes`\n (area related to the total size of the subbasin except lake areas):\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(3)\n >>> zonetype(FIELD, ILAKE, GLACIER)\n >>> derived.rellandzonearea = 2.0/3.0, 0.0, 1.0/3.0\n >>> fluxes.r = 6.0, 0.0, 2.0\n >>> fluxes.cf = 2.0, 0.0, 1.0\n >>> model.calc_inuz_v1()\n >>> fluxes.inuz\n inuz(3.0)\n\n Internal lakes do not contribute to the upper zone layer. Hence\n for a subbasin consisting only of interal lakes a zero input\n value would be calculated:\n\n >>> zonetype(ILAKE, ILAKE, ILAKE)\n >>> model.calc_inuz_v1()\n >>> fluxes.inuz\n inuz(0.0)\n\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.inuz = 0.\n for k in range(con.nmbzones):\n if con.zonetype[k] != ILAKE:\n flu.inuz += der.rellandzonearea[k]*(flu.r[k]-flu.cf[k])"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_contriarea_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n if con.resparea and (der.relsoilarea > 0.):\n flu.contriarea = 0.\n for k in range(con.nmbzones):\n if con.zonetype[k] in (FIELD, FOREST):\n if con.fc[k] > 0.:\n flu.contriarea += (der.relsoilzonearea[k] *\n (sta.sm[k]/con.fc[k])**con.beta[k])\n else:\n flu.contriarea += der.relsoilzonearea[k]\n else:\n flu.contriarea = 1.", "response": "Calculates the relative size of the contributing area of the whole subbasin."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef calc_q0_perc_uz_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n flu.perc = 0.\n flu.q0 = 0.\n for dummy in range(con.recstep):\n # First state update related to the upper zone input.\n sta.uz += der.dt*flu.inuz\n # Second state update related to percolation.\n d_perc = min(der.dt*con.percmax*flu.contriarea, sta.uz)\n sta.uz -= d_perc\n flu.perc += d_perc\n # Third state update related to fast runoff response.\n if sta.uz > 0.:\n if flu.contriarea > 0.:\n d_q0 = (der.dt*con.k *\n (sta.uz/flu.contriarea)**(1.+con.alpha))\n d_q0 = min(d_q0, sta.uz)\n else:\n d_q0 = sta.uz\n sta.uz -= d_q0\n flu.q0 += d_q0\n else:\n d_q0 = 0.", "response": "This method calculates the fast response of the percolation of the current state of the current state of the current state of the current state of the current state of the current state of the current state of the current state of the current state of the current state of the current state of the current state of the current state."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the lower zone layer in accordance with percolation from the upper groundwater to lower groundwater and in accordance with percolation from the lower groundwater and in accordance with percolation from the lower groundwater and in accordance with lake precipitation.", "response": "def calc_lz_v1(self):\n \"\"\"Update the lower zone layer in accordance with percolation from\n upper groundwater to lower groundwater and/or in accordance with\n lake precipitation.\n\n Required control parameters:\n |NmbZones|\n |ZoneType|\n\n Required derived parameters:\n |RelLandArea|\n |RelZoneArea|\n\n Required fluxes sequences:\n |PC|\n |Perc|\n\n Updated state sequence:\n |LZ|\n\n Basic equation:\n :math:`\\\\frac{dLZ}{dt} = Perc + Pc`\n\n Examples:\n\n At first, a subbasin with two field zones is assumed (the zones\n could be of type forest or glacier as well). In such zones,\n precipitation does not fall directly into the lower zone layer,\n hence the given precipitation of 2mm has no impact. Only\n the actual percolation from the upper zone layer (underneath\n both field zones) is added to the lower zone storage:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(2)\n >>> zonetype(FIELD, FIELD)\n >>> derived.rellandarea = 1.0\n >>> derived.relzonearea = 2.0/3.0, 1.0/3.0\n >>> fluxes.perc = 2.0\n >>> fluxes.pc = 5.0\n >>> states.lz = 10.0\n >>> model.calc_lz_v1()\n >>> states.lz\n lz(12.0)\n\n If the second zone is an internal lake, its precipitation falls\n on the lower zone layer directly. Note that only 5/3mm\n precipitation are added, due to the relative size of the\n internal lake within the subbasin. Percolation from the upper\n zone layer increases the lower zone storage only by two thirds\n of its original value, due to the larger spatial extend of\n the lower zone layer:\n\n >>> zonetype(FIELD, ILAKE)\n >>> derived.rellandarea = 2.0/3.0\n >>> derived.relzonearea = 2.0/3.0, 1.0/3.0\n >>> states.lz = 10.0\n >>> model.calc_lz_v1()\n >>> states.lz\n lz(13.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n sta.lz += der.rellandarea*flu.perc\n for k in range(con.nmbzones):\n if con.zonetype[k] == ILAKE:\n sta.lz += der.relzonearea[k]*flu.pc[k]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the internal lake evaporation for a single entry in the current state.", "response": "def calc_el_lz_v1(self):\n \"\"\"Calculate lake evaporation.\n\n Required control parameters:\n |NmbZones|\n |ZoneType|\n |TTIce|\n\n Required derived parameters:\n |RelZoneArea|\n\n Required fluxes sequences:\n |TC|\n |EPC|\n\n Updated state sequence:\n |LZ|\n\n Basic equations:\n :math:`\\\\frac{dLZ}{dt} = -EL` \\n\n :math:`EL = \\\\Bigl \\\\lbrace\n {\n {EPC \\\\ | \\\\ TC > TTIce}\n \\\\atop\n {0 \\\\ | \\\\ TC \\\\leq TTIce}\n }`\n\n Examples:\n\n Six zones of the same size are initialized. The first three\n zones are no internal lakes, they can not exhibit any lake\n evaporation. Of the last three zones, which are internal lakes,\n only the last one evaporates water. For zones five and six,\n evaporation is suppressed due to an assumed ice layer, whenever\n the associated theshold temperature is not exceeded:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(6)\n >>> zonetype(FIELD, FOREST, GLACIER, ILAKE, ILAKE, ILAKE)\n >>> ttice(-1.0)\n >>> derived.relzonearea = 1.0/6.0\n >>> fluxes.epc = 0.6\n >>> fluxes.tc = 0.0, 0.0, 0.0, 0.0, -1.0, -2.0\n >>> states.lz = 10.0\n >>> model.calc_el_lz_v1()\n >>> fluxes.el\n el(0.0, 0.0, 0.0, 0.6, 0.0, 0.0)\n >>> states.lz\n lz(9.9)\n\n Note that internal lakes always contain water. Hence, the\n HydPy-H-Land model allows for negative values of the lower\n zone storage:\n\n >>> states.lz = 0.05\n >>> model.calc_el_lz_v1()\n >>> fluxes.el\n el(0.0, 0.0, 0.0, 0.6, 0.0, 0.0)\n >>> states.lz\n lz(-0.05)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n for k in range(con.nmbzones):\n if (con.zonetype[k] == ILAKE) and (flu.tc[k] > con.ttice[k]):\n flu.el[k] = flu.epc[k]\n sta.lz -= der.relzonearea[k]*flu.el[k]\n else:\n flu.el[k] = 0."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalculates the slow response of the lower zone layer.", "response": "def calc_q1_lz_v1(self):\n \"\"\"Calculate the slow response of the lower zone layer.\n\n Required control parameters:\n |K4|\n |Gamma|\n\n Calculated fluxes sequence:\n |Q1|\n\n Updated state sequence:\n |LZ|\n\n Basic equations:\n :math:`\\\\frac{dLZ}{dt} = -Q1` \\n\n :math:`Q1 = \\\\Bigl \\\\lbrace\n {\n {K4 \\\\cdot LZ^{1+Gamma} \\\\ | \\\\ LZ > 0}\n \\\\atop\n {0 \\\\ | \\\\ LZ\\\\leq 0}\n }`\n\n Examples:\n\n As long as the lower zone storage is negative...\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> k4(0.2)\n >>> gamma(0.0)\n >>> states.lz = -2.0\n >>> model.calc_q1_lz_v1()\n >>> fluxes.q1\n q1(0.0)\n >>> states.lz\n lz(-2.0)\n\n ...or zero, no slow discharge response occurs:\n\n >>> states.lz = 0.0\n >>> model.calc_q1_lz_v1()\n >>> fluxes.q1\n q1(0.0)\n >>> states.lz\n lz(0.0)\n\n For storage values above zero the linear...\n\n >>> states.lz = 2.0\n >>> model.calc_q1_lz_v1()\n >>> fluxes.q1\n q1(0.2)\n >>> states.lz\n lz(1.8)\n\n ...or nonlinear storage routing equation applies:\n\n >>> gamma(1.)\n >>> states.lz = 2.0\n >>> model.calc_q1_lz_v1()\n >>> fluxes.q1\n q1(0.4)\n >>> states.lz\n lz(1.6)\n\n Note that the assumed length of the simulation step is only a\n half day. Hence the effective value of the storage coefficient\n is not 0.2 but 0.1:\n\n >>> k4\n k4(0.2)\n >>> k4.value\n 0.1\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n sta = self.sequences.states.fastaccess\n if sta.lz > 0.:\n flu.q1 = con.k4*sta.lz**(1.+con.gamma)\n else:\n flu.q1 = 0.\n sta.lz -= flu.q1"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef calc_inuh_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.inuh = der.rellandarea*flu.q0+flu.q1", "response": "Calculate the unit hydrograph input."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_outuh_quh_v1(self):\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n flu.outuh = der.uh[0]*flu.inuh+log.quh[0]\n for jdx in range(1, len(der.uh)):\n log.quh[jdx-1] = der.uh[jdx]*flu.inuh+log.quh[jdx]", "response": "Calculates the output of a fast catchment response from the given log sequence and the remaining ordinates of the given log sequence."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the total discharge after possible abstractions.", "response": "def calc_qt_v1(self):\n \"\"\"Calculate the total discharge after possible abstractions.\n\n Required control parameter:\n |Abstr|\n\n Required flux sequence:\n |OutUH|\n\n Calculated flux sequence:\n |QT|\n\n Basic equation:\n :math:`QT = max(OutUH - Abstr, 0)`\n\n Examples:\n\n Trying to abstract less then available, as much as available and\n less then available results in:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> abstr(2.0)\n >>> fluxes.outuh = 2.0\n >>> model.calc_qt_v1()\n >>> fluxes.qt\n qt(1.0)\n >>> fluxes.outuh = 1.0\n >>> model.calc_qt_v1()\n >>> fluxes.qt\n qt(0.0)\n >>> fluxes.outuh = 0.5\n >>> model.calc_qt_v1()\n >>> fluxes.qt\n qt(0.0)\n\n Note that \"negative abstractions\" are allowed:\n\n >>> abstr(-2.0)\n >>> fluxes.outuh = 1.0\n >>> model.calc_qt_v1()\n >>> fluxes.qt\n qt(2.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.qt = max(flu.outuh-con.abstr, 0.)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save(self, parameterstep=None, simulationstep=None):\n par = parametertools.Parameter\n for (modelname, var2aux) in self:\n for filename in var2aux.filenames:\n with par.parameterstep(parameterstep), \\\n par.simulationstep(simulationstep):\n lines = [parametertools.get_controlfileheader(\n modelname, parameterstep, simulationstep)]\n for par in getattr(var2aux, filename):\n lines.append(repr(par) + '\\n')\n hydpy.pub.controlmanager.save_file(filename, ''.join(lines))", "response": "Save all defined auxiliary control files."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nremove the defined variables from the current set of variables.", "response": "def remove(self, *values):\n \"\"\"Remove the defined variables.\n\n The variables to be removed can be selected in two ways. But the\n first example shows that passing nothing or an empty iterable to\n method |Variable2Auxfile.remove| does not remove any variable:\n\n >>> from hydpy import dummies\n >>> v2af = dummies.v2af\n >>> v2af.remove()\n >>> v2af.remove([])\n >>> from hydpy import print_values\n >>> print_values(v2af.filenames)\n file1, file2\n >>> print_values(v2af.variables, width=30)\n eqb(5000.0), eqb(10000.0),\n eqd1(100.0), eqd2(50.0),\n eqi1(2000.0), eqi2(1000.0)\n\n The first option is to pass auxiliary file names:\n\n >>> v2af.remove('file1')\n >>> print_values(v2af.filenames)\n file2\n >>> print_values(v2af.variables)\n eqb(10000.0), eqd1(100.0), eqd2(50.0)\n\n The second option is, to pass variables of the correct type\n and value:\n\n >>> v2af = dummies.v2af\n >>> v2af.remove(v2af.eqb[0])\n >>> print_values(v2af.filenames)\n file1, file2\n >>> print_values(v2af.variables)\n eqb(10000.0), eqd1(100.0), eqd2(50.0), eqi1(2000.0), eqi2(1000.0)\n\n One can pass multiple variables or iterables containing variables\n at once:\n\n >>> v2af = dummies.v2af\n >>> v2af.remove(v2af.eqb, v2af.eqd1, v2af.eqd2)\n >>> print_values(v2af.filenames)\n file1\n >>> print_values(v2af.variables)\n eqi1(2000.0), eqi2(1000.0)\n\n Passing an argument that equals neither a registered file name or a\n registered variable results in the following exception:\n\n >>> v2af.remove('test')\n Traceback (most recent call last):\n ...\n ValueError: While trying to remove the given object `test` of type \\\n`str` from the actual Variable2AuxFile object, the following error occurred: \\\n`'test'` is neither a registered filename nor a registered variable.\n \"\"\"\n for value in objecttools.extract(values, (str, variabletools.Variable)):\n try:\n deleted_something = False\n for fn2var in list(self._type2filename2variable.values()):\n for fn_, var in list(fn2var.items()):\n if value in (fn_, var):\n del fn2var[fn_]\n deleted_something = True\n if not deleted_something:\n raise ValueError(\n f'`{repr(value)}` is neither a registered '\n f'filename nor a registered variable.')\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to remove the given object `{value}` '\n f'of type `{objecttools.classname(value)}` from the '\n f'actual Variable2AuxFile object')"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef filenames(self):\n fns = set()\n for fn2var in self._type2filename2variable.values():\n fns.update(fn2var.keys())\n return sorted(fns)", "response": "A list of all handled auxiliary file names."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the auxiliary file name of the given variable.", "response": "def get_filename(self, variable):\n \"\"\"Return the auxiliary file name the given variable is allocated\n to or |None| if the given variable is not allocated to any\n auxiliary file name.\n\n >>> from hydpy import dummies\n >>> eqb = dummies.v2af.eqb[0]\n >>> dummies.v2af.get_filename(eqb)\n 'file1'\n >>> eqb += 500.0\n >>> dummies.v2af.get_filename(eqb)\n \"\"\"\n fn2var = self._type2filename2variable.get(type(variable), {})\n for (fn_, var) in fn2var.items():\n if var == variable:\n return fn_\n return None"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update(self):\n metapar = self.subpars.pars.control.remotedischargesafety\n self.shape = metapar.shape\n self(tuple(smoothtools.calc_smoothpar_logistic1(mp)\n for mp in metapar.values))", "response": "Calculate the smoothing parameter values."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the smoothing parameter value.", "response": "def update(self):\n \"\"\"Calculate the smoothing parameter value.\n\n The following example is explained in some detail in module\n |smoothtools|:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> waterlevelminimumremotetolerance(0.0)\n >>> derived.waterlevelminimumremotesmoothpar.update()\n >>> from hydpy.cythons.smoothutils import smooth_logistic1\n >>> from hydpy import round_\n >>> round_(smooth_logistic1(0.1,\n ... derived.waterlevelminimumremotesmoothpar))\n 1.0\n >>> waterlevelminimumremotetolerance(2.5)\n >>> derived.waterlevelminimumremotesmoothpar.update()\n >>> round_(smooth_logistic1(2.5,\n ... derived.waterlevelminimumremotesmoothpar))\n 0.99\n \"\"\"\n metapar = self.subpars.pars.control.waterlevelminimumremotetolerance\n self(smoothtools.calc_smoothpar_logistic1(metapar))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalculates the smoothing parameter value for the current locale.", "response": "def update(self):\n \"\"\"Calculate the smoothing parameter value.\n\n The following example is explained in some detail in module\n |smoothtools|:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> highestremotedischarge(1.0)\n >>> highestremotetolerance(0.0)\n >>> derived.highestremotesmoothpar.update()\n >>> from hydpy.cythons.smoothutils import smooth_min1\n >>> from hydpy import round_\n >>> round_(smooth_min1(-4.0, 1.5, derived.highestremotesmoothpar))\n -4.0\n >>> highestremotetolerance(2.5)\n >>> derived.highestremotesmoothpar.update()\n >>> round_(smooth_min1(-4.0, -1.5, derived.highestremotesmoothpar))\n -4.01\n\n Note that the example above corresponds to the example on function\n |calc_smoothpar_min1|, due to the value of parameter\n |HighestRemoteDischarge| being 1 m\u00b3/s. Doubling the value of\n |HighestRemoteDischarge| also doubles the value of\n |HighestRemoteSmoothPar| proportional. This leads to the following\n result:\n\n >>> highestremotedischarge(2.0)\n >>> derived.highestremotesmoothpar.update()\n >>> round_(smooth_min1(-4.0, 1.0, derived.highestremotesmoothpar))\n -4.02\n\n This relationship between |HighestRemoteDischarge| and\n |HighestRemoteSmoothPar| prevents from any smoothing when\n the value of |HighestRemoteDischarge| is zero:\n\n >>> highestremotedischarge(0.0)\n >>> derived.highestremotesmoothpar.update()\n >>> round_(smooth_min1(1.0, 1.0, derived.highestremotesmoothpar))\n 1.0\n\n In addition, |HighestRemoteSmoothPar| is set to zero if\n |HighestRemoteDischarge| is infinity (because no actual value\n will ever come in the vicinit of infinity), which is why no\n value would be changed through smoothing anyway):\n\n >>> highestremotedischarge(inf)\n >>> derived.highestremotesmoothpar.update()\n >>> round_(smooth_min1(1.0, 1.0, derived.highestremotesmoothpar))\n 1.0\n \"\"\"\n control = self.subpars.pars.control\n if numpy.isinf(control.highestremotedischarge):\n self(0.0)\n else:\n self(control.highestremotedischarge *\n smoothtools.calc_smoothpar_min1(control.highestremotetolerance)\n )"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nexecute the given command in a new process.", "response": "def run_subprocess(command: str, verbose: bool = True, blocking: bool = True) \\\n -> Optional[subprocess.Popen]:\n \"\"\"Execute the given command in a new process.\n\n Only when both `verbose` and `blocking` are |True|, |run_subprocess|\n prints all responses to the current value of |sys.stdout|:\n\n >>> from hydpy import run_subprocess\n >>> import platform\n >>> esc = '' if 'windows' in platform.platform().lower() else '\\\\\\\\'\n >>> run_subprocess(f'python -c print{esc}(1+1{esc})')\n 2\n\n With verbose being |False|, |run_subprocess| does never print out\n anything:\n\n >>> run_subprocess(f'python -c print{esc}(1+1{esc})', verbose=False)\n\n >>> process = run_subprocess('python', blocking=False, verbose=False)\n >>> process.kill()\n >>> _ = process.communicate()\n\n When `verbose` is |True| and `blocking` is |False|, |run_subprocess|\n prints all responses to the console (\"invisible\" for doctests):\n\n >>> process = run_subprocess('python', blocking=False)\n >>> process.kill()\n >>> _ = process.communicate()\n \"\"\"\n if blocking:\n result1 = subprocess.run(\n command,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n encoding='utf-8',\n shell=True)\n if verbose: # due to doctest replacing sys.stdout\n for output in (result1.stdout, result1.stderr):\n output = output.strip()\n if output:\n print(output)\n return None\n stdouterr = None if verbose else subprocess.DEVNULL\n result2 = subprocess.Popen(\n command,\n stdout=stdouterr,\n stderr=stdouterr,\n encoding='utf-8',\n shell=True)\n return result2"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nexecutes the given Python commands.", "response": "def exec_commands(commands: str, **parameters: Any) -> None:\n \"\"\"Execute the given Python commands.\n\n Function |exec_commands| is thought for testing purposes only (see\n the main documentation on module |hyd|). Seperate individual commands\n by semicolons and replaced whitespaces with underscores:\n\n >>> from hydpy.exe.commandtools import exec_commands\n >>> import sys\n >>> exec_commands(\"x_=_1+1;print(x)\")\n Start to execute the commands ['x_=_1+1', 'print(x)'] for testing purposes.\n 2\n\n |exec_commands| interprets double underscores as a single underscores:\n\n >>> exec_commands(\"x_=_1;print(x.____class____)\")\n Start to execute the commands ['x_=_1', 'print(x.____class____)'] \\\nfor testing purposes.\n \n\n |exec_commands| evaluates additional keyword arguments before it\n executes the given commands:\n\n >>> exec_commands(\"e=x==y;print(e)\", x=1, y=2)\n Start to execute the commands ['e=x==y', 'print(e)'] for testing purposes.\n False\n \"\"\"\n cmdlist = commands.split(';')\n print(f'Start to execute the commands {cmdlist} for testing purposes.')\n for par, value in parameters.items():\n exec(f'{par} = {value}')\n for command in cmdlist:\n command = command.replace('__', 'temptemptemp')\n command = command.replace('_', ' ')\n command = command.replace('temptemptemp', '_')\n exec(command)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprepares an empty log file eventually and returns its absolute path.", "response": "def prepare_logfile(filename: str) -> str:\n \"\"\"Prepare an empty log file eventually and return its absolute path.\n\n When passing the \"filename\" `stdout`, |prepare_logfile| does not\n prepare any file and just returns `stdout`:\n\n >>> from hydpy.exe.commandtools import prepare_logfile\n >>> prepare_logfile('stdout')\n 'stdout'\n\n When passing the \"filename\" `default`, |prepare_logfile| generates a\n filename containing the actual date and time, prepares an empty file\n on disk, and returns its path:\n\n >>> from hydpy import repr_, TestIO\n >>> from hydpy.core.testtools import mock_datetime_now\n >>> from datetime import datetime\n >>> with TestIO():\n ... with mock_datetime_now(datetime(2000, 1, 1, 12, 30, 0)):\n ... filepath = prepare_logfile('default')\n >>> import os\n >>> os.path.exists(filepath)\n True\n >>> repr_(filepath) # doctest: +ELLIPSIS\n '...hydpy/tests/iotesting/hydpy_2000-01-01_12-30-00.log'\n\n For all other strings, |prepare_logfile| does not add any date or time\n information to the filename:\n\n >>> with TestIO():\n ... with mock_datetime_now(datetime(2000, 1, 1, 12, 30, 0)):\n ... filepath = prepare_logfile('my_log_file.txt')\n >>> os.path.exists(filepath)\n True\n >>> repr_(filepath) # doctest: +ELLIPSIS\n '...hydpy/tests/iotesting/my_log_file.txt'\n \"\"\"\n if filename == 'stdout':\n return filename\n if filename == 'default':\n filename = datetime.datetime.now().strftime(\n 'hydpy_%Y-%m-%d_%H-%M-%S.log')\n with open(filename, 'w'):\n pass\n return os.path.abspath(filename)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexecute a HydPy script function.", "response": "def execute_scriptfunction() -> None:\n \"\"\"Execute a HydPy script function.\n\n Function |execute_scriptfunction| is indirectly applied and\n explained in the documentation on module |hyd|.\n \"\"\"\n try:\n args_given = []\n kwargs_given = {}\n for arg in sys.argv[1:]:\n if len(arg) < 3:\n args_given.append(arg)\n else:\n try:\n key, value = parse_argument(arg)\n kwargs_given[key] = value\n except ValueError:\n args_given.append(arg)\n logfilepath = prepare_logfile(kwargs_given.pop('logfile', 'stdout'))\n logstyle = kwargs_given.pop('logstyle', 'plain')\n try:\n funcname = str(args_given.pop(0))\n except IndexError:\n raise ValueError(\n 'The first positional argument defining the function '\n 'to be called is missing.')\n try:\n func = hydpy.pub.scriptfunctions[funcname]\n except KeyError:\n available_funcs = objecttools.enumeration(\n sorted(hydpy.pub.scriptfunctions.keys()))\n raise ValueError(\n f'There is no `{funcname}` function callable by `hyd.py`. '\n f'Choose one of the following instead: {available_funcs}.')\n args_required = inspect.getfullargspec(func).args\n nmb_args_required = len(args_required)\n nmb_args_given = len(args_given)\n if nmb_args_given != nmb_args_required:\n enum_args_given = ''\n if nmb_args_given:\n enum_args_given = (\n f' ({objecttools.enumeration(args_given)})')\n enum_args_required = ''\n if nmb_args_required:\n enum_args_required = (\n f' ({objecttools.enumeration(args_required)})')\n raise ValueError(\n f'Function `{funcname}` requires `{nmb_args_required:d}` '\n f'positional arguments{enum_args_required}, but '\n f'`{nmb_args_given:d}` are given{enum_args_given}.')\n with _activate_logfile(logfilepath, logstyle, 'info', 'warning'):\n func(*args_given, **kwargs_given)\n except BaseException as exc:\n if logstyle not in LogFileInterface.style2infotype2string:\n logstyle = 'plain'\n with _activate_logfile(logfilepath, logstyle, 'exception', 'exception'):\n arguments = ', '.join(sys.argv)\n print(f'Invoking hyd.py with arguments `{arguments}` '\n f'resulted in the following error:\\n{str(exc)}\\n\\n'\n f'See the following stack traceback for debugging:\\n',\n file=sys.stderr)\n traceback.print_tb(sys.exc_info()[2])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing a string as a positional argument or a tuple containing a keyword and its value for a single argument.", "response": "def parse_argument(string: str) -> Union[str, Tuple[str, str]]:\n \"\"\"Return a single value for a string understood as a positional\n argument or a |tuple| containing a keyword and its value for a\n string understood as a keyword argument.\n\n |parse_argument| is intended to be used as a helper function for\n function |execute_scriptfunction| only. See the following\n examples to see which types of keyword arguments |execute_scriptfunction|\n covers:\n\n >>> from hydpy.exe.commandtools import parse_argument\n >>> parse_argument('x=3')\n ('x', '3')\n >>> parse_argument('\"x=3\"')\n '\"x=3\"'\n >>> parse_argument(\"'x=3'\")\n \"'x=3'\"\n >>> parse_argument('x=\"3==3\"')\n ('x', '\"3==3\"')\n >>> parse_argument(\"x='3==3'\")\n ('x', \"'3==3'\")\n \"\"\"\n idx_equal = string.find('=')\n if idx_equal == -1:\n return string\n idx_quote = idx_equal+1\n for quote in ('\"', \"'\"):\n idx = string.find(quote)\n if -1 < idx < idx_quote:\n idx_quote = idx\n if idx_equal < idx_quote:\n return string[:idx_equal], string[idx_equal+1:]\n return string"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprints the given string and the current date and time.", "response": "def print_textandtime(text: str) -> None:\n \"\"\"Print the given string and the current date and time with high\n precision for logging purposes.\n\n >>> from hydpy.exe.commandtools import print_textandtime\n >>> from hydpy.core.testtools import mock_datetime_now\n >>> from datetime import datetime\n >>> with mock_datetime_now(datetime(2000, 1, 1, 12, 30, 0, 123456)):\n ... print_textandtime('something happens')\n something happens (2000-01-01 12:30:00.123456).\n \"\"\"\n timestring = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')\n print(f'{text} ({timestring}).')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nwriting the given string as explained in the main documentation on class |LogFileInterface|.", "response": "def write(self, string: str) -> None:\n \"\"\"Write the given string as explained in the main documentation\n on class |LogFileInterface|.\"\"\"\n self.logfile.write('\\n'.join(\n f'{self._string}{substring}' if substring else ''\n for substring in string.split('\\n')))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsolving the differential equation of HydPy - L.", "response": "def solve_dv_dt_v1(self):\n \"\"\"Solve the differential equation of HydPy-L.\n\n At the moment, HydPy-L only implements a simple numerical solution of\n its underlying ordinary differential equation. To increase the accuracy\n (or sometimes even to prevent instability) of this approximation, one\n can set the value of parameter |MaxDT| to a value smaller than the actual\n simulation step size. Method |solve_dv_dt_v1| then applies the methods\n related to the numerical approximation multiple times and aggregates\n the results.\n\n Note that the order of convergence is one only. It is hard to tell how\n short the internal simulation step needs to be to ensure a certain degree\n of accuracy. In most cases one hour or very often even one day should be\n sufficient to gain acceptable results. However, this strongly depends on\n the given water stage-volume-discharge relationship. Hence it seems\n advisable to always define a few test waves and apply the llake model with\n different |MaxDT| values. Afterwards, select a |MaxDT| value lower than\n one which results in acceptable approximations for all test waves. The\n computation time of the llake mode per substep is rather small, so always\n include a safety factor.\n\n Of course, an adaptive step size determination would be much more\n convenient...\n\n Required derived parameter:\n |NmbSubsteps|\n\n Used aide sequence:\n |llake_aides.V|\n |llake_aides.QA|\n\n Updated state sequence:\n |llake_states.V|\n\n Calculated flux sequence:\n |llake_fluxes.QA|\n\n Note that method |solve_dv_dt_v1| calls the versions of `calc_vq`,\n `interp_qa` and `calc_v_qa` selected by the respective application model.\n Hence, also their parameter and sequence specifications need to be\n considered.\n\n Basic equation:\n :math:`\\\\frac{dV}{dt}= QZ - QA(V)`\n \"\"\"\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n aid = self.sequences.aides.fastaccess\n flu.qa = 0.\n aid.v = old.v\n for _ in range(der.nmbsubsteps):\n self.calc_vq()\n self.interp_qa()\n self.calc_v_qa()\n flu.qa += aid.qa\n flu.qa /= der.nmbsubsteps\n new.v = aid.v"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the auxiliary term v1.", "response": "def calc_vq_v1(self):\n \"\"\"Calculate the auxiliary term.\n\n Required derived parameters:\n |Seconds|\n |NmbSubsteps|\n\n Required flux sequence:\n |QZ|\n\n Required aide sequence:\n |llake_aides.V|\n\n Calculated aide sequence:\n |llake_aides.VQ|\n\n Basic equation:\n :math:`VQ = 2 \\\\cdot V + \\\\frac{Seconds}{NmbSubsteps} \\\\cdot QZ`\n\n Example:\n\n The following example shows that the auxiliary term `vq` does not\n depend on the (outer) simulation step size but on the (inner)\n calculation step size defined by parameter `maxdt`:\n\n >>> from hydpy.models.llake import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> maxdt('6h')\n >>> derived.seconds.update()\n >>> derived.nmbsubsteps.update()\n >>> fluxes.qz = 2.\n >>> aides.v = 1e5\n >>> model.calc_vq_v1()\n >>> aides.vq\n vq(243200.0)\n \"\"\"\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n aid.vq = 2.*aid.v+der.seconds/der.nmbsubsteps*flu.qz"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef interp_qa_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n aid = self.sequences.aides.fastaccess\n idx = der.toy[self.idx_sim]\n for jdx in range(1, con.n):\n if der.vq[idx, jdx] >= aid.vq:\n break\n aid.qa = ((aid.vq-der.vq[idx, jdx-1]) *\n (con.q[idx, jdx]-con.q[idx, jdx-1]) /\n (der.vq[idx, jdx]-der.vq[idx, jdx-1]) +\n con.q[idx, jdx-1])\n aid.qa = max(aid.qa, 0.)", "response": "Calculate the lake outflow based on linear interpolation."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating the stored water volume based on the equation of continuity.", "response": "def calc_v_qa_v1(self):\n \"\"\"Update the stored water volume based on the equation of continuity.\n\n Note that for too high outflow values, which would result in overdraining\n the lake, the outflow is trimmed.\n\n Required derived parameters:\n |Seconds|\n |NmbSubsteps|\n\n Required flux sequence:\n |QZ|\n\n Updated aide sequences:\n |llake_aides.QA|\n |llake_aides.V|\n\n Basic Equation:\n :math:`\\\\frac{dV}{dt}= QZ - QA`\n\n Examples:\n\n Prepare a lake model with an initial storage of 100.000 m\u00b3 and an\n inflow of 2 m\u00b3/s and a (potential) outflow of 6 m\u00b3/s:\n\n >>> from hydpy.models.llake import *\n >>> parameterstep()\n >>> simulationstep('12h')\n >>> maxdt('6h')\n >>> derived.seconds.update()\n >>> derived.nmbsubsteps.update()\n >>> aides.v = 1e5\n >>> fluxes.qz = 2.\n >>> aides.qa = 6.\n\n Through calling method `calc_v_qa_v1` three times with the same inflow\n and outflow values, the storage is emptied after the second step and\n outflow is equal to inflow after the third step:\n\n >>> model.calc_v_qa_v1()\n >>> aides.v\n v(13600.0)\n >>> aides.qa\n qa(6.0)\n >>> model.new2old()\n >>> model.calc_v_qa_v1()\n >>> aides.v\n v(0.0)\n >>> aides.qa\n qa(2.62963)\n >>> model.new2old()\n >>> model.calc_v_qa_v1()\n >>> aides.v\n v(0.0)\n >>> aides.qa\n qa(2.0)\n\n Note that the results of method |calc_v_qa_v1| are not based\n depend on the (outer) simulation step size but on the (inner)\n calculation step size defined by parameter `maxdt`.\n \"\"\"\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n aid.qa = min(aid.qa, flu.qz+der.nmbsubsteps/der.seconds*aid.v)\n aid.v = max(aid.v+der.seconds/der.nmbsubsteps*(flu.qz-aid.qa), 0.)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the actual water stage based on linear interpolation.", "response": "def interp_w_v1(self):\n \"\"\"Calculate the actual water stage based on linear interpolation.\n\n Required control parameters:\n |N|\n |llake_control.V|\n |llake_control.W|\n\n Required state sequence:\n |llake_states.V|\n\n Calculated state sequence:\n |llake_states.W|\n\n Examples:\n\n Prepare a model object:\n\n >>> from hydpy.models.llake import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n\n For the sake of brevity, define a test function:\n\n >>> def test(*vs):\n ... for v in vs:\n ... states.v.new = v\n ... model.interp_w_v1()\n ... print(repr(states.v), repr(states.w))\n\n Define a simple `w`-`v` relationship consisting of three nodes and\n calculate the water stages for different volumes:\n\n >>> n(3)\n >>> v(0., 2., 4.)\n >>> w(-1., 1., 2.)\n\n Perform the interpolation for a few test points:\n\n >>> test(0., .5, 2., 3., 4., 5.)\n v(0.0) w(-1.0)\n v(0.5) w(-0.5)\n v(2.0) w(1.0)\n v(3.0) w(1.5)\n v(4.0) w(2.0)\n v(5.0) w(2.5)\n\n The reference water stage of the relationship can be selected\n arbitrarily. Even negative water stages are returned, as is\n demonstrated by the first two calculations. For volumes outside\n the range of the (`v`,`w`) pairs, the outer two highest pairs are\n used for linear extrapolation.\n \"\"\"\n con = self.parameters.control.fastaccess\n new = self.sequences.states.fastaccess_new\n for jdx in range(1, con.n):\n if con.v[jdx] >= new.v:\n break\n new.w = ((new.v-con.v[jdx-1]) *\n (con.w[jdx]-con.w[jdx-1]) /\n (con.v[jdx]-con.v[jdx-1]) +\n con.w[jdx-1])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadjusting the water stage drop to the highest value allowed and correct the associated fluxes.", "response": "def corr_dw_v1(self):\n \"\"\"Adjust the water stage drop to the highest value allowed and correct\n the associated fluxes.\n\n Note that method |corr_dw_v1| calls the method `interp_v` of the\n respective application model. Hence the requirements of the actual\n `interp_v` need to be considered additionally.\n\n Required control parameter:\n |MaxDW|\n\n Required derived parameters:\n |llake_derived.TOY|\n |Seconds|\n\n Required flux sequence:\n |QZ|\n\n Updated flux sequence:\n |llake_fluxes.QA|\n\n Updated state sequences:\n |llake_states.W|\n |llake_states.V|\n\n Basic Restriction:\n :math:`W_{old} - W_{new} \\\\leq MaxDW`\n\n Examples:\n\n In preparation for the following examples, define a short simulation\n time period with a simulation step size of 12 hours and initialize\n the required model object:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2000.01.01', '2000.01.04', '12h'\n >>> from hydpy.models.llake import *\n >>> parameterstep('1d')\n >>> derived.toy.update()\n >>> derived.seconds.update()\n\n Select the first half of the second day of January as the simulation\n step relevant for the following examples:\n\n >>> model.idx_sim = pub.timegrids.init['2000.01.02']\n\n The following tests are based on method |interp_v_v1| for the\n interpolation of the stored water volume based on the corrected\n water stage:\n\n >>> model.interp_v = model.interp_v_v1\n\n For the sake of simplicity, the underlying `w`-`v` relationship is\n assumed to be linear:\n\n >>> n(2.)\n >>> w(0., 1.)\n >>> v(0., 1e6)\n\n The maximum drop in water stage for the first half of the second\n day of January is set to 0.4 m/d. Note that, due to the difference\n between the parameter step size and the simulation step size, the\n actual value used for calculation is 0.2 m/12h:\n\n >>> maxdw(_1_1_18=.1,\n ... _1_2_6=.4,\n ... _1_2_18=.1)\n >>> maxdw\n maxdw(toy_1_1_18_0_0=0.1,\n toy_1_2_6_0_0=0.4,\n toy_1_2_18_0_0=0.1)\n >>> from hydpy import round_\n >>> round_(maxdw.value[2])\n 0.2\n\n Define old and new water stages and volumes in agreement with the\n given linear relationship:\n\n >>> states.w.old = 1.\n >>> states.v.old = 1e6\n >>> states.w.new = .9\n >>> states.v.new = 9e5\n\n Also define an inflow and an outflow value. Note the that the latter\n is set to zero, which is inconsistent with the actual water stage drop\n defined above, but done for didactic reasons:\n\n >>> fluxes.qz = 1.\n >>> fluxes.qa = 0.\n\n Calling the |corr_dw_v1| method does not change the values of\n either of following sequences, as the actual drop (0.1 m/12h) is\n smaller than the allowed drop (0.2 m/12h):\n\n >>> model.corr_dw_v1()\n >>> states.w\n w(0.9)\n >>> states.v\n v(900000.0)\n >>> fluxes.qa\n qa(0.0)\n\n Note that the values given above are not recalculated, which can\n clearly be seen for the lake outflow, which is still zero.\n\n Through setting the new value of the water stage to 0.6 m, the actual\n drop (0.4 m/12h) exceeds the allowed drop (0.2 m/12h). Hence the\n water stage is trimmed and the other values are recalculated:\n\n >>> states.w.new = .6\n >>> model.corr_dw_v1()\n >>> states.w\n w(0.8)\n >>> states.v\n v(800000.0)\n >>> fluxes.qa\n qa(5.62963)\n\n Through setting the maximum water stage drop to zero, method\n |corr_dw_v1| is effectively disabled. Regardless of the actual\n change in water stage, no trimming or recalculating is performed:\n\n >>> maxdw.toy_01_02_06 = 0.\n >>> states.w.new = .6\n >>> model.corr_dw_v1()\n >>> states.w\n w(0.6)\n >>> states.v\n v(800000.0)\n >>> fluxes.qa\n qa(5.62963)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n idx = der.toy[self.idx_sim]\n if (con.maxdw[idx] > 0.) and ((old.w-new.w) > con.maxdw[idx]):\n new.w = old.w-con.maxdw[idx]\n self.interp_v()\n flu.qa = flu.qz+(old.v-new.v)/der.seconds"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef modify_qa_v1(self):\n\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n idx = der.toy[self.idx_sim]\n flu.qa = max(flu.qa-con.verzw[idx], 0.)", "response": "Modify the flux sequence of the current model object to include or remove water from the calculated lake outflow."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pass_q_v1(self):\n flu = self.sequences.fluxes.fastaccess\n out = self.sequences.outlets.fastaccess\n out.q[0] += flu.qa", "response": "Update the link sequence."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nthresholding values of the response functions.", "response": "def thresholds(self):\n \"\"\"Threshold values of the response functions.\"\"\"\n return numpy.array(\n sorted(self._key2float(key) for key in self._coefs), dtype=float)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef prepare_arrays(sim=None, obs=None, node=None, skip_nan=False):\n if node:\n if sim is not None:\n raise ValueError(\n 'Values are passed to both arguments `sim` and `node`, '\n 'which is not allowed.')\n if obs is not None:\n raise ValueError(\n 'Values are passed to both arguments `obs` and `node`, '\n 'which is not allowed.')\n sim = node.sequences.sim.series\n obs = node.sequences.obs.series\n elif (sim is not None) and (obs is None):\n raise ValueError(\n 'A value is passed to argument `sim` '\n 'but no value is passed to argument `obs`.')\n elif (obs is not None) and (sim is None):\n raise ValueError(\n 'A value is passed to argument `obs` '\n 'but no value is passed to argument `sim`.')\n elif (sim is None) and (obs is None):\n raise ValueError(\n 'Neither a `Node` object is passed to argument `node` nor '\n 'are arrays passed to arguments `sim` and `obs`.')\n sim = numpy.asarray(sim)\n obs = numpy.asarray(obs)\n if skip_nan:\n idxs = ~numpy.isnan(sim) * ~numpy.isnan(obs)\n sim = sim[idxs]\n obs = obs[idxs]\n return sim, obs", "response": "Prepare and return two |numpy| arrays based on the given arguments."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef nse(sim=None, obs=None, node=None, skip_nan=False):\n sim, obs = prepare_arrays(sim, obs, node, skip_nan)\n return 1.-numpy.sum((sim-obs)**2)/numpy.sum((obs-numpy.mean(obs))**2)", "response": "Calculate the efficiency criteria after Nash & Sutcliffe."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating the absolute difference between the means of the simulated and observed values.", "response": "def bias_abs(sim=None, obs=None, node=None, skip_nan=False):\n \"\"\"Calculate the absolute difference between the means of the simulated\n and the observed values.\n\n >>> from hydpy import round_\n >>> from hydpy import bias_abs\n >>> round_(bias_abs(sim=[2.0, 2.0, 2.0], obs=[1.0, 2.0, 3.0]))\n 0.0\n >>> round_(bias_abs(sim=[5.0, 2.0, 2.0], obs=[1.0, 2.0, 3.0]))\n 1.0\n >>> round_(bias_abs(sim=[1.0, 1.0, 1.0], obs=[1.0, 2.0, 3.0]))\n -1.0\n\n See the documentation on function |prepare_arrays| for some\n additional instructions for use of function |bias_abs|.\n \"\"\"\n sim, obs = prepare_arrays(sim, obs, node, skip_nan)\n return numpy.mean(sim-obs)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the ratio between the standard deviation of the simulated and observed values.", "response": "def std_ratio(sim=None, obs=None, node=None, skip_nan=False):\n \"\"\"Calculate the ratio between the standard deviation of the simulated\n and the observed values.\n\n >>> from hydpy import round_\n >>> from hydpy import std_ratio\n >>> round_(std_ratio(sim=[1.0, 2.0, 3.0], obs=[1.0, 2.0, 3.0]))\n 0.0\n >>> round_(std_ratio(sim=[1.0, 1.0, 1.0], obs=[1.0, 2.0, 3.0]))\n -1.0\n >>> round_(std_ratio(sim=[0.0, 3.0, 6.0], obs=[1.0, 2.0, 3.0]))\n 2.0\n\n See the documentation on function |prepare_arrays| for some\n additional instructions for use of function |std_ratio|.\n \"\"\"\n sim, obs = prepare_arrays(sim, obs, node, skip_nan)\n return numpy.std(sim)/numpy.std(obs)-1."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncalculate the product - moment correlation coefficient after Pearson.", "response": "def corr(sim=None, obs=None, node=None, skip_nan=False):\n \"\"\"Calculate the product-moment correlation coefficient after Pearson.\n\n >>> from hydpy import round_\n >>> from hydpy import corr\n >>> round_(corr(sim=[0.5, 1.0, 1.5], obs=[1.0, 2.0, 3.0]))\n 1.0\n >>> round_(corr(sim=[4.0, 2.0, 0.0], obs=[1.0, 2.0, 3.0]))\n -1.0\n >>> round_(corr(sim=[1.0, 2.0, 1.0], obs=[1.0, 2.0, 3.0]))\n 0.0\n\n See the documentation on function |prepare_arrays| for some\n additional instructions for use of function |corr|.\n \"\"\"\n sim, obs = prepare_arrays(sim, obs, node, skip_nan)\n return numpy.corrcoef(sim, obs)[0, 1]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef hsepd_pdf(sigma1, sigma2, xi, beta,\n sim=None, obs=None, node=None, skip_nan=False):\n \"\"\"Calculate the probability densities based on the\n heteroskedastic skewed exponential power distribution.\n\n For convenience, the required parameters of the probability density\n function as well as the simulated and observed values are stored\n in a dictonary:\n\n >>> import numpy\n >>> from hydpy import round_\n >>> from hydpy import hsepd_pdf\n >>> general = {'sigma1': 0.2,\n ... 'sigma2': 0.0,\n ... 'xi': 1.0,\n ... 'beta': 0.0,\n ... 'sim': numpy.arange(10.0, 41.0),\n ... 'obs': numpy.full(31, 25.0)}\n\n The following test function allows the variation of one parameter\n and prints some and plots all of probability density values\n corresponding to different simulated values:\n\n >>> def test(**kwargs):\n ... from matplotlib import pyplot\n ... special = general.copy()\n ... name, values = list(kwargs.items())[0]\n ... results = numpy.zeros((len(general['sim']), len(values)+1))\n ... results[:, 0] = general['sim']\n ... for jdx, value in enumerate(values):\n ... special[name] = value\n ... results[:, jdx+1] = hsepd_pdf(**special)\n ... pyplot.plot(results[:, 0], results[:, jdx+1],\n ... label='%s=%.1f' % (name, value))\n ... pyplot.legend()\n ... for idx, result in enumerate(results):\n ... if not (idx % 5):\n ... round_(result)\n\n When varying parameter `beta`, the resulting probabilities correspond\n to the Laplace distribution (1.0), normal distribution (0.0), and the\n uniform distribution (-1.0), respectively. Note that we use -0.99\n instead of -1.0 for approximating the uniform distribution to prevent\n from running into numerical problems, which are not solved yet:\n\n >>> test(beta=[1.0, 0.0, -0.99])\n 10.0, 0.002032, 0.000886, 0.0\n 15.0, 0.008359, 0.010798, 0.0\n 20.0, 0.034382, 0.048394, 0.057739\n 25.0, 0.141421, 0.079788, 0.057739\n 30.0, 0.034382, 0.048394, 0.057739\n 35.0, 0.008359, 0.010798, 0.0\n 40.0, 0.002032, 0.000886, 0.0\n\n .. testsetup::\n\n >>> from matplotlib import pyplot\n >>> pyplot.close()\n\n When varying parameter `xi`, the resulting density is negatively\n skewed (0.2), symmetric (1.0), and positively skewed (5.0),\n respectively:\n\n >>> test(xi=[0.2, 1.0, 5.0])\n 10.0, 0.0, 0.000886, 0.003175\n 15.0, 0.0, 0.010798, 0.012957\n 20.0, 0.092845, 0.048394, 0.036341\n 25.0, 0.070063, 0.079788, 0.070063\n 30.0, 0.036341, 0.048394, 0.092845\n 35.0, 0.012957, 0.010798, 0.0\n 40.0, 0.003175, 0.000886, 0.0\n\n .. testsetup::\n\n >>> from matplotlib import pyplot\n >>> pyplot.close()\n\n In the above examples, the actual `sigma` (5.0) is calculated by\n multiplying `sigma1` (0.2) with the mean simulated value (25.0),\n internally. This can be done for modelling homoscedastic errors.\n Instead, `sigma2` is multiplied with the individual simulated values\n to account for heteroscedastic errors. With increasing values of\n `sigma2`, the resulting densities are modified as follows:\n\n >>> test(sigma2=[0.0, 0.1, 0.2])\n 10.0, 0.000886, 0.002921, 0.005737\n 15.0, 0.010798, 0.018795, 0.022831\n 20.0, 0.048394, 0.044159, 0.037988\n 25.0, 0.079788, 0.053192, 0.039894\n 30.0, 0.048394, 0.04102, 0.032708\n 35.0, 0.010798, 0.023493, 0.023493\n 40.0, 0.000886, 0.011053, 0.015771\n\n .. testsetup::\n\n >>> from matplotlib import pyplot\n >>> pyplot.close()\n \"\"\"\n sim, obs = prepare_arrays(sim, obs, node, skip_nan)\n sigmas = _pars_h(sigma1, sigma2, sim)\n mu_xi, sigma_xi, w_beta, c_beta = _pars_sepd(xi, beta)\n x, mu = obs, sim\n a = (x-mu)/sigmas\n a_xi = numpy.empty(a.shape)\n idxs = mu_xi+sigma_xi*a < 0.\n a_xi[idxs] = numpy.absolute(xi*(mu_xi+sigma_xi*a[idxs]))\n a_xi[~idxs] = numpy.absolute(1./xi*(mu_xi+sigma_xi*a[~idxs]))\n ps = (2.*sigma_xi/(xi+1./xi)*w_beta *\n numpy.exp(-c_beta*a_xi**(2./(1.+beta))))/sigmas\n return ps", "response": "This function calculates the probability densities based on the HSEPD distribution."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the mean of the logarithmised probability densities of the heteroskedastic skewed exponential power distribution.", "response": "def hsepd_manual(sigma1, sigma2, xi, beta,\n sim=None, obs=None, node=None, skip_nan=False):\n \"\"\"Calculate the mean of the logarithmised probability densities of the\n 'heteroskedastic skewed exponential power distribution.\n\n The following examples are taken from the documentation of function\n |hsepd_pdf|, which is used by function |hsepd_manual|. The first\n one deals with a heteroscedastic normal distribution:\n\n >>> from hydpy import round_\n >>> from hydpy import hsepd_manual\n >>> round_(hsepd_manual(sigma1=0.2, sigma2=0.2,\n ... xi=1.0, beta=0.0,\n ... sim=numpy.arange(10.0, 41.0),\n ... obs=numpy.full(31, 25.0)))\n -3.682842\n\n The second one is supposed to show to small zero probability density\n values are set to 1e-200 before calculating their logarithm (which\n means that the lowest possible value returned by function\n |hsepd_manual| is approximately -460):\n\n >>> round_(hsepd_manual(sigma1=0.2, sigma2=0.0,\n ... xi=1.0, beta=-0.99,\n ... sim=numpy.arange(10.0, 41.0),\n ... obs=numpy.full(31, 25.0)))\n -209.539335\n \"\"\"\n sim, obs = prepare_arrays(sim, obs, node, skip_nan)\n return _hsepd_manual(sigma1, sigma2, xi, beta, sim, obs)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the mean of the logarithmised probability densities of the heteroskedastic skewed exponential power distribution.", "response": "def hsepd(sim=None, obs=None, node=None, skip_nan=False,\n inits=None, return_pars=False, silent=True):\n \"\"\"Calculate the mean of the logarithmised probability densities of the\n 'heteroskedastic skewed exponential power distribution.\n\n Function |hsepd| serves the same purpose as function |hsepd_manual|,\n but tries to estimate the parameters of the heteroscedastic skewed\n exponential distribution via an optimization algorithm. This\n is shown by generating a random sample. 1000 simulated values\n are scattered around the observed (true) value of 10.0 with a\n standard deviation of 2.0:\n\n >>> import numpy\n >>> numpy.random.seed(0)\n >>> sim = numpy.random.normal(10.0, 2.0, 1000)\n >>> obs = numpy.full(1000, 10.0)\n\n First, as a reference, we calculate the \"true\" value based on\n function |hsepd_manual| and the correct distribution parameters:\n\n >>> from hydpy import round_\n >>> from hydpy import hsepd, hsepd_manual\n >>> round_(hsepd_manual(sigma1=0.2, sigma2=0.0,\n ... xi=1.0, beta=0.0,\n ... sim=sim, obs=obs))\n -2.100093\n\n When using function |hsepd|, the returned value is even a little\n \"better\":\n\n >>> round_(hsepd(sim=sim, obs=obs))\n -2.09983\n\n This is due to the deviation from the random sample to its\n theoretical distribution. This is reflected by small differences\n between the estimated values and the theoretical values of\n `sigma1` (0.2), , `sigma2` (0.0), `xi` (1.0), and `beta` (0.0).\n The estimated values are returned in the mentioned order through\n enabling the `return_pars` option:\n\n >>> value, pars = hsepd(sim=sim, obs=obs, return_pars=True)\n >>> round_(pars, decimals=5)\n 0.19966, 0.0, 0.96836, 0.0188\n\n There is no guarantee that the optimization numerical optimization\n algorithm underlying function |hsepd| will always find the parameters\n resulting in the largest value returned by function |hsepd_manual|.\n You can increase its robustness (and decrease computation time) by\n supplying good initial parameter values:\n\n >>> value, pars = hsepd(sim=sim, obs=obs, return_pars=True,\n ... inits=(0.2, 0.0, 1.0, 0.0))\n >>> round_(pars, decimals=5)\n 0.19966, 0.0, 0.96836, 0.0188\n\n However, the following example shows a case when this strategie\n results in worse results:\n\n >>> value, pars = hsepd(sim=sim, obs=obs, return_pars=True,\n ... inits=(0.0, 0.2, 1.0, 0.0))\n >>> round_(value)\n -2.174492\n >>> round_(pars)\n 0.0, 0.213179, 1.705485, 0.505112\n \"\"\"\n\n def transform(pars):\n \"\"\"Transform the actual optimization problem into a function to\n be minimized and apply parameter constraints.\"\"\"\n sigma1, sigma2, xi, beta = constrain(*pars)\n return -_hsepd_manual(sigma1, sigma2, xi, beta, sim, obs)\n\n def constrain(sigma1, sigma2, xi, beta):\n \"\"\"Apply constrains on the given parameter values.\"\"\"\n sigma1 = numpy.clip(sigma1, 0.0, None)\n sigma2 = numpy.clip(sigma2, 0.0, None)\n xi = numpy.clip(xi, 0.1, 10.0)\n beta = numpy.clip(beta, -0.99, 5.0)\n return sigma1, sigma2, xi, beta\n\n sim, obs = prepare_arrays(sim, obs, node, skip_nan)\n if not inits:\n inits = [0.1, 0.2, 3.0, 1.0]\n values = optimize.fmin(transform, inits,\n ftol=1e-12, xtol=1e-12,\n disp=not silent)\n values = constrain(*values)\n result = _hsepd_manual(*values, sim=sim, obs=obs)\n if return_pars:\n return result, values\n return result"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_mean_time(timepoints, weights):\n timepoints = numpy.array(timepoints)\n weights = numpy.array(weights)\n validtools.test_equal_shape(timepoints=timepoints, weights=weights)\n validtools.test_non_negative(weights=weights)\n return numpy.dot(timepoints, weights)/numpy.sum(weights)", "response": "Calculate the weighted mean time of the given timepoints."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the weighted deviation of the given timepoints from their mean time.", "response": "def calc_mean_time_deviation(timepoints, weights, mean_time=None):\n \"\"\"Return the weighted deviation of the given timepoints from their mean\n time.\n\n With equal given weights, the is simply the standard deviation of the\n given time points:\n\n >>> from hydpy import calc_mean_time_deviation\n >>> calc_mean_time_deviation(timepoints=[3., 7.],\n ... weights=[2., 2.])\n 2.0\n\n One can pass a precalculated or alternate mean time:\n\n >>> from hydpy import round_\n >>> round_(calc_mean_time_deviation(timepoints=[3., 7.],\n ... weights=[2., 2.],\n ... mean_time=4.))\n 2.236068\n\n >>> round_(calc_mean_time_deviation(timepoints=[3., 7.],\n ... weights=[1., 3.]))\n 1.732051\n\n Or, in the most extreme case:\n\n >>> calc_mean_time_deviation(timepoints=[3., 7.],\n ... weights=[0., 4.])\n 0.0\n\n There will be some checks for input plausibility perfomed, e.g.:\n\n >>> calc_mean_time_deviation(timepoints=[3., 7.],\n ... weights=[-2., 2.])\n Traceback (most recent call last):\n ...\n ValueError: While trying to calculate the weighted time deviation \\\nfrom mean time, the following error occurred: For the following objects, \\\nat least one value is negative: weights.\n \"\"\"\n timepoints = numpy.array(timepoints)\n weights = numpy.array(weights)\n validtools.test_equal_shape(timepoints=timepoints, weights=weights)\n validtools.test_non_negative(weights=weights)\n if mean_time is None:\n mean_time = calc_mean_time(timepoints, weights)\n return (numpy.sqrt(numpy.dot(weights, (timepoints-mean_time)**2) /\n numpy.sum(weights)))"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a table containing the results of the given evaluation criteria for the given |Node| objects.", "response": "def evaluationtable(nodes, criteria, nodenames=None,\n critnames=None, skip_nan=False):\n \"\"\"Return a table containing the results of the given evaluation\n criteria for the given |Node| objects.\n\n First, we define two nodes with different simulation and observation\n data (see function |prepare_arrays| for some explanations):\n\n >>> from hydpy import pub, Node, nan\n >>> pub.timegrids = '01.01.2000', '04.01.2000', '1d'\n >>> nodes = Node('test1'), Node('test2')\n >>> for node in nodes:\n ... node.prepare_simseries()\n ... node.sequences.sim.series = 1.0, 2.0, 3.0\n ... node.sequences.obs.ramflag = True\n ... node.sequences.obs.series = 4.0, 5.0, 6.0\n >>> nodes[0].sequences.sim.series = 1.0, 2.0, 3.0\n >>> nodes[0].sequences.obs.series = 4.0, 5.0, 6.0\n >>> nodes[1].sequences.sim.series = 1.0, 2.0, 3.0\n >>> with pub.options.checkseries(False):\n ... nodes[1].sequences.obs.series = 3.0, nan, 1.0\n\n Selecting functions |corr| and |bias_abs| as evaluation criteria,\n function |evaluationtable| returns the following table (which is\n actually a pandas data frame):\n\n >>> from hydpy import evaluationtable, corr, bias_abs\n >>> evaluationtable(nodes, (corr, bias_abs))\n corr bias_abs\n test1 1.0 -3.0\n test2 NaN NaN\n\n One can pass alternative names for both the node objects and the\n criteria functions. Also, `nan` values can be skipped:\n\n >>> evaluationtable(nodes, (corr, bias_abs),\n ... nodenames=('first node', 'second node'),\n ... critnames=('corrcoef', 'bias'),\n ... skip_nan=True)\n corrcoef bias\n first node 1.0 -3.0\n second node -1.0 0.0\n\n The number of assigned node objects and criteria functions must\n match the number of givern alternative names:\n\n >>> evaluationtable(nodes, (corr, bias_abs),\n ... nodenames=('first node',))\n Traceback (most recent call last):\n ...\n ValueError: While trying to evaluate the simulation results of some \\\nnode objects, the following error occurred: 2 node objects are given \\\nwhich does not match with number of given alternative names beeing 1.\n\n >>> evaluationtable(nodes, (corr, bias_abs),\n ... critnames=('corrcoef',))\n Traceback (most recent call last):\n ...\n ValueError: While trying to evaluate the simulation results of some \\\nnode objects, the following error occurred: 2 criteria functions are given \\\nwhich does not match with number of given alternative names beeing 1.\n \"\"\"\n if nodenames:\n if len(nodes) != len(nodenames):\n raise ValueError(\n '%d node objects are given which does not match with '\n 'number of given alternative names beeing %s.'\n % (len(nodes), len(nodenames)))\n else:\n nodenames = [node.name for node in nodes]\n if critnames:\n if len(criteria) != len(critnames):\n raise ValueError(\n '%d criteria functions are given which does not match '\n 'with number of given alternative names beeing %s.'\n % (len(criteria), len(critnames)))\n else:\n critnames = [crit.__name__ for crit in criteria]\n data = numpy.empty((len(nodes), len(criteria)), dtype=float)\n for idx, node in enumerate(nodes):\n sim, obs = prepare_arrays(None, None, node, skip_nan)\n for jdx, criterion in enumerate(criteria):\n data[idx, jdx] = criterion(sim, obs)\n table = pandas.DataFrame(\n data=data, index=nodenames, columns=critnames)\n return table"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset all primary parameters at once.", "response": "def set_primary_parameters(self, **kwargs):\n \"\"\"Set all primary parameters at once.\"\"\"\n given = sorted(kwargs.keys())\n required = sorted(self._PRIMARY_PARAMETERS)\n if given == required:\n for (key, value) in kwargs.items():\n setattr(self, key, value)\n else:\n raise ValueError(\n 'When passing primary parameter values as initialization '\n 'arguments of the instantaneous unit hydrograph class `%s`, '\n 'or when using method `set_primary_parameters, one has to '\n 'to define all values at once via keyword arguments. '\n 'But instead of the primary parameter names `%s` the '\n 'following keywords were given: %s.'\n % (objecttools.classname(self),\n ', '.join(required), ', '.join(given)))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef primary_parameters_complete(self):\n for primpar in self._PRIMARY_PARAMETERS.values():\n if primpar.__get__(self) is None:\n return False\n return True", "response": "True if all primary parameters are defined or not."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates the values of the values of the MA and AR models.", "response": "def update(self):\n \"\"\"Delete the coefficients of the pure MA model and also all MA and\n AR coefficients of the ARMA model. Also calculate or delete the values\n of all secondary iuh parameters, depending on the completeness of the\n values of the primary parameters.\n \"\"\"\n del self.ma.coefs\n del self.arma.ma_coefs\n del self.arma.ar_coefs\n if self.primary_parameters_complete:\n self.calc_secondary_parameters()\n else:\n for secpar in self._SECONDARY_PARAMETERS.values():\n secpar.__delete__(self)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot(self, threshold=None, **kwargs):\n delays, responses = self.delay_response_series\n pyplot.plot(delays, responses, **kwargs)\n pyplot.xlabel('time')\n pyplot.ylabel('response')\n if threshold is not None:\n threshold = numpy.clip(threshold, 0., 1.)\n cumsum = numpy.cumsum(responses)\n idx = numpy.where(cumsum >= threshold*cumsum[-1])[0][0]\n pyplot.xlim(0., delays[idx])", "response": "Plot the instanteneous unit hydrograph."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef moment1(self):\n delays, response = self.delay_response_series\n return statstools.calc_mean_time(delays, response)", "response": "The first time delay weighted statistical moment of the\n instantaneous unit hydrograph."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef moment2(self):\n moment1 = self.moment1\n delays, response = self.delay_response_series\n return statstools.calc_mean_time_deviation(\n delays, response, moment1)", "response": "The second time delay weighted statistical momens of the\n instantaneous unit hydrograph."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calc_secondary_parameters(self):\n self.a = self.x/(2.*self.d**.5)\n self.b = self.u/(2.*self.d**.5)", "response": "Determine the values of the secondary parameters a and b."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef calc_secondary_parameters(self):\n self.c = 1./(self.k*special.gamma(self.n))", "response": "Determine the value of the secondary parameter c."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntrim values in accordance with WAeS \\\\ leq PWMax \\\\ cdot WATS.", "response": "def trim(self, lower=None, upper=None):\n \"\"\"Trim values in accordance with :math:`WAeS \\\\leq PWMax \\\\cdot WATS`,\n or at least in accordance with if :math:`WATS \\\\geq 0`.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(7)\n >>> pwmax(2.0)\n >>> states.waes = -1., 0., 1., -1., 5., 10., 20.\n >>> states.wats(-1., 0., 0., 5., 5., 5., 5.)\n >>> states.wats\n wats(0.0, 0.0, 0.5, 5.0, 5.0, 5.0, 10.0)\n \"\"\"\n pwmax = self.subseqs.seqs.model.parameters.control.pwmax\n waes = self.subseqs.waes\n if lower is None:\n lower = numpy.clip(waes/pwmax, 0., numpy.inf)\n lower[numpy.isnan(lower)] = 0.0\n lland_sequences.State1DSequence.trim(self, lower, upper)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntrim values in accordance with WAeS \\\\ leq PWMax \\\\ cdot WATS.", "response": "def trim(self, lower=None, upper=None):\n \"\"\"Trim values in accordance with :math:`WAeS \\\\leq PWMax \\\\cdot WATS`.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(7)\n >>> pwmax(2.)\n >>> states.wats = 0., 0., 0., 5., 5., 5., 5.\n >>> states.waes(-1., 0., 1., -1., 5., 10., 20.)\n >>> states.waes\n waes(0.0, 0.0, 0.0, 0.0, 5.0, 10.0, 10.0)\n \"\"\"\n pwmax = self.subseqs.seqs.model.parameters.control.pwmax\n wats = self.subseqs.wats\n if upper is None:\n upper = pwmax*wats\n lland_sequences.State1DSequence.trim(self, lower, upper)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ntrimming values in accordance with Bowa \\\\ leq NFk.", "response": "def trim(self, lower=None, upper=None):\n \"\"\"Trim values in accordance with :math:`BoWa \\\\leq NFk`.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(5)\n >>> nfk(200.)\n >>> states.bowa(-100.,0., 100., 200., 300.)\n >>> states.bowa\n bowa(0.0, 0.0, 100.0, 200.0, 200.0)\n \"\"\"\n if upper is None:\n upper = self.subseqs.seqs.model.parameters.control.nfk\n lland_sequences.State1DSequence.trim(self, lower, upper)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef post(self, request, pk):\n location = self.get_object()\n # open days, disabled widget data won't make it into request.POST\n present_prefixes = [x.split('-')[0] for x in request.POST.keys()]\n day_forms = OrderedDict()\n for day_no, day_name in WEEKDAYS:\n for slot_no in (1, 2):\n prefix = self.form_prefix(day_no, slot_no)\n # skip closed day as it would be invalid form due to no data\n if prefix not in present_prefixes:\n continue\n day_forms[prefix] = (day_no, Slot(request.POST, prefix=prefix))\n\n if all([day_form[1].is_valid() for pre, day_form in day_forms.items()]):\n OpeningHours.objects.filter(company=location).delete()\n for prefix, day_form in day_forms.items():\n day, form = day_form\n opens, shuts = [str_to_time(form.cleaned_data[x])\n for x in ('opens', 'shuts')]\n if opens != shuts:\n OpeningHours(from_hour=opens, to_hour=shuts,\n company=location, weekday=day).save()\n return redirect(request.path_info)", "response": "Clean the data and save opening hours in the database."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninitializes the editing form 1. Build opening_hours, a lookup dictionary to populate the form slots: keys are day numbers, values are lists of opening hours for that day. 2. Build days, a list of days with 2 slot forms each. 3. Build form initials for the 2 slots padding/trimming opening_hours to end up with exactly 2 slots even if it's just None values.", "response": "def get(self, request, pk):\n \"\"\" Initialize the editing form\n\n 1. Build opening_hours, a lookup dictionary to populate the form\n slots: keys are day numbers, values are lists of opening\n hours for that day.\n 2. Build days, a list of days with 2 slot forms each.\n 3. Build form initials for the 2 slots padding/trimming\n opening_hours to end up with exactly 2 slots even if it's\n just None values.\n \"\"\"\n location = self.get_object()\n two_sets = False\n closed = None\n opening_hours = {}\n for o in OpeningHours.objects.filter(company=location):\n opening_hours.setdefault(o.weekday, []).append(o)\n days = []\n for day_no, day_name in WEEKDAYS:\n if day_no not in opening_hours.keys():\n if opening_hours:\n closed = True\n ini1, ini2 = [None, None]\n else:\n closed = False\n ini = [{'opens': time_to_str(oh.from_hour),\n 'shuts': time_to_str(oh.to_hour)}\n for oh in opening_hours[day_no]]\n ini += [None] * (2 - len(ini[:2])) # pad\n ini1, ini2 = ini[:2] # trim\n if ini2:\n two_sets = True\n days.append({\n 'name': day_name,\n 'number': day_no,\n 'slot1': Slot(prefix=self.form_prefix(day_no, 1), initial=ini1),\n 'slot2': Slot(prefix=self.form_prefix(day_no, 2), initial=ini2),\n 'closed': closed\n })\n return render(request, self.template_name, {\n 'days': days,\n 'two_sets': two_sets,\n 'location': location,\n })"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_qjoints_v1(self):\n der = self.parameters.derived.fastaccess\n new = self.sequences.states.fastaccess_new\n old = self.sequences.states.fastaccess_old\n for j in range(der.nmbsegments):\n new.qjoints[j+1] = (der.c1*new.qjoints[j] +\n der.c2*old.qjoints[j] +\n der.c3*old.qjoints[j+1])", "response": "Calculates the routing equation for the current state sequence."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nassigning the actual value of the inlet sequence to the upper joint of the subreach upstream.", "response": "def pick_q_v1(self):\n \"\"\"Assign the actual value of the inlet sequence to the upper joint\n of the subreach upstream.\"\"\"\n inl = self.sequences.inlets.fastaccess\n new = self.sequences.states.fastaccess_new\n new.qjoints[0] = 0.\n for idx in range(inl.len_q):\n new.qjoints[0] += inl.q[idx][0]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pass_q_v1(self):\n der = self.parameters.derived.fastaccess\n new = self.sequences.states.fastaccess_new\n out = self.sequences.outlets.fastaccess\n out.q[0] += new.qjoints[der.nmbsegments]", "response": "Assing the actual value of the lower joint of the subreach\n downstream to the outlet sequence."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntries to detect the encoding of the data. If data is passed try to decode the data with the default system encoding. If data is not passed try to decode the data with the default system encoding.", "response": "def _detect_encoding(data=None):\n \"\"\"Return the default system encoding. If data is passed, try\n to decode the data with the default system encoding or from a short\n list of encoding types to test.\n\n Args:\n data - list of lists\n Returns:\n enc - system encoding\n \"\"\"\n import locale\n enc_list = ['utf-8', 'latin-1', 'iso8859-1', 'iso8859-2',\n 'utf-16', 'cp720']\n code = locale.getpreferredencoding(False)\n if data is None:\n return code\n if code.lower() not in enc_list:\n enc_list.insert(0, code.lower())\n for c in enc_list:\n try:\n for line in data:\n line.decode(c)\n except (UnicodeDecodeError, UnicodeError, AttributeError):\n continue\n return c\n print(\"Encoding not detected. Please pass encoding value manually\")"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef parameterstep(timestep=None):\n if timestep is not None:\n parametertools.Parameter.parameterstep(timestep)\n namespace = inspect.currentframe().f_back.f_locals\n model = namespace.get('model')\n if model is None:\n model = namespace['Model']()\n namespace['model'] = model\n if hydpy.pub.options.usecython and 'cythonizer' in namespace:\n cythonizer = namespace['cythonizer']\n namespace['cythonmodule'] = cythonizer.cymodule\n model.cymodel = cythonizer.cymodule.Model()\n namespace['cymodel'] = model.cymodel\n model.cymodel.parameters = cythonizer.cymodule.Parameters()\n model.cymodel.sequences = cythonizer.cymodule.Sequences()\n for numpars_name in ('NumConsts', 'NumVars'):\n if hasattr(cythonizer.cymodule, numpars_name):\n numpars_new = getattr(cythonizer.cymodule, numpars_name)()\n numpars_old = getattr(model, numpars_name.lower())\n for (name_numpar, numpar) in vars(numpars_old).items():\n setattr(numpars_new, name_numpar, numpar)\n setattr(model.cymodel, numpars_name.lower(), numpars_new)\n for name in dir(model.cymodel):\n if (not name.startswith('_')) and hasattr(model, name):\n setattr(model, name, getattr(model.cymodel, name))\n if 'Parameters' not in namespace:\n namespace['Parameters'] = parametertools.Parameters\n model.parameters = namespace['Parameters'](namespace)\n if 'Sequences' not in namespace:\n namespace['Sequences'] = sequencetools.Sequences\n model.sequences = namespace['Sequences'](**namespace)\n namespace['parameters'] = model.parameters\n for pars in model.parameters:\n namespace[pars.name] = pars\n namespace['sequences'] = model.sequences\n for seqs in model.sequences:\n namespace[seqs.name] = seqs\n if 'Masks' in namespace:\n model.masks = namespace['Masks'](model)\n namespace['masks'] = model.masks\n try:\n namespace.update(namespace['CONSTANTS'])\n except KeyError:\n pass\n focus = namespace.get('focus')\n for par in model.parameters.control:\n try:\n if (focus is None) or (par is focus):\n namespace[par.name] = par\n else:\n namespace[par.name] = lambda *args, **kwargs: None\n except AttributeError:\n pass", "response": "Define a parameter time step size within a parameter control file."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nclears the local namespace from a wildcard import of HydPy - L - Land_v1 model.", "response": "def reverse_model_wildcard_import():\n \"\"\"Clear the local namespace from a model wildcard import.\n\n Calling this method should remove the critical imports into the local\n namespace due the last wildcard import of a certain application model.\n It is thought for securing the successive preperation of different\n types of models via wildcard imports. See the following example, on\n how it can be applied.\n\n >>> from hydpy import reverse_model_wildcard_import\n\n Assume you wildcard import the first version of HydPy-L-Land (|lland_v1|):\n\n >>> from hydpy.models.lland_v1 import *\n\n This for example adds the collection class for handling control\n parameters of `lland_v1` into the local namespace:\n\n >>> print(ControlParameters(None).name)\n control\n\n Calling function |parameterstep| for example prepares the control\n parameter object |lland_control.NHRU|:\n\n >>> parameterstep('1d')\n >>> nhru\n nhru(?)\n\n Calling function |reverse_model_wildcard_import| removes both\n objects (and many more, but not all) from the local namespace:\n\n >>> reverse_model_wildcard_import()\n\n >>> ControlParameters\n Traceback (most recent call last):\n ...\n NameError: name 'ControlParameters' is not defined\n\n >>> nhru\n Traceback (most recent call last):\n ...\n NameError: name 'nhru' is not defined\n \"\"\"\n namespace = inspect.currentframe().f_back.f_locals\n model = namespace.get('model')\n if model is not None:\n for subpars in model.parameters:\n for par in subpars:\n namespace.pop(par.name, None)\n namespace.pop(objecttools.classname(par), None)\n namespace.pop(subpars.name, None)\n namespace.pop(objecttools.classname(subpars), None)\n for subseqs in model.sequences:\n for seq in subseqs:\n namespace.pop(seq.name, None)\n namespace.pop(objecttools.classname(seq), None)\n namespace.pop(subseqs.name, None)\n namespace.pop(objecttools.classname(subseqs), None)\n for name in ('parameters', 'sequences', 'masks', 'model',\n 'Parameters', 'Sequences', 'Masks', 'Model',\n 'cythonizer', 'cymodel', 'cythonmodule'):\n namespace.pop(name, None)\n for key in list(namespace.keys()):\n try:\n if namespace[key].__module__ == model.__module__:\n del namespace[key]\n except AttributeError:\n pass"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef prepare_model(module: Union[types.ModuleType, str],\n timestep: PeriodABC.ConstrArg = None):\n \"\"\"Prepare and return the model of the given module.\n\n In usual HydPy projects, each hydrological model instance is prepared\n in an individual control file. This allows for \"polluting\" the\n namespace with different model attributes. There is no danger of\n name conflicts, as long as no other (wildcard) imports are performed.\n\n However, there are situations when different models are to be loaded\n into the same namespace. Then it is advisable to use function\n |prepare_model|, which just returns a reference to the model\n and nothing else.\n\n See the documentation of |dam_v001| on how to apply function\n |prepare_model| properly.\n \"\"\"\n if timestep is not None:\n parametertools.Parameter.parameterstep(timetools.Period(timestep))\n try:\n model = module.Model()\n except AttributeError:\n module = importlib.import_module(f'hydpy.models.{module}')\n model = module.Model()\n if hydpy.pub.options.usecython and hasattr(module, 'cythonizer'):\n cymodule = module.cythonizer.cymodule\n cymodel = cymodule.Model()\n cymodel.parameters = cymodule.Parameters()\n cymodel.sequences = cymodule.Sequences()\n model.cymodel = cymodel\n for numpars_name in ('NumConsts', 'NumVars'):\n if hasattr(cymodule, numpars_name):\n numpars_new = getattr(cymodule, numpars_name)()\n numpars_old = getattr(model, numpars_name.lower())\n for (name_numpar, numpar) in vars(numpars_old).items():\n setattr(numpars_new, name_numpar, numpar)\n setattr(cymodel, numpars_name.lower(), numpars_new)\n for name in dir(cymodel):\n if (not name.startswith('_')) and hasattr(model, name):\n setattr(model, name, getattr(cymodel, name))\n dict_ = {'cythonmodule': cymodule,\n 'cymodel': cymodel}\n else:\n dict_ = {}\n dict_.update(vars(module))\n dict_['model'] = model\n if hasattr(module, 'Parameters'):\n model.parameters = module.Parameters(dict_)\n else:\n model.parameters = parametertools.Parameters(dict_)\n if hasattr(module, 'Sequences'):\n model.sequences = module.Sequences(**dict_)\n else:\n model.sequences = sequencetools.Sequences(**dict_)\n if hasattr(module, 'Masks'):\n model.masks = module.Masks(model)\n return model", "response": "Prepare and return the model of the given module."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndefining a simulation time step size for testing purposes within a single parameter control file.", "response": "def simulationstep(timestep):\n \"\"\" Define a simulation time step size for testing purposes within a\n parameter control file.\n\n Using |simulationstep| only affects the values of time dependent\n parameters, when `pub.timegrids.stepsize` is not defined. It thus has\n no influence on usual hydpy simulations at all. Use it just to check\n your parameter control files. Write it in a line immediately behind\n the one calling |parameterstep|.\n\n To clarify its purpose, executing raises a warning, when executing\n it from within a control file:\n\n >>> from hydpy import pub\n >>> with pub.options.warnsimulationstep(True):\n ... from hydpy.models.hland_v1 import *\n ... parameterstep('1d')\n ... simulationstep('1h')\n Traceback (most recent call last):\n ...\n UserWarning: Note that the applied function `simulationstep` is intended \\\nfor testing purposes only. When doing a HydPy simulation, parameter values \\\nare initialised based on the actual simulation time step as defined under \\\n`pub.timegrids.stepsize` and the value given to `simulationstep` is ignored.\n >>> k4.simulationstep\n Period('1h')\n \"\"\"\n if hydpy.pub.options.warnsimulationstep:\n warnings.warn(\n 'Note that the applied function `simulationstep` is intended for '\n 'testing purposes only. When doing a HydPy simulation, parameter '\n 'values are initialised based on the actual simulation time step '\n 'as defined under `pub.timegrids.stepsize` and the value given '\n 'to `simulationstep` is ignored.')\n parametertools.Parameter.simulationstep(timestep)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndefining the corresponding control file within a condition file. Function |controlcheck| serves similar purposes as function |parameterstep|. It is the reason why one can interactively access the state and/or the log sequences within condition files as `land_dill.py` of the example project `LahnH`. It is called `controlcheck` due to its implicite feature to check upon the execution of the condition file if eventual specifications within both files disagree. The following test, where we write a number of soil moisture values (|hland_states.SM|) into condition file `land_dill.py` which does not agree with the number of hydrological response units (|hland_control.NmbZones|) defined in control file `land_dill.py`, verifies that this actually works within a new Python process: >>> from hydpy.core.examples import prepare_full_example_1 >>> prepare_full_example_1() >>> import os, subprocess >>> from hydpy import TestIO >>> cwd = os.path.join('LahnH', 'conditions', 'init_1996_01_01') >>> with TestIO(): ... os.chdir(cwd) ... with open('land_dill.py') as file_: ... lines = file_.readlines() ... lines[10:12] = 'sm(185.13164, 181.18755)', '' ... with open('land_dill.py', 'w') as file_: ... _ = file_.write('\\\\n'.join(lines)) ... result = subprocess.run( ... 'python land_dill.py', ... stdout=subprocess.PIPE, ... stderr=subprocess.PIPE, ... universal_newlines=True, ... shell=True) >>> print(result.stderr.split('ValueError:')[-1].strip()) While trying to set the value(s) of variable `sm`, the following error \\ occurred: While trying to convert the value(s) `(185.13164, 181.18755)` to \\ a numpy ndarray with shape `(12,)` and type `float`, the following error \\ occurred: could not broadcast input array from shape (2) into shape (12) With a little trick, we can fake to be \"inside\" condition file `land_dill.py`. Calling |controlcheck| then e.g. prepares the shape of sequence |hland_states.Ic| as specified by the value of parameter |hland_control.NmbZones| given in the corresponding control file: >>> from hydpy.models.hland_v1 import * >>> __file__ = 'land_dill.py' # ToDo: undo? >>> with TestIO(): ... os.chdir(cwd) ... controlcheck() >>> ic.shape (12,) In the above example, the standard names for the project directory (the one containing the executed condition file) and the control directory (`default`) are used. The following example shows how to change them: >>> del model >>> with TestIO(): # doctest: +ELLIPSIS ... os.chdir(cwd) ... controlcheck(projectdir='somewhere', controldir='nowhere') Traceback (most recent call last): ... FileNotFoundError: While trying to load the control file \\ `...hydpy...tests...iotesting...control...nowhere...land_dill.py`, the \\ following error occurred: [Errno 2] No such file or directory: '...land_dill.py' Note that the functionalities of function |controlcheck| are disabled when there is already a `model` variable in the namespace, which is the case when a condition file is executed within the context of a complete HydPy project.", "response": "def controlcheck(controldir='default', projectdir=None, controlfile=None):\n \"\"\"Define the corresponding control file within a condition file.\n\n Function |controlcheck| serves similar purposes as function\n |parameterstep|. It is the reason why one can interactively\n access the state and/or the log sequences within condition files\n as `land_dill.py` of the example project `LahnH`. It is called\n `controlcheck` due to its implicite feature to check upon the execution\n of the condition file if eventual specifications within both files\n disagree. The following test, where we write a number of soil moisture\n values (|hland_states.SM|) into condition file `land_dill.py` which\n does not agree with the number of hydrological response units\n (|hland_control.NmbZones|) defined in control file `land_dill.py`,\n verifies that this actually works within a new Python process:\n\n >>> from hydpy.core.examples import prepare_full_example_1\n >>> prepare_full_example_1()\n\n >>> import os, subprocess\n >>> from hydpy import TestIO\n >>> cwd = os.path.join('LahnH', 'conditions', 'init_1996_01_01')\n >>> with TestIO():\n ... os.chdir(cwd)\n ... with open('land_dill.py') as file_:\n ... lines = file_.readlines()\n ... lines[10:12] = 'sm(185.13164, 181.18755)', ''\n ... with open('land_dill.py', 'w') as file_:\n ... _ = file_.write('\\\\n'.join(lines))\n ... result = subprocess.run(\n ... 'python land_dill.py',\n ... stdout=subprocess.PIPE,\n ... stderr=subprocess.PIPE,\n ... universal_newlines=True,\n ... shell=True)\n >>> print(result.stderr.split('ValueError:')[-1].strip())\n While trying to set the value(s) of variable `sm`, the following error \\\noccurred: While trying to convert the value(s) `(185.13164, 181.18755)` to \\\na numpy ndarray with shape `(12,)` and type `float`, the following error \\\noccurred: could not broadcast input array from shape (2) into shape (12)\n\n With a little trick, we can fake to be \"inside\" condition file\n `land_dill.py`. Calling |controlcheck| then e.g. prepares the shape\n of sequence |hland_states.Ic| as specified by the value of parameter\n |hland_control.NmbZones| given in the corresponding control file:\n\n >>> from hydpy.models.hland_v1 import *\n >>> __file__ = 'land_dill.py' # ToDo: undo?\n >>> with TestIO():\n ... os.chdir(cwd)\n ... controlcheck()\n >>> ic.shape\n (12,)\n\n In the above example, the standard names for the project directory\n (the one containing the executed condition file) and the control\n directory (`default`) are used. The following example shows how\n to change them:\n\n >>> del model\n >>> with TestIO(): # doctest: +ELLIPSIS\n ... os.chdir(cwd)\n ... controlcheck(projectdir='somewhere', controldir='nowhere')\n Traceback (most recent call last):\n ...\n FileNotFoundError: While trying to load the control file \\\n`...hydpy...tests...iotesting...control...nowhere...land_dill.py`, the \\\nfollowing error occurred: [Errno 2] No such file or directory: '...land_dill.py'\n\n Note that the functionalities of function |controlcheck| are disabled\n when there is already a `model` variable in the namespace, which is\n the case when a condition file is executed within the context of a\n complete HydPy project.\n \"\"\"\n namespace = inspect.currentframe().f_back.f_locals\n model = namespace.get('model')\n if model is None:\n if not controlfile:\n controlfile = os.path.split(namespace['__file__'])[-1]\n if projectdir is None:\n projectdir = (\n os.path.split(\n os.path.split(\n os.path.split(os.getcwd())[0])[0])[-1])\n dirpath = os.path.abspath(os.path.join(\n '..', '..', '..', projectdir, 'control', controldir))\n\n class CM(filetools.ControlManager):\n currentpath = dirpath\n\n model = CM().load_file(filename=controlfile)['model']\n model.parameters.update()\n namespace['model'] = model\n for name in ('states', 'logs'):\n subseqs = getattr(model.sequences, name, None)\n if subseqs is not None:\n for seq in subseqs:\n namespace[seq.name] = seq"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update(self):\n con = self.subpars.pars.control\n temp = con.zonearea.values.copy()\n temp[con.zonetype.values == GLACIER] = 0.\n temp[con.zonetype.values == ILAKE] = 0.\n self(numpy.sum(temp)/con.area)", "response": "Update |RelSoilArea| based on |Area Area Zone Area and ZoneType|."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self):\n con = self.subpars.pars.control\n self(con.tt+con.dttm)", "response": "Update |TTM| based on : math : TT + DTTM."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating |UH| based on |MaxBaz| and |QuH|.", "response": "def update(self):\n \"\"\"Update |UH| based on |MaxBaz|.\n\n .. note::\n\n This method also updates the shape of log sequence |QUH|.\n\n |MaxBaz| determines the end point of the triangle. A value of\n |MaxBaz| being not larger than the simulation step size is\n identical with applying no unit hydrograph at all:\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> maxbaz(0.0)\n >>> derived.uh.update()\n >>> logs.quh.shape\n (1,)\n >>> derived.uh\n uh(1.0)\n\n Note that, due to difference of the parameter and the simulation\n step size in the given example, the largest assignment resulting\n in a `inactive` unit hydrograph is 1/2:\n\n >>> maxbaz(0.5)\n >>> derived.uh.update()\n >>> logs.quh.shape\n (1,)\n >>> derived.uh\n uh(1.0)\n\n When |MaxBaz| is in accordance with two simulation steps, both\n unit hydrograph ordinats must be 1/2 due to symmetry of the\n triangle:\n\n >>> maxbaz(1.0)\n >>> derived.uh.update()\n >>> logs.quh.shape\n (2,)\n >>> derived.uh\n uh(0.5)\n >>> derived.uh.values\n array([ 0.5, 0.5])\n\n A |MaxBaz| value in accordance with three simulation steps results\n in the ordinate values 2/9, 5/9, and 2/9:\n\n >>> maxbaz(1.5)\n >>> derived.uh.update()\n >>> logs.quh.shape\n (3,)\n >>> derived.uh\n uh(0.222222, 0.555556, 0.222222)\n\n And a final example, where the end of the triangle lies within\n a simulation step, resulting in the fractions 8/49, 23/49, 16/49,\n and 2/49:\n\n >>> maxbaz(1.75)\n >>> derived.uh.update()\n >>> logs.quh.shape\n (4,)\n >>> derived.uh\n uh(0.163265, 0.469388, 0.326531, 0.040816)\n \"\"\"\n maxbaz = self.subpars.pars.control.maxbaz.value\n quh = self.subpars.pars.model.sequences.logs.quh\n # Determine UH parameters...\n if maxbaz <= 1.:\n # ...when MaxBaz smaller than or equal to the simulation time step.\n self.shape = 1\n self(1.)\n quh.shape = 1\n else:\n # ...when MaxBaz is greater than the simulation time step.\n # Define some shortcuts for the following calculations.\n full = maxbaz\n # Now comes a terrible trick due to rounding problems coming from\n # the conversation of the SMHI parameter set to the HydPy\n # parameter set. Time to get rid of it...\n if (full % 1.) < 1e-4:\n full //= 1.\n full_f = int(numpy.floor(full))\n full_c = int(numpy.ceil(full))\n half = full/2.\n half_f = int(numpy.floor(half))\n half_c = int(numpy.ceil(half))\n full_2 = full**2.\n # Calculate the triangle ordinate(s)...\n self.shape = full_c\n uh = self.values\n quh.shape = full_c\n # ...of the rising limb.\n points = numpy.arange(1, half_f+1)\n uh[:half_f] = (2.*points-1.)/(2.*full_2)\n # ...around the peak (if it exists).\n if numpy.mod(half, 1.) != 0.:\n uh[half_f] = (\n (half_c-half)/full +\n (2*half**2.-half_f**2.-half_c**2.)/(2.*full_2))\n # ...of the falling limb (eventually except the last one).\n points = numpy.arange(half_c+1., full_f+1.)\n uh[half_c:full_f] = 1./full-(2.*points-1.)/(2.*full_2)\n # ...at the end (if not already done).\n if numpy.mod(full, 1.) != 0.:\n uh[full_f] = (\n (full-full_f)/full-(full_2-full_f**2.)/(2.*full_2))\n # Normalize the ordinates.\n self(uh/numpy.sum(uh))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nupdating |QFactor| based on |Area| and the current simulation step size.", "response": "def update(self):\n \"\"\"Update |QFactor| based on |Area| and the current simulation\n step size.\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> area(50.0)\n >>> derived.qfactor.update()\n >>> derived.qfactor\n qfactor(1.157407)\n \"\"\"\n self(self.subpars.pars.control.area*1000. /\n self.subpars.qfactor.simulationstep.seconds)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef nmb_neurons(self) -> Tuple[int, ...]:\n return tuple(numpy.asarray(self._cann.nmb_neurons))", "response": "Number of neurons of the hidden layers."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nshape of the array containing the activation of the hidden neurons.", "response": "def shape_weights_hidden(self) -> Tuple[int, int, int]:\n \"\"\"Shape of the array containing the activation of the hidden neurons.\n\n The first integer value is the number of connection between the\n hidden layers, the second integer value is maximum number of\n neurons of all hidden layers feeding information into another\n hidden layer (all except the last one), and the third integer\n value is the maximum number of the neurons of all hidden layers\n receiving information from another hidden layer (all except the\n first one):\n\n >>> from hydpy import ANN\n >>> ann = ANN(None)\n >>> ann(nmb_inputs=6, nmb_neurons=(4, 3, 2), nmb_outputs=6)\n >>> ann.shape_weights_hidden\n (2, 4, 3)\n >>> ann(nmb_inputs=6, nmb_neurons=(4,), nmb_outputs=6)\n >>> ann.shape_weights_hidden\n (0, 0, 0)\n \"\"\"\n if self.nmb_layers > 1:\n nmb_neurons = self.nmb_neurons\n return (self.nmb_layers-1,\n max(nmb_neurons[:-1]),\n max(nmb_neurons[1:]))\n return 0, 0, 0"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef nmb_weights_hidden(self) -> int:\n nmb = 0\n for idx_layer in range(self.nmb_layers-1):\n nmb += self.nmb_neurons[idx_layer] * self.nmb_neurons[idx_layer+1]\n return nmb", "response": "Number of hidden weights."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nraise a |RuntimeError| if the network s shape is not defined completely.", "response": "def verify(self) -> None:\n \"\"\"Raise a |RuntimeError| if the network's shape is not defined\n completely.\n\n >>> from hydpy import ANN\n >>> ANN(None).verify()\n Traceback (most recent call last):\n ...\n RuntimeError: The shape of the the artificial neural network \\\nparameter `ann` of element `?` has not been defined so far.\n \"\"\"\n if not self.__protectedproperties.allready(self):\n raise RuntimeError(\n 'The shape of the the artificial neural network '\n 'parameter %s has not been defined so far.'\n % objecttools.elementphrase(self))"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef assignrepr(self, prefix) -> str:\n prefix = '%s%s(' % (prefix, self.name)\n blanks = len(prefix)*' '\n lines = [\n objecttools.assignrepr_value(\n self.nmb_inputs, '%snmb_inputs=' % prefix)+',',\n objecttools.assignrepr_tuple(\n self.nmb_neurons, '%snmb_neurons=' % blanks)+',',\n objecttools.assignrepr_value(\n self.nmb_outputs, '%snmb_outputs=' % blanks)+',',\n objecttools.assignrepr_list2(\n self.weights_input, '%sweights_input=' % blanks)+',']\n if self.nmb_layers > 1:\n lines.append(objecttools.assignrepr_list3(\n self.weights_hidden, '%sweights_hidden=' % blanks)+',')\n lines.append(objecttools.assignrepr_list2(\n self.weights_output, '%sweights_output=' % blanks)+',')\n lines.append(objecttools.assignrepr_list2(\n self.intercepts_hidden, '%sintercepts_hidden=' % blanks)+',')\n lines.append(objecttools.assignrepr_list(\n self.intercepts_output, '%sintercepts_output=' % blanks)+')')\n return '\\n'.join(lines)", "response": "Return a string representation of the actual |anntools. ANN| object containing the given string."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef plot(self, xmin, xmax, idx_input=0, idx_output=0, points=100,\n **kwargs) -> None:\n \"\"\"Plot the relationship between a certain input (`idx_input`) and a\n certain output (`idx_output`) variable described by the actual\n |anntools.ANN| object.\n\n Define the lower and the upper bound of the x axis via arguments\n `xmin` and `xmax`. The number of plotting points can be modified\n by argument `points`. Additional `matplotlib` plotting arguments\n can be passed as keyword arguments.\n \"\"\"\n xs_ = numpy.linspace(xmin, xmax, points)\n ys_ = numpy.zeros(xs_.shape)\n for idx, x__ in enumerate(xs_):\n self.inputs[idx_input] = x__\n self.process_actual_input()\n ys_[idx] = self.outputs[idx_output]\n pyplot.plot(xs_, ys_, **kwargs)", "response": "Plots the relationship between a certain input and a certain output."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef refresh(self) -> None:\n # pylint: disable=unsupported-assignment-operation\n if self._do_refresh:\n if self.anns:\n self.__sann = annutils.SeasonalANN(self.anns)\n setattr(self.fastaccess, self.name, self._sann)\n self._set_shape((None, self._sann.nmb_anns))\n if self._sann.nmb_anns > 1:\n self._interp()\n else:\n self._sann.ratios[:, 0] = 1.\n self.verify()\n else:\n self.__sann = None", "response": "Refreshes the internal state of the SeasonalANN object for calculations."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nraises a |RuntimeError| and removes all handled neural networks and SeasonalANN objects from the internal cache.", "response": "def verify(self) -> None:\n \"\"\"Raise a |RuntimeError| and removes all handled neural networks,\n if the they are defined inconsistently.\n\n Dispite all automated safety checks explained in the general\n documentation on class |anntools.SeasonalANN|, it is still possible\n to destroy the inner consistency of a |anntools.SeasonalANN| instance,\n as it stores its |anntools.ANN| objects by reference. This is shown\n by the following example:\n\n >>> from hydpy import SeasonalANN, ann\n >>> seasonalann = SeasonalANN(None)\n >>> seasonalann.simulationstep = '1d'\n >>> jan = ann(nmb_inputs=1, nmb_neurons=(1,), nmb_outputs=1,\n ... weights_input=0.0, weights_output=0.0,\n ... intercepts_hidden=0.0, intercepts_output=1.0)\n >>> seasonalann(_1_1_12=jan)\n >>> jan.nmb_inputs, jan.nmb_outputs = 2, 3\n >>> jan.nmb_inputs, jan.nmb_outputs\n (2, 3)\n >>> seasonalann.nmb_inputs, seasonalann.nmb_outputs\n (1, 1)\n\n Due to the C level implementation of the mathematical core of both\n |anntools.ANN| and |anntools.SeasonalANN| in module |annutils|,\n such an inconsistency might result in a program crash without any\n informative error message. Whenever you are afraid some\n inconsistency might have crept in, and you want to find out if this\n is actually the case, call method |anntools.SeasonalANN.verify|\n explicitly:\n\n >>> seasonalann.verify()\n Traceback (most recent call last):\n ...\n RuntimeError: The number of input and output values of all neural \\\nnetworks contained by a seasonal neural network collection must be \\\nidentical and be known by the containing object. But the seasonal \\\nneural network collection `seasonalann` of element `?` assumes `1` input \\\nand `1` output values, while the network corresponding to the time of \\\nyear `toy_1_1_12_0_0` requires `2` input and `3` output values.\n\n >>> seasonalann\n seasonalann()\n\n >>> seasonalann.verify()\n Traceback (most recent call last):\n ...\n RuntimeError: Seasonal artificial neural network collections need \\\nto handle at least one \"normal\" single neural network, but for the seasonal \\\nneural network `seasonalann` of element `?` none has been defined so far.\n \"\"\"\n if not self.anns:\n self._toy2ann.clear()\n raise RuntimeError(\n 'Seasonal artificial neural network collections need '\n 'to handle at least one \"normal\" single neural network, '\n 'but for the seasonal neural network `%s` of element '\n '`%s` none has been defined so far.'\n % (self.name, objecttools.devicename(self)))\n for toy, ann_ in self:\n ann_.verify()\n if ((self.nmb_inputs != ann_.nmb_inputs) or\n (self.nmb_outputs != ann_.nmb_outputs)):\n self._toy2ann.clear()\n raise RuntimeError(\n 'The number of input and output values of all neural '\n 'networks contained by a seasonal neural network '\n 'collection must be identical and be known by the '\n 'containing object. But the seasonal neural '\n 'network collection `%s` of element `%s` assumes '\n '`%d` input and `%d` output values, while the network '\n 'corresponding to the time of year `%s` requires '\n '`%d` input and `%d` output values.'\n % (self.name, objecttools.devicename(self),\n self.nmb_inputs, self.nmb_outputs,\n toy,\n ann_.nmb_inputs, ann_.nmb_outputs))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef shape(self) -> Tuple[int, ...]:\n return tuple(int(sub) for sub in self.ratios.shape)", "response": "The shape of the array |anntools. SeasonalANN. ratios|."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot(self, xmin, xmax, idx_input=0, idx_output=0, points=100,\n **kwargs) -> None:\n \"\"\"Call method |anntools.ANN.plot| of all |anntools.ANN| objects\n handled by the actual |anntools.SeasonalANN| object.\n \"\"\"\n for toy, ann_ in self:\n ann_.plot(xmin, xmax,\n idx_input=idx_input, idx_output=idx_output,\n points=points,\n label=str(toy),\n **kwargs)\n pyplot.legend()", "response": "Plot the |ANN| object with the current |LEA| object."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef collect_variables(self, selections) -> None:\n self.insert_variables(self.device2target, self.targetspecs, selections)", "response": "This method is used to collect the relevant target variables handled by the given |Selections| object."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninserting the relevant target or base variables into the given device2variable dictionary.", "response": "def insert_variables(\n self, device2variable, exchangespec, selections) -> None:\n \"\"\"Determine the relevant target or base variables (as defined by\n the given |ExchangeSpecification| object ) handled by the given\n |Selections| object and insert them into the given `device2variable`\n dictionary.\"\"\"\n if self.targetspecs.master in ('node', 'nodes'):\n for node in selections.nodes:\n variable = self._query_nodevariable(node, exchangespec)\n device2variable[node] = variable\n else:\n for element in self._iter_relevantelements(selections):\n variable = self._query_elementvariable(element, exchangespec)\n device2variable[element] = variable"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nassigns the given value to the given target or base variable.", "response": "def update_variable(self, variable, value) -> None:\n \"\"\"Assign the given value(s) to the given target or base variable.\n\n If the assignment fails, |ChangeItem.update_variable| raises an\n error like the following:\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, TestIO = prepare_full_example_2()\n >>> item = SetItem('alpha', 'hland_v1', 'control.alpha', 0)\n >>> item.collect_variables(pub.selections)\n >>> item.update_variables() # doctest: +ELLIPSIS\n Traceback (most recent call last):\n ...\n TypeError: When trying to update a target variable of SetItem `alpha` \\\nwith the value(s) `None`, the following error occurred: While trying to set \\\nthe value(s) of variable `alpha` of element `...`, the following error \\\noccurred: The given value `None` cannot be converted to type `float`.\n \"\"\"\n try:\n variable(value)\n except BaseException:\n objecttools.augment_excmessage(\n f'When trying to update a target variable of '\n f'{objecttools.classname(self)} `{self.name}` '\n f'with the value(s) `{value}`')"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_variables(self) -> None:\n value = self.value\n for variable in self.device2target.values():\n self.update_variable(variable, value)", "response": "Assign the current values of the target variables to the values\n of the target variables."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef collect_variables(self, selections) -> None:\n super().collect_variables(selections)\n self.insert_variables(self.device2base, self.basespecs, selections)", "response": "Collect the base variables of the given |Selections| object and insert them into the base class."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds the general base base", "response": "def update_variables(self) -> None:\n \"\"\"Add the general |ChangeItem.value| with the |Device| specific base\n variable and assign the result to the respective target variable.\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, TestIO = prepare_full_example_2()\n >>> from hydpy.models.hland_v1 import FIELD\n >>> for element in hp.elements.catchment:\n ... control = element.model.parameters.control\n ... control.nmbzones(3)\n ... control.zonetype(FIELD)\n ... control.rfcf(1.1)\n >>> from hydpy.core.itemtools import AddItem\n >>> item = AddItem(\n ... 'sfcf', 'hland_v1', 'control.sfcf', 'control.rfcf', 1)\n >>> item.collect_variables(pub.selections)\n >>> land_dill = hp.elements.land_dill\n >>> land_dill.model.parameters.control.sfcf\n sfcf(?)\n >>> item.value = -0.1, 0.0, 0.1\n >>> item.update_variables()\n >>> land_dill.model.parameters.control.sfcf\n sfcf(1.0, 1.1, 1.2)\n\n >>> land_dill.model.parameters.control.rfcf.shape = 2\n >>> land_dill.model.parameters.control.rfcf = 1.1\n >>> item.update_variables() # doctest: +ELLIPSIS\n Traceback (most recent call last):\n ...\n ValueError: When trying to add the value(s) `[-0.1 0. 0.1]` of \\\nAddItem `sfcf` and the value(s) `[ 1.1 1.1]` of variable `rfcf` of element \\\n`land_dill`, the following error occurred: operands could not be broadcast \\\ntogether with shapes (2,) (3,)...\n \"\"\"\n value = self.value\n for device, target in self.device2target.items():\n base = self.device2base[device]\n try:\n result = base.value + value\n except BaseException:\n raise objecttools.augment_excmessage(\n f'When trying to add the value(s) `{value}` of '\n f'AddItem `{self.name}` and the value(s) `{base.value}` '\n f'of variable {objecttools.devicephrase(base)}')\n self.update_variable(target, result)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\napplying method |ExchangeItem. collect_variables| of the base class and determine the number of variables that are used in the current object.", "response": "def collect_variables(self, selections) -> None:\n \"\"\"Apply method |ExchangeItem.collect_variables| of the base class\n |ExchangeItem| and determine the `ndim` attribute of the current\n |ChangeItem| object afterwards.\n\n The value of `ndim` depends on whether the values of the target\n variable or its time series is of interest:\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, TestIO = prepare_full_example_2()\n >>> from hydpy.core.itemtools import SetItem\n >>> for target in ('states.lz', 'states.lz.series',\n ... 'states.sm', 'states.sm.series'):\n ... item = GetItem('hland_v1', target)\n ... item.collect_variables(pub.selections)\n ... print(item, item.ndim)\n GetItem('hland_v1', 'states.lz') 0\n GetItem('hland_v1', 'states.lz.series') 1\n GetItem('hland_v1', 'states.sm') 1\n GetItem('hland_v1', 'states.sm.series') 2\n \"\"\"\n super().collect_variables(selections)\n for device in sorted(self.device2target.keys(), key=lambda x: x.name):\n self._device2name[device] = f'{device.name}_{self.target}'\n for target in self.device2target.values():\n self.ndim = target.NDIM\n if self.targetspecs.series:\n self.ndim += 1\n break"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn False if the location is closed or the OpeningHours object containing the specified attribute is currently open.", "response": "def is_open(location=None, attr=None):\n \"\"\"\n Returns False if the location is closed, or the OpeningHours object\n to show the location is currently open.\n \"\"\"\n obj = utils.is_open(location)\n if obj is False:\n return False\n if attr is not None:\n return getattr(obj, attr)\n return obj"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning True if the location is currently open.", "response": "def is_open_now(location=None, attr=None):\n \"\"\"\n Returns False if the location is closed, or the OpeningHours object\n to show the location is currently open.\n Same as `is_open` but passes `now` to `utils.is_open` to bypass `get_now()`. \n \"\"\"\n obj = utils.is_open(location, now=datetime.datetime.now())\n if obj is False:\n return False\n if attr is not None:\n return getattr(obj, attr)\n return obj"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef opening_hours(location=None, concise=False):\n template_name = 'openinghours/opening_hours_list.html'\n days = [] # [{'hours': '9:00am to 5:00pm', 'name': u'Monday'}, {'hours...\n\n # Without `location`, choose the first company.\n if location:\n ohrs = OpeningHours.objects.filter(company=location)\n else:\n try:\n Location = utils.get_premises_model()\n ohrs = Location.objects.first().openinghours_set.all()\n except AttributeError:\n raise Exception(\"You must define some opening hours\"\n \" to use the opening hours tags.\")\n\n ohrs.order_by('weekday', 'from_hour')\n\n for o in ohrs:\n days.append({\n 'day_number': o.weekday,\n 'name': o.get_weekday_display(),\n 'from_hour': o.from_hour,\n 'to_hour': o.to_hour,\n 'hours': '%s%s to %s%s' % (\n o.from_hour.strftime('%I:%M').lstrip('0'),\n o.from_hour.strftime('%p').lower(),\n o.to_hour.strftime('%I:%M').lstrip('0'),\n o.to_hour.strftime('%p').lower()\n )\n })\n\n open_days = [o.weekday for o in ohrs]\n for day_number, day_name in WEEKDAYS:\n if day_number not in open_days:\n days.append({\n 'day_number': day_number,\n 'name': day_name,\n 'hours': 'Closed'\n })\n days = sorted(days, key=lambda k: k['day_number'])\n\n if concise:\n # [{'hours': '9:00am to 5:00pm', 'day_names': u'Monday to Friday'},\n # {'hours':...\n template_name = 'openinghours/opening_hours_list_concise.html'\n concise_days = []\n current_set = {}\n for day in days:\n if 'hours' not in current_set.keys():\n current_set = {'day_names': [day['name']],\n 'hours': day['hours']}\n elif day['hours'] != current_set['hours']:\n concise_days.append(current_set)\n current_set = {'day_names': [day['name']],\n 'hours': day['hours']}\n else:\n current_set['day_names'].append(day['name'])\n concise_days.append(current_set)\n\n for day_set in concise_days:\n if len(day_set['day_names']) > 2:\n day_set['day_names'] = '%s to %s' % (day_set['day_names'][0],\n day_set['day_names'][-1])\n elif len(day_set['day_names']) > 1:\n day_set['day_names'] = '%s and %s' % (day_set['day_names'][0],\n day_set['day_names'][-1])\n else:\n day_set['day_names'] = '%s' % day_set['day_names'][0]\n\n days = concise_days\n\n template = get_template(template_name)\n return template.render({'days': days})", "response": "Creates a rendered listing of opening hours."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef prepare_everything(self):\n self.prepare_network()\n self.init_models()\n self.load_conditions()\n with hydpy.pub.options.warnmissingobsfile(False):\n self.prepare_nodeseries()\n self.prepare_modelseries()\n self.load_inputseries()", "response": "Convenience method to make the actual |HydPy| instance runable."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nloading all network files and assign the complete selection to the HydPy object.", "response": "def prepare_network(self):\n \"\"\"Load all network files as |Selections| (stored in module |pub|)\n and assign the \"complete\" selection to the |HydPy| object.\"\"\"\n hydpy.pub.selections = selectiontools.Selections()\n hydpy.pub.selections += hydpy.pub.networkmanager.load_files()\n self.update_devices(hydpy.pub.selections.complete)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncall method |Elements.save_controls| of the |Elements| object currently handled by the |HydPy| object. We use the `LahnH` example project to demonstrate how to write a complete set parameter control files. For convenience, we let function |prepare_full_example_2| prepare a fully functional |HydPy| object, handling seven |Element| objects controlling four |hland_v1| and three |hstream_v1| application models: >>> from hydpy.core.examples import prepare_full_example_2 >>> hp, pub, TestIO = prepare_full_example_2() At first, there is only one control subfolder named \"default\", containing the seven control files used in the step above: >>> import os >>> with TestIO(): ... os.listdir('LahnH/control') ['default'] Next, we use the |ControlManager| to create a new directory and dump all control file into it: >>> with TestIO(): ... pub.controlmanager.currentdir = 'newdir' ... hp.save_controls() ... sorted(os.listdir('LahnH/control')) ['default', 'newdir'] We focus our examples on the (smaller) control files of application model |hstream_v1|. The values of parameter |hstream_control.Lag| and |hstream_control.Damp| for the river channel connecting the outlets of subcatchment `lahn_1` and `lahn_2` are 0.583 days and 0.0, respectively: >>> model = hp.elements.stream_lahn_1_lahn_2.model >>> model.parameters.control lag(0.583) damp(0.0) The corresponding written control file defines the same values: >>> dir_ = 'LahnH/control/newdir/' >>> with TestIO(): ... with open(dir_ + 'stream_lahn_1_lahn_2.py') as controlfile: ... print(controlfile.read()) # -*- coding: utf-8 -*- from hydpy.models.hstream_v1 import * simulationstep('1d') parameterstep('1d') lag(0.583) damp(0.0) Its name equals the element name and the time step information is taken for the |Timegrid| object available via |pub|: >>> pub.timegrids.stepsize Period('1d') Use the |Auxfiler| class To avoid redefining the same parameter values in multiple control files. Here, we prepare an |Auxfiler| object which handles the two parameters of the model discussed above: >>> from hydpy import Auxfiler >>> aux = Auxfiler() >>> aux += 'hstream_v1' >>> aux.hstream_v1.stream = model.parameters.control.damp >>> aux.hstream_v1.stream = model.parameters.control.lag When passing the |Auxfiler| object to |HydPy.save_controls|, both parameters the control file of element `stream_lahn_1_lahn_2` do not define their values on their own, but reference the auxiliary file `stream.py` instead: >>> with TestIO(): ... pub.controlmanager.currentdir = 'newdir' ... hp.save_controls(auxfiler=aux) ... with open(dir_ + 'stream_lahn_1_lahn_2.py') as controlfile: ... print(controlfile.read()) # -*- coding: utf-8 -*- from hydpy.models.hstream_v1 import * simulationstep('1d') parameterstep('1d') lag(auxfile='stream') damp(auxfile='stream') `stream.py` contains the actual value definitions: >>> with TestIO(): ... with open(dir_ + 'stream.py') as controlfile: ... print(controlfile.read()) # -*- coding: utf-8 -*- from hydpy.models.hstream_v1 import * simulationstep('1d') parameterstep('1d') damp(0.0) lag(0.583) The |hstream_v1| model of element `stream_lahn_2_lahn_3` defines the same value for parameter |hstream_control.Damp| but a different one for parameter |hstream_control.Lag|. Hence, only |hstream_control.Damp| can reference control file `stream.py` without distorting data: >>> with TestIO(): ... with open(dir_ + 'stream_lahn_2_lahn_3.py') as controlfile: ... print(controlfile.read()) # -*- coding: utf-8 -*- from hydpy.models.hstream_v1 import * simulationstep('1d') parameterstep('1d') lag(0.417) damp(auxfile='stream') Another option is to pass alternative step size information. The `simulationstep` information, which is not really required in control files but useful for testing them, has no impact on the written data. However, passing an alternative `parameterstep` information changes the written values of time dependent parameters both in the primary and the auxiliary control files, as to be expected: >>> with TestIO(): ... pub.controlmanager.currentdir = 'newdir' ... hp.save_controls( ... auxfiler=aux, parameterstep='2d', simulationstep='1h') ... with open(dir_ + 'stream_lahn_1_lahn_2.py') as controlfile: ... print(controlfile.read()) # -*- coding: utf-8 -*- from hydpy.models.hstream_v1 import * simulationstep('1h') parameterstep('2d') lag(auxfile='stream') damp(auxfile='stream') >>> with TestIO(): ... with open(dir_ + 'stream.py') as controlfile: ... print(controlfile.read()) # -*- coding: utf-8 -*- from hydpy.models.hstream_v1 import * simulationstep('1h') parameterstep('2d') damp(0.0) lag(0.2915) >>> with TestIO(): ... with open(dir_ + 'stream_lahn_2_lahn_3.py') as controlfile: ... print(controlfile.read()) # -*- coding: utf-8 -*- from hydpy.models.hstream_v1 import * simulationstep('1h') parameterstep('2d') lag(0.2085) damp(auxfile='stream') ", "response": "def save_controls(self, parameterstep=None, simulationstep=None,\n auxfiler=None):\n \"\"\"Call method |Elements.save_controls| of the |Elements| object\n currently handled by the |HydPy| object.\n\n We use the `LahnH` example project to demonstrate how to write\n a complete set parameter control files. For convenience, we let\n function |prepare_full_example_2| prepare a fully functional\n |HydPy| object, handling seven |Element| objects controlling\n four |hland_v1| and three |hstream_v1| application models:\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, TestIO = prepare_full_example_2()\n\n At first, there is only one control subfolder named \"default\",\n containing the seven control files used in the step above:\n\n >>> import os\n >>> with TestIO():\n ... os.listdir('LahnH/control')\n ['default']\n\n Next, we use the |ControlManager| to create a new directory\n and dump all control file into it:\n\n >>> with TestIO():\n ... pub.controlmanager.currentdir = 'newdir'\n ... hp.save_controls()\n ... sorted(os.listdir('LahnH/control'))\n ['default', 'newdir']\n\n We focus our examples on the (smaller) control files of\n application model |hstream_v1|. The values of parameter\n |hstream_control.Lag| and |hstream_control.Damp| for the\n river channel connecting the outlets of subcatchment `lahn_1`\n and `lahn_2` are 0.583 days and 0.0, respectively:\n\n >>> model = hp.elements.stream_lahn_1_lahn_2.model\n >>> model.parameters.control\n lag(0.583)\n damp(0.0)\n\n The corresponding written control file defines the same values:\n\n >>> dir_ = 'LahnH/control/newdir/'\n >>> with TestIO():\n ... with open(dir_ + 'stream_lahn_1_lahn_2.py') as controlfile:\n ... print(controlfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1d')\n parameterstep('1d')\n \n lag(0.583)\n damp(0.0)\n \n\n Its name equals the element name and the time step information\n is taken for the |Timegrid| object available via |pub|:\n\n >>> pub.timegrids.stepsize\n Period('1d')\n\n Use the |Auxfiler| class To avoid redefining the same parameter\n values in multiple control files. Here, we prepare an |Auxfiler|\n object which handles the two parameters of the model discussed\n above:\n\n >>> from hydpy import Auxfiler\n >>> aux = Auxfiler()\n >>> aux += 'hstream_v1'\n >>> aux.hstream_v1.stream = model.parameters.control.damp\n >>> aux.hstream_v1.stream = model.parameters.control.lag\n\n When passing the |Auxfiler| object to |HydPy.save_controls|,\n both parameters the control file of element `stream_lahn_1_lahn_2`\n do not define their values on their own, but reference the\n auxiliary file `stream.py` instead:\n\n >>> with TestIO():\n ... pub.controlmanager.currentdir = 'newdir'\n ... hp.save_controls(auxfiler=aux)\n ... with open(dir_ + 'stream_lahn_1_lahn_2.py') as controlfile:\n ... print(controlfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1d')\n parameterstep('1d')\n \n lag(auxfile='stream')\n damp(auxfile='stream')\n \n\n `stream.py` contains the actual value definitions:\n\n >>> with TestIO():\n ... with open(dir_ + 'stream.py') as controlfile:\n ... print(controlfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1d')\n parameterstep('1d')\n \n damp(0.0)\n lag(0.583)\n \n\n The |hstream_v1| model of element `stream_lahn_2_lahn_3` defines\n the same value for parameter |hstream_control.Damp| but a different\n one for parameter |hstream_control.Lag|. Hence, only\n |hstream_control.Damp| can reference control file `stream.py`\n without distorting data:\n\n >>> with TestIO():\n ... with open(dir_ + 'stream_lahn_2_lahn_3.py') as controlfile:\n ... print(controlfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1d')\n parameterstep('1d')\n \n lag(0.417)\n damp(auxfile='stream')\n \n\n Another option is to pass alternative step size information.\n The `simulationstep` information, which is not really required\n in control files but useful for testing them, has no impact\n on the written data. However, passing an alternative\n `parameterstep` information changes the written values of\n time dependent parameters both in the primary and the auxiliary\n control files, as to be expected:\n\n >>> with TestIO():\n ... pub.controlmanager.currentdir = 'newdir'\n ... hp.save_controls(\n ... auxfiler=aux, parameterstep='2d', simulationstep='1h')\n ... with open(dir_ + 'stream_lahn_1_lahn_2.py') as controlfile:\n ... print(controlfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1h')\n parameterstep('2d')\n \n lag(auxfile='stream')\n damp(auxfile='stream')\n \n\n >>> with TestIO():\n ... with open(dir_ + 'stream.py') as controlfile:\n ... print(controlfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1h')\n parameterstep('2d')\n \n damp(0.0)\n lag(0.2915)\n \n\n >>> with TestIO():\n ... with open(dir_ + 'stream_lahn_2_lahn_3.py') as controlfile:\n ... print(controlfile.read())\n # -*- coding: utf-8 -*-\n \n from hydpy.models.hstream_v1 import *\n \n simulationstep('1h')\n parameterstep('2d')\n \n lag(0.2085)\n damp(auxfile='stream')\n \n \"\"\"\n self.elements.save_controls(parameterstep=parameterstep,\n simulationstep=simulationstep,\n auxfiler=auxfiler)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef networkproperties(self):\n print('Number of nodes: %d' % len(self.nodes))\n print('Number of elements: %d' % len(self.elements))\n print('Number of end nodes: %d' % len(self.endnodes))\n print('Number of distinct networks: %d' % len(self.numberofnetworks))\n print('Applied node variables: %s' % ', '.join(self.variables))", "response": "Print out some properties of the network defined by the Node and Element objects currently handled by the HydPy object."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef endnodes(self):\n endnodes = devicetools.Nodes()\n for node in self.nodes:\n for element in node.exits:\n if ((element in self.elements) and\n (node not in element.receivers)):\n break\n else:\n endnodes += node\n return endnodes", "response": "|Nodes| object containing all |Node| objects currently handled by\n the HydPy object which define a downstream end point of a network."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a sorted list of strings summarizing all variables handled by the |Node| objects.", "response": "def variables(self):\n \"\"\"Sorted list of strings summarizing all variables handled by the\n |Node| objects\"\"\"\n variables = set([])\n for node in self.nodes:\n variables.add(node.variable)\n return sorted(variables)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef simindices(self):\n return (hydpy.pub.timegrids.init[hydpy.pub.timegrids.sim.firstdate],\n hydpy.pub.timegrids.init[hydpy.pub.timegrids.sim.lastdate])", "response": "Tuple containing the start and end indices of the simulation period\n regarding the initialization period defined by the object\n stored in module |pub|."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open_files(self, idx=0):\n self.elements.open_files(idx=idx)\n self.nodes.open_files(idx=idx)", "response": "Call method |Devices. open_files| of the Nodes and Elements objects currently handled by the HydPy object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating the order of the |Node| and |Element| objects currently handled by the HydPy| objects in the simulation time step.", "response": "def update_devices(self, selection=None):\n \"\"\"Determines the order, in which the |Node| and |Element| objects\n currently handled by the |HydPy| objects need to be processed during\n a simulation time step. Optionally, a |Selection| object for defining\n new |Node| and |Element| objects can be passed.\"\"\"\n if selection is not None:\n self.nodes = selection.nodes\n self.elements = selection.elements\n self._update_deviceorder()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nperform a simulation run over the actual simulation time period defined by the Timegrids object stored in module |pub|.", "response": "def doit(self):\n \"\"\"Perform a simulation run over the actual simulation time period\n defined by the |Timegrids| object stored in module |pub|.\"\"\"\n idx_start, idx_end = self.simindices\n self.open_files(idx_start)\n methodorder = self.methodorder\n for idx in printtools.progressbar(range(idx_start, idx_end)):\n for func in methodorder:\n func(idx)\n self.close_files()"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nupdate the inlet link sequence.", "response": "def pic_inflow_v1(self):\n \"\"\"Update the inlet link sequence.\n\n Required inlet sequence:\n |dam_inlets.Q|\n\n Calculated flux sequence:\n |Inflow|\n\n Basic equation:\n :math:`Inflow = Q`\n \"\"\"\n flu = self.sequences.fluxes.fastaccess\n inl = self.sequences.inlets.fastaccess\n flu.inflow = inl.q[0]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pic_inflow_v2(self):\n flu = self.sequences.fluxes.fastaccess\n inl = self.sequences.inlets.fastaccess\n flu.inflow = inl.q[0]+inl.s[0]+inl.r[0]", "response": "Update the inlet link sequences."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pic_totalremotedischarge_v1(self):\n flu = self.sequences.fluxes.fastaccess\n rec = self.sequences.receivers.fastaccess\n flu.totalremotedischarge = rec.q[0]", "response": "Update the total discharge of the receiver link sequence."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pic_loggedrequiredremoterelease_v1(self):\n log = self.sequences.logs.fastaccess\n rec = self.sequences.receivers.fastaccess\n log.loggedrequiredremoterelease[0] = rec.d[0]", "response": "Update the log of the receiver link sequence."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates the log of the receiver link sequence.", "response": "def pic_loggedrequiredremoterelease_v2(self):\n \"\"\"Update the receiver link sequence.\"\"\"\n log = self.sequences.logs.fastaccess\n rec = self.sequences.receivers.fastaccess\n log.loggedrequiredremoterelease[0] = rec.s[0]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nupdate the log of the first receiver link sequence.", "response": "def pic_loggedallowedremoterelieve_v1(self):\n \"\"\"Update the receiver link sequence.\"\"\"\n log = self.sequences.logs.fastaccess\n rec = self.sequences.receivers.fastaccess\n log.loggedallowedremoterelieve[0] = rec.r[0]"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the logged total remote discharge at a cross section far downstream.", "response": "def update_loggedtotalremotedischarge_v1(self):\n \"\"\"Log a new entry of discharge at a cross section far downstream.\n\n Required control parameter:\n |NmbLogEntries|\n\n Required flux sequence:\n |TotalRemoteDischarge|\n\n Calculated flux sequence:\n |LoggedTotalRemoteDischarge|\n\n Example:\n\n The following example shows that, with each new method call, the\n three memorized values are successively moved to the right and the\n respective new value is stored on the bare left position:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> nmblogentries(3)\n >>> logs.loggedtotalremotedischarge = 0.0\n >>> from hydpy import UnitTest\n >>> test = UnitTest(model, model.update_loggedtotalremotedischarge_v1,\n ... last_example=4,\n ... parseqs=(fluxes.totalremotedischarge,\n ... logs.loggedtotalremotedischarge))\n >>> test.nexts.totalremotedischarge = [1., 3., 2., 4]\n >>> del test.inits.loggedtotalremotedischarge\n >>> test()\n | ex. | totalremotedischarge | loggedtotalremotedischarge |\n ---------------------------------------------------------------------\n | 1 | 1.0 | 1.0 0.0 0.0 |\n | 2 | 3.0 | 3.0 1.0 0.0 |\n | 3 | 2.0 | 2.0 3.0 1.0 |\n | 4 | 4.0 | 4.0 2.0 3.0 |\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n for idx in range(con.nmblogentries-1, 0, -1):\n log.loggedtotalremotedischarge[idx] = \\\n log.loggedtotalremotedischarge[idx-1]\n log.loggedtotalremotedischarge[0] = flu.totalremotedischarge"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the water level based on an artificial neural network and aide.", "response": "def calc_waterlevel_v1(self):\n \"\"\"Determine the water level based on an artificial neural network\n describing the relationship between water level and water stage.\n\n Required control parameter:\n |WaterVolume2WaterLevel|\n\n Required state sequence:\n |WaterVolume|\n\n Calculated aide sequence:\n |WaterLevel|\n\n Example:\n\n Prepare a dam model:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n\n Prepare a very simple relationship based on one single neuron:\n\n >>> watervolume2waterlevel(\n ... nmb_inputs=1, nmb_neurons=(1,), nmb_outputs=1,\n ... weights_input=0.5, weights_output=1.0,\n ... intercepts_hidden=0.0, intercepts_output=-0.5)\n\n At least in the water volume range used in the following examples,\n the shape of the relationship looks acceptable:\n\n >>> from hydpy import UnitTest\n >>> test = UnitTest(\n ... model, model.calc_waterlevel_v1,\n ... last_example=10,\n ... parseqs=(states.watervolume, aides.waterlevel))\n >>> test.nexts.watervolume = range(10)\n >>> test()\n | ex. | watervolume | waterlevel |\n ----------------------------------\n | 1 | 0.0 | 0.0 |\n | 2 | 1.0 | 0.122459 |\n | 3 | 2.0 | 0.231059 |\n | 4 | 3.0 | 0.317574 |\n | 5 | 4.0 | 0.380797 |\n | 6 | 5.0 | 0.424142 |\n | 7 | 6.0 | 0.452574 |\n | 8 | 7.0 | 0.470688 |\n | 9 | 8.0 | 0.482014 |\n | 10 | 9.0 | 0.489013 |\n\n For more realistic approximations of measured relationships between\n water level and volume, larger neural networks are required.\n \"\"\"\n con = self.parameters.control.fastaccess\n new = self.sequences.states.fastaccess_new\n aid = self.sequences.aides.fastaccess\n con.watervolume2waterlevel.inputs[0] = new.watervolume\n con.watervolume2waterlevel.process_actual_input()\n aid.waterlevel = con.watervolume2waterlevel.outputs[0]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the allowed maximum relieve another location in the dam.", "response": "def calc_allowedremoterelieve_v2(self):\n \"\"\"Calculate the allowed maximum relieve another location\n is allowed to discharge into the dam.\n\n Required control parameters:\n |HighestRemoteRelieve|\n |WaterLevelRelieveThreshold|\n\n Required derived parameter:\n |WaterLevelRelieveSmoothPar|\n\n Required aide sequence:\n |WaterLevel|\n\n Calculated flux sequence:\n |AllowedRemoteRelieve|\n\n Basic equation:\n :math:`ActualRemoteRelease = HighestRemoteRelease \\\\cdot\n smooth_{logistic1}(WaterLevelRelieveThreshold-WaterLevel,\n WaterLevelRelieveSmoothPar)`\n\n Used auxiliary method:\n |smooth_logistic1|\n\n Examples:\n\n All control parameters that are involved in the calculation of\n |AllowedRemoteRelieve| are derived from |SeasonalParameter|.\n This allows to simulate seasonal dam control schemes.\n To show how this works, we first define a short simulation\n time period of only two days:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2001.03.30', '2001.04.03', '1d'\n\n Now we prepare the dam model and define two different control\n schemes for the hydrological summer (April to October) and\n winter month (November to May)\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> highestremoterelieve(_11_1_12=1.0, _03_31_12=1.0,\n ... _04_1_12=2.0, _10_31_12=2.0)\n >>> waterlevelrelievethreshold(_11_1_12=3.0, _03_31_12=2.0,\n ... _04_1_12=4.0, _10_31_12=4.0)\n >>> waterlevelrelievetolerance(_11_1_12=0.0, _03_31_12=0.0,\n ... _04_1_12=1.0, _10_31_12=1.0)\n >>> derived.waterlevelrelievesmoothpar.update()\n >>> derived.toy.update()\n\n The following test function is supposed to calculate\n |AllowedRemoteRelieve| for values of |WaterLevel| ranging\n from 0 and 8 m:\n\n >>> from hydpy import UnitTest\n >>> test = UnitTest(model,\n ... model.calc_allowedremoterelieve_v2,\n ... last_example=9,\n ... parseqs=(aides.waterlevel,\n ... fluxes.allowedremoterelieve))\n >>> test.nexts.waterlevel = range(9)\n\n On March 30 (which is the last day of the winter month and the\n first day of the simulation period), the value of\n |WaterLevelRelieveSmoothPar| is zero. Hence, |AllowedRemoteRelieve|\n drops abruptly from 1 m\u00b3/s (the value of |HighestRemoteRelieve|) to\n 0 m\u00b3/s, as soon as |WaterLevel| reaches 3 m (the value\n of |WaterLevelRelieveThreshold|):\n\n >>> model.idx_sim = pub.timegrids.init['2001.03.30']\n >>> test(first_example=2, last_example=6)\n | ex. | waterlevel | allowedremoterelieve |\n -------------------------------------------\n | 3 | 1.0 | 1.0 |\n | 4 | 2.0 | 1.0 |\n | 5 | 3.0 | 0.0 |\n | 6 | 4.0 | 0.0 |\n\n On April 1 (which is the first day of the sommer month and the\n last day of the simulation period), all parameter values are\n increased. The value of parameter |WaterLevelRelieveSmoothPar|\n is 1 m. Hence, loosely speaking, |AllowedRemoteRelieve| approaches\n the \"discontinuous extremes (2 m\u00b3/s -- which is the value of\n |HighestRemoteRelieve| -- and 0 m\u00b3/s) to 99 % within a span of\n 2 m\u00b3/s around the original threshold value of 4 m\u00b3/s defined by\n |WaterLevelRelieveThreshold|:\n\n >>> model.idx_sim = pub.timegrids.init['2001.04.01']\n >>> test()\n | ex. | waterlevel | allowedremoterelieve |\n -------------------------------------------\n | 1 | 0.0 | 2.0 |\n | 2 | 1.0 | 1.999998 |\n | 3 | 2.0 | 1.999796 |\n | 4 | 3.0 | 1.98 |\n | 5 | 4.0 | 1.0 |\n | 6 | 5.0 | 0.02 |\n | 7 | 6.0 | 0.000204 |\n | 8 | 7.0 | 0.000002 |\n | 9 | 8.0 | 0.0 |\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n toy = der.toy[self.idx_sim]\n flu.allowedremoterelieve = (\n con.highestremoterelieve[toy] *\n smoothutils.smooth_logistic1(\n con.waterlevelrelievethreshold[toy]-aid.waterlevel,\n der.waterlevelrelievesmoothpar[toy]))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_requiredremotesupply_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n toy = der.toy[self.idx_sim]\n flu.requiredremotesupply = (\n con.highestremotesupply[toy] *\n smoothutils.smooth_logistic1(\n con.waterlevelsupplythreshold[toy]-aid.waterlevel,\n der.waterlevelsupplysmoothpar[toy]))", "response": "Calculates the required maximum supply from another location\n that can be discharged into the dam."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_naturalremotedischarge_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n flu.naturalremotedischarge = 0.\n for idx in range(con.nmblogentries):\n flu.naturalremotedischarge += (\n log.loggedtotalremotedischarge[idx] - log.loggedoutflow[idx])\n if flu.naturalremotedischarge > 0.:\n flu.naturalremotedischarge /= con.nmblogentries\n else:\n flu.naturalremotedischarge = 0.", "response": "Estimate the natural discharge of a cross section far downstream\n ."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the remote discharge demand of a cross section far downstream.", "response": "def calc_remotedemand_v1(self):\n \"\"\"Estimate the discharge demand of a cross section far downstream.\n\n Required control parameter:\n |RemoteDischargeMinimum|\n\n Required derived parameters:\n |dam_derived.TOY|\n\n Required flux sequence:\n |dam_derived.TOY|\n\n Calculated flux sequence:\n |RemoteDemand|\n\n Basic equation:\n :math:`RemoteDemand =\n max(RemoteDischargeMinimum - NaturalRemoteDischarge, 0`\n\n Examples:\n\n Low water elevation is often restricted to specific month of the year.\n Sometimes the pursued lowest discharge value varies over the year\n to allow for a low flow variability that is in some agreement with\n the natural flow regime. The HydPy-Dam model supports such\n variations. Hence we define a short simulation time period first.\n This enables us to show how the related parameters values can be\n defined and how the calculation of the `remote` water demand\n throughout the year actually works:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2001.03.30', '2001.04.03', '1d'\n\n Prepare the dam model:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n\n Assume the required discharge at a gauge downstream being 2 m\u00b3/s\n in the hydrological summer half-year (April to October). In the\n winter month (November to May), there is no such requirement:\n\n >>> remotedischargeminimum(_11_1_12=0.0, _03_31_12=0.0,\n ... _04_1_12=2.0, _10_31_12=2.0)\n >>> derived.toy.update()\n\n Prepare a test function, that calculates the remote discharge demand\n based on the parameter values defined above and for natural remote\n discharge values ranging between 0 and 3 m\u00b3/s:\n\n >>> from hydpy import UnitTest\n >>> test = UnitTest(model, model.calc_remotedemand_v1, last_example=4,\n ... parseqs=(fluxes.naturalremotedischarge,\n ... fluxes.remotedemand))\n >>> test.nexts.naturalremotedischarge = range(4)\n\n On April 1, the required discharge is 2 m\u00b3/s:\n\n >>> model.idx_sim = pub.timegrids.init['2001.04.01']\n >>> test()\n | ex. | naturalremotedischarge | remotedemand |\n -----------------------------------------------\n | 1 | 0.0 | 2.0 |\n | 2 | 1.0 | 1.0 |\n | 3 | 2.0 | 0.0 |\n | 4 | 3.0 | 0.0 |\n\n On May 31, the required discharge is 0 m\u00b3/s:\n\n >>> model.idx_sim = pub.timegrids.init['2001.03.31']\n >>> test()\n | ex. | naturalremotedischarge | remotedemand |\n -----------------------------------------------\n | 1 | 0.0 | 0.0 |\n | 2 | 1.0 | 0.0 |\n | 3 | 2.0 | 0.0 |\n | 4 | 3.0 | 0.0 |\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.remotedemand = max(con.remotedischargeminimum[der.toy[self.idx_sim]] -\n flu.naturalremotedischarge, 0.)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nestimating the shortfall of actual discharge under the required discharge at the cross section far downstream.", "response": "def calc_remotefailure_v1(self):\n \"\"\"Estimate the shortfall of actual discharge under the required discharge\n of a cross section far downstream.\n\n Required control parameters:\n |NmbLogEntries|\n |RemoteDischargeMinimum|\n\n Required derived parameters:\n |dam_derived.TOY|\n\n Required log sequence:\n |LoggedTotalRemoteDischarge|\n\n Calculated flux sequence:\n |RemoteFailure|\n\n Basic equation:\n :math:`RemoteFailure =\n \\\\frac{\\\\Sigma(LoggedTotalRemoteDischarge)}{NmbLogEntries} -\n RemoteDischargeMinimum`\n\n Examples:\n\n As explained in the documentation on method |calc_remotedemand_v1|,\n we have to define a simulation period first:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2001.03.30', '2001.04.03', '1d'\n\n Now we prepare a dam model with log sequences memorizing three values:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> nmblogentries(3)\n\n Again, the required discharge is 2 m\u00b3/s in summer and 0 m\u00b3/s in winter:\n\n >>> remotedischargeminimum(_11_1_12=0.0, _03_31_12=0.0,\n ... _04_1_12=2.0, _10_31_12=2.0)\n >>> derived.toy.update()\n\n Let it be supposed that the actual discharge at the remote\n cross section droped from 2 m\u00b3/s to 0 m\u00b3/s over the last three days:\n\n >>> logs.loggedtotalremotedischarge(0.0, 1.0, 2.0)\n\n This means that for the April 1 there would have been an averaged\n shortfall of 1 m\u00b3/s:\n\n >>> model.idx_sim = pub.timegrids.init['2001.04.01']\n >>> model.calc_remotefailure_v1()\n >>> fluxes.remotefailure\n remotefailure(1.0)\n\n Instead for May 31 there would have been an excess of 1 m\u00b3/s, which\n is interpreted to be a \"negative failure\":\n\n >>> model.idx_sim = pub.timegrids.init['2001.03.31']\n >>> model.calc_remotefailure_v1()\n >>> fluxes.remotefailure\n remotefailure(-1.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n flu.remotefailure = 0\n for idx in range(con.nmblogentries):\n flu.remotefailure -= log.loggedtotalremotedischarge[idx]\n flu.remotefailure /= con.nmblogentries\n flu.remotefailure += con.remotedischargeminimum[der.toy[self.idx_sim]]"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_requiredremoterelease_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.requiredremoterelease = (\n flu.remotedemand+con.remotedischargesafety[der.toy[self.idx_sim]] *\n smoothutils.smooth_logistic1(\n flu.remotefailure,\n der.remotedischargesmoothpar[der.toy[self.idx_sim]]))", "response": "Calculates the required release needed to not fall below the threshold."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef calc_requiredremoterelease_v2(self):\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n flu.requiredremoterelease = log.loggedrequiredremoterelease[0]", "response": "Calculate the required remote release of the last simulation step."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef calc_allowedremoterelieve_v1(self):\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n flu.allowedremoterelieve = log.loggedallowedremoterelieve[0]", "response": "Calculate the allowed remote relieve of the last simulation step."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalculates the total water release required for reducing drought events.", "response": "def calc_requiredrelease_v1(self):\n \"\"\"Calculate the total water release (immediately and far downstream)\n required for reducing drought events.\n\n Required control parameter:\n |NearDischargeMinimumThreshold|\n\n Required derived parameters:\n |NearDischargeMinimumSmoothPar2|\n |dam_derived.TOY|\n\n Required flux sequence:\n |RequiredRemoteRelease|\n\n Calculated flux sequence:\n |RequiredRelease|\n\n Basic equation:\n :math:`RequiredRelease = RequiredRemoteRelease\n \\\\cdot smooth_{logistic2}(\n RequiredRemoteRelease-NearDischargeMinimumThreshold,\n NearDischargeMinimumSmoothPar2)`\n\n Used auxiliary method:\n |smooth_logistic2|\n\n Examples:\n\n As in the examples above, define a short simulation time period first:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2001.03.30', '2001.04.03', '1d'\n\n Prepare the dam model:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> derived.toy.update()\n\n Define a minimum discharge value for a cross section immediately\n downstream of 4 m\u00b3/s for the summer months and of 0 m\u00b3/s for the\n winter months:\n\n >>> neardischargeminimumthreshold(_11_1_12=0.0, _03_31_12=0.0,\n ... _04_1_12=4.0, _10_31_12=4.0)\n\n Also define related tolerance values that are 1 m\u00b3/s in summer and\n 0 m\u00b3/s in winter:\n\n >>> neardischargeminimumtolerance(_11_1_12=0.0, _03_31_12=0.0,\n ... _04_1_12=1.0, _10_31_12=1.0)\n >>> derived.neardischargeminimumsmoothpar2.update()\n\n Prepare a test function, that calculates the required total discharge\n based on the parameter values defined above and for a required value\n for a cross section far downstream ranging from 0 m\u00b3/s to 8 m\u00b3/s:\n\n >>> from hydpy import UnitTest\n >>> test = UnitTest(model, model.calc_requiredrelease_v1,\n ... last_example=9,\n ... parseqs=(fluxes.requiredremoterelease,\n ... fluxes.requiredrelease))\n >>> test.nexts.requiredremoterelease = range(9)\n\n On May 31, both the threshold and the tolerance value are 0 m\u00b3/s.\n Hence the required total and the required remote release are equal:\n\n >>> model.idx_sim = pub.timegrids.init['2001.03.31']\n >>> test()\n | ex. | requiredremoterelease | requiredrelease |\n -------------------------------------------------\n | 1 | 0.0 | 0.0 |\n | 2 | 1.0 | 1.0 |\n | 3 | 2.0 | 2.0 |\n | 4 | 3.0 | 3.0 |\n | 5 | 4.0 | 4.0 |\n | 6 | 5.0 | 5.0 |\n | 7 | 6.0 | 6.0 |\n | 8 | 7.0 | 7.0 |\n | 9 | 8.0 | 8.0 |\n\n On April 1, the threshold value is 4 m\u00b3/s and the tolerance value\n is 1 m\u00b3/s. For low values of the required remote release, the\n required total release approximates the threshold value. For large\n values, it approximates the required remote release itself. Around\n the threshold value, due to the tolerance value of 1 m\u00b3/s, the\n required total release is a little larger than both the treshold\n value and the required remote release value:\n\n >>> model.idx_sim = pub.timegrids.init['2001.04.01']\n >>> test()\n | ex. | requiredremoterelease | requiredrelease |\n -------------------------------------------------\n | 1 | 0.0 | 4.0 |\n | 2 | 1.0 | 4.000012 |\n | 3 | 2.0 | 4.000349 |\n | 4 | 3.0 | 4.01 |\n | 5 | 4.0 | 4.205524 |\n | 6 | 5.0 | 5.01 |\n | 7 | 6.0 | 6.000349 |\n | 8 | 7.0 | 7.000012 |\n | 9 | 8.0 | 8.0 |\n\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.requiredrelease = con.neardischargeminimumthreshold[\n der.toy[self.idx_sim]]\n flu.requiredrelease = (\n flu.requiredrelease +\n smoothutils.smooth_logistic2(\n flu.requiredremoterelease-flu.requiredrelease,\n der.neardischargeminimumsmoothpar2[\n der.toy[self.idx_sim]]))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the required water release for a single base on the next stage of the water release.", "response": "def calc_requiredrelease_v2(self):\n \"\"\"Calculate the water release (immediately downstream) required for\n reducing drought events.\n\n Required control parameter:\n |NearDischargeMinimumThreshold|\n\n Required derived parameter:\n |dam_derived.TOY|\n\n Calculated flux sequence:\n |RequiredRelease|\n\n Basic equation:\n :math:`RequiredRelease = NearDischargeMinimumThreshold`\n\n Examples:\n\n As in the examples above, define a short simulation time period first:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2001.03.30', '2001.04.03', '1d'\n\n Prepare the dam model:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> derived.toy.update()\n\n Define a minimum discharge value for a cross section immediately\n downstream of 4 m\u00b3/s for the summer months and of 0 m\u00b3/s for the\n winter months:\n\n >>> neardischargeminimumthreshold(_11_1_12=0.0, _03_31_12=0.0,\n ... _04_1_12=4.0, _10_31_12=4.0)\n\n\n As to be expected, the calculated required release is 0.0 m\u00b3/s\n on May 31 and 4.0 m\u00b3/s on April 1:\n\n >>> model.idx_sim = pub.timegrids.init['2001.03.31']\n >>> model.calc_requiredrelease_v2()\n >>> fluxes.requiredrelease\n requiredrelease(0.0)\n >>> model.idx_sim = pub.timegrids.init['2001.04.01']\n >>> model.calc_requiredrelease_v2()\n >>> fluxes.requiredrelease\n requiredrelease(4.0)\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n flu.requiredrelease = con.neardischargeminimumthreshold[\n der.toy[self.idx_sim]]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the highest possible water release that can be routed to another location based on an artificial network describing the possible remote relieve between the two possible release and the water stage.", "response": "def calc_possibleremoterelieve_v1(self):\n \"\"\"Calculate the highest possible water release that can be routed to\n a remote location based on an artificial neural network describing the\n relationship between possible release and water stage.\n\n Required control parameter:\n |WaterLevel2PossibleRemoteRelieve|\n\n Required aide sequence:\n |WaterLevel|\n\n Calculated flux sequence:\n |PossibleRemoteRelieve|\n\n Example:\n\n For simplicity, the example of method |calc_flooddischarge_v1|\n is reused. See the documentation on the mentioned method for\n further information:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> waterlevel2possibleremoterelieve(\n ... nmb_inputs=1,\n ... nmb_neurons=(2,),\n ... nmb_outputs=1,\n ... weights_input=[[50., 4]],\n ... weights_output=[[2.], [30]],\n ... intercepts_hidden=[[-13000, -1046]],\n ... intercepts_output=[0.])\n >>> from hydpy import UnitTest\n >>> test = UnitTest(\n ... model, model.calc_possibleremoterelieve_v1,\n ... last_example=21,\n ... parseqs=(aides.waterlevel, fluxes.possibleremoterelieve))\n >>> test.nexts.waterlevel = numpy.arange(257, 261.1, 0.2)\n >>> test()\n | ex. | waterlevel | possibleremoterelieve |\n --------------------------------------------\n | 1 | 257.0 | 0.0 |\n | 2 | 257.2 | 0.000001 |\n | 3 | 257.4 | 0.000002 |\n | 4 | 257.6 | 0.000005 |\n | 5 | 257.8 | 0.000011 |\n | 6 | 258.0 | 0.000025 |\n | 7 | 258.2 | 0.000056 |\n | 8 | 258.4 | 0.000124 |\n | 9 | 258.6 | 0.000275 |\n | 10 | 258.8 | 0.000612 |\n | 11 | 259.0 | 0.001362 |\n | 12 | 259.2 | 0.003031 |\n | 13 | 259.4 | 0.006745 |\n | 14 | 259.6 | 0.015006 |\n | 15 | 259.8 | 0.033467 |\n | 16 | 260.0 | 1.074179 |\n | 17 | 260.2 | 2.164498 |\n | 18 | 260.4 | 2.363853 |\n | 19 | 260.6 | 2.79791 |\n | 20 | 260.8 | 3.719725 |\n | 21 | 261.0 | 5.576088 |\n \"\"\"\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n con.waterlevel2possibleremoterelieve.inputs[0] = aid.waterlevel\n con.waterlevel2possibleremoterelieve.process_actual_input()\n flu.possibleremoterelieve = con.waterlevel2possibleremoterelieve.outputs[0]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_actualremoterelieve_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n d_smoothpar = con.remoterelievetolerance*flu.allowedremoterelieve\n flu.actualremoterelieve = smoothutils.smooth_min1(\n flu.possibleremoterelieve, flu.allowedremoterelieve, d_smoothpar)\n for dummy in range(5):\n d_smoothpar /= 5.\n flu.actualremoterelieve = smoothutils.smooth_max1(\n flu.actualremoterelieve, 0., d_smoothpar)\n d_smoothpar /= 5.\n flu.actualremoterelieve = smoothutils.smooth_min1(\n flu.actualremoterelieve, flu.possibleremoterelieve, d_smoothpar)\n flu.actualremoterelieve = min(flu.actualremoterelieve,\n flu.possibleremoterelieve)\n flu.actualremoterelieve = min(flu.actualremoterelieve,\n flu.allowedremoterelieve)\n flu.actualremoterelieve = max(flu.actualremoterelieve, 0.)", "response": "Calculates the actual amount of water released to a remote location."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef calc_targetedrelease_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n if con.restricttargetedrelease:\n flu.targetedrelease = smoothutils.smooth_logistic1(\n flu.inflow-con.neardischargeminimumthreshold[\n der.toy[self.idx_sim]],\n der.neardischargeminimumsmoothpar1[der.toy[self.idx_sim]])\n flu.targetedrelease = (flu.targetedrelease * flu.requiredrelease +\n (1.-flu.targetedrelease) * flu.inflow)\n else:\n flu.targetedrelease = flu.requiredrelease", "response": "Calculates targeted water release for reducing drought events in a single low flow and the actual water release."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef calc_actualrelease_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n flu.actualrelease = (flu.targetedrelease *\n smoothutils.smooth_logistic1(\n aid.waterlevel-con.waterlevelminimumthreshold,\n der.waterlevelminimumsmoothpar))", "response": "Calculate the actual water release that can be supplied by the aide model considering the targeted release and the given water level."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate the portion of the required remote demand that could not be met by the actual discharge release.", "response": "def calc_missingremoterelease_v1(self):\n \"\"\"Calculate the portion of the required remote demand that could not\n be met by the actual discharge release.\n\n Required flux sequences:\n |RequiredRemoteRelease|\n |ActualRelease|\n\n Calculated flux sequence:\n |MissingRemoteRelease|\n\n Basic equation:\n :math:`MissingRemoteRelease = max(\n RequiredRemoteRelease-ActualRelease, 0)`\n\n Example:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> fluxes.requiredremoterelease = 2.0\n >>> fluxes.actualrelease = 1.0\n >>> model.calc_missingremoterelease_v1()\n >>> fluxes.missingremoterelease\n missingremoterelease(1.0)\n >>> fluxes.actualrelease = 3.0\n >>> model.calc_missingremoterelease_v1()\n >>> fluxes.missingremoterelease\n missingremoterelease(0.0)\n \"\"\"\n flu = self.sequences.fluxes.fastaccess\n flu.missingremoterelease = max(\n flu.requiredremoterelease-flu.actualrelease, 0.)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef calc_actualremoterelease_v1(self):\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n flu.actualremoterelease = (\n flu.requiredremoterelease *\n smoothutils.smooth_logistic1(\n aid.waterlevel-con.waterlevelminimumremotethreshold,\n der.waterlevelminimumremotesmoothpar))", "response": "Calculate the actual remote water release that can be supplied by the specified remote release and the given water level."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconstrains the actual relieve discharge to a remote location.", "response": "def update_actualremoterelieve_v1(self):\n \"\"\"Constrain the actual relieve discharge to a remote location.\n\n Required control parameter:\n |HighestRemoteDischarge|\n\n Required derived parameter:\n |HighestRemoteSmoothPar|\n\n Updated flux sequence:\n |ActualRemoteRelieve|\n\n Basic equation - discontinous:\n :math:`ActualRemoteRelieve = min(ActualRemoteRelease,\n HighestRemoteDischarge)`\n\n Basic equation - continous:\n :math:`ActualRemoteRelieve = smooth_min1(ActualRemoteRelieve,\n HighestRemoteDischarge, HighestRemoteSmoothPar)`\n\n Used auxiliary methods:\n |smooth_min1|\n |smooth_max1|\n\n Note that the given continous basic equation is a simplification of\n the complete algorithm to update |ActualRemoteRelieve|, which also\n makes use of |smooth_max1| to prevent from gaining negative values\n in a smooth manner.\n\n Examples:\n\n Prepare a dam model:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n\n Prepare a test function object that performs eight examples with\n |ActualRemoteRelieve| ranging from 0 to 8 m\u00b3/s and a fixed\n initial value of parameter |HighestRemoteDischarge| of 4 m\u00b3/s:\n\n >>> highestremotedischarge(4.0)\n >>> from hydpy import UnitTest\n >>> test = UnitTest(model, model.update_actualremoterelieve_v1,\n ... last_example=8,\n ... parseqs=(fluxes.actualremoterelieve,))\n >>> test.nexts.actualremoterelieve = range(8)\n\n Through setting the value of |HighestRemoteTolerance| to the\n lowest possible value, there is no smoothing. Instead, the\n shown relationship agrees with a combination of the discontinuous\n minimum and maximum function:\n\n >>> highestremotetolerance(0.0)\n >>> derived.highestremotesmoothpar.update()\n >>> test()\n | ex. | actualremoterelieve |\n -----------------------------\n | 1 | 0.0 |\n | 2 | 1.0 |\n | 3 | 2.0 |\n | 4 | 3.0 |\n | 5 | 4.0 |\n | 6 | 4.0 |\n | 7 | 4.0 |\n | 8 | 4.0 |\n\n Setting a sensible |HighestRemoteTolerance| value results in\n a moderate smoothing:\n\n >>> highestremotetolerance(0.1)\n >>> derived.highestremotesmoothpar.update()\n >>> test()\n | ex. | actualremoterelieve |\n -----------------------------\n | 1 | 0.0 |\n | 2 | 0.999999 |\n | 3 | 1.99995 |\n | 4 | 2.996577 |\n | 5 | 3.836069 |\n | 6 | 3.991578 |\n | 7 | 3.993418 |\n | 8 | 3.993442 |\n\n Method |update_actualremoterelieve_v1| is defined in a similar\n way as method |calc_actualremoterelieve_v1|. Please read the\n documentation on |calc_actualremoterelieve_v1| for further\n information.\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n d_smooth = der.highestremotesmoothpar\n d_highest = con.highestremotedischarge\n d_value = smoothutils.smooth_min1(\n flu.actualremoterelieve, d_highest, d_smooth)\n for dummy in range(5):\n d_smooth /= 5.\n d_value = smoothutils.smooth_max1(\n d_value, 0., d_smooth)\n d_smooth /= 5.\n d_value = smoothutils.smooth_min1(\n d_value, d_highest, d_smooth)\n d_value = min(d_value, flu.actualremoterelieve)\n d_value = min(d_value, d_highest)\n flu.actualremoterelieve = max(d_value, 0.)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating the discharge during and after a flood event based on the aide sequence of the next stage and the next stage.", "response": "def calc_flooddischarge_v1(self):\n \"\"\"Calculate the discharge during and after a flood event based on an\n |anntools.SeasonalANN| describing the relationship(s) between discharge\n and water stage.\n\n Required control parameter:\n |WaterLevel2FloodDischarge|\n\n Required derived parameter:\n |dam_derived.TOY|\n\n Required aide sequence:\n |WaterLevel|\n\n Calculated flux sequence:\n |FloodDischarge|\n\n Example:\n\n\n The control parameter |WaterLevel2FloodDischarge| is derived from\n |SeasonalParameter|. This allows to simulate different seasonal\n dam control schemes. To show that the seasonal selection mechanism\n is implemented properly, we define a short simulation period of\n three days:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2001.01.01', '2001.01.04', '1d'\n\n Now we prepare a dam model and define two different relationships\n between water level and flood discharge. The first relatively\n simple relationship (for January, 2) is based on two neurons\n contained in a single hidden layer and is used in the following\n example. The second neural network (for January, 3) is not\n applied at all, which is why we do not need to assign any parameter\n values to it:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> waterlevel2flooddischarge(\n ... _01_02_12 = ann(nmb_inputs=1,\n ... nmb_neurons=(2,),\n ... nmb_outputs=1,\n ... weights_input=[[50., 4]],\n ... weights_output=[[2.], [30]],\n ... intercepts_hidden=[[-13000, -1046]],\n ... intercepts_output=[0.]),\n ... _01_03_12 = ann(nmb_inputs=1,\n ... nmb_neurons=(2,),\n ... nmb_outputs=1))\n >>> derived.toy.update()\n >>> model.idx_sim = pub.timegrids.sim['2001.01.02']\n\n The following example shows two distinct effects of both neurons\n in the first network. One neuron describes a relatively sharp\n increase between 259.8 and 260.2 meters from about 0 to 2 m\u00b3/s.\n This could describe a release of water through a bottom outlet\n controlled by a valve. The add something like an exponential\n increase between 260 and 261 meters, which could describe the\n uncontrolled flow over a spillway:\n\n >>> from hydpy import UnitTest\n >>> test = UnitTest(model, model.calc_flooddischarge_v1,\n ... last_example=21,\n ... parseqs=(aides.waterlevel,\n ... fluxes.flooddischarge))\n >>> test.nexts.waterlevel = numpy.arange(257, 261.1, 0.2)\n >>> test()\n | ex. | waterlevel | flooddischarge |\n -------------------------------------\n | 1 | 257.0 | 0.0 |\n | 2 | 257.2 | 0.000001 |\n | 3 | 257.4 | 0.000002 |\n | 4 | 257.6 | 0.000005 |\n | 5 | 257.8 | 0.000011 |\n | 6 | 258.0 | 0.000025 |\n | 7 | 258.2 | 0.000056 |\n | 8 | 258.4 | 0.000124 |\n | 9 | 258.6 | 0.000275 |\n | 10 | 258.8 | 0.000612 |\n | 11 | 259.0 | 0.001362 |\n | 12 | 259.2 | 0.003031 |\n | 13 | 259.4 | 0.006745 |\n | 14 | 259.6 | 0.015006 |\n | 15 | 259.8 | 0.033467 |\n | 16 | 260.0 | 1.074179 |\n | 17 | 260.2 | 2.164498 |\n | 18 | 260.4 | 2.363853 |\n | 19 | 260.6 | 2.79791 |\n | 20 | 260.8 | 3.719725 |\n | 21 | 261.0 | 5.576088 |\n \"\"\"\n con = self.parameters.control.fastaccess\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n aid = self.sequences.aides.fastaccess\n con.waterlevel2flooddischarge.inputs[0] = aid.waterlevel\n con.waterlevel2flooddischarge.process_actual_input(der.toy[self.idx_sim])\n flu.flooddischarge = con.waterlevel2flooddischarge.outputs[0]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_outflow_v1(self):\n flu = self.sequences.fluxes.fastaccess\n flu.outflow = max(flu.actualrelease + flu.flooddischarge, 0.)", "response": "Calculate the total outflow of the dam."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate the actual water volume.", "response": "def update_watervolume_v1(self):\n \"\"\"Update the actual water volume.\n\n Required derived parameter:\n |Seconds|\n\n Required flux sequences:\n |Inflow|\n |Outflow|\n\n Updated state sequence:\n |WaterVolume|\n\n Basic equation:\n :math:`\\\\frac{d}{dt}WaterVolume = 1e-6 \\\\cdot (Inflow-Outflow)`\n\n Example:\n\n >>> from hydpy.models.dam import *\n >>> parameterstep()\n >>> derived.seconds = 2e6\n >>> states.watervolume.old = 5.0\n >>> fluxes.inflow = 2.0\n >>> fluxes.outflow = 3.0\n >>> model.update_watervolume_v1()\n >>> states.watervolume\n watervolume(3.0)\n \"\"\"\n der = self.parameters.derived.fastaccess\n flu = self.sequences.fluxes.fastaccess\n old = self.sequences.states.fastaccess_old\n new = self.sequences.states.fastaccess_new\n new.watervolume = (old.watervolume +\n der.seconds*(flu.inflow-flu.outflow)/1e6)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates the outlet link sequence |dam_outlets. Q|.", "response": "def pass_outflow_v1(self):\n \"\"\"Update the outlet link sequence |dam_outlets.Q|.\"\"\"\n flu = self.sequences.fluxes.fastaccess\n out = self.sequences.outlets.fastaccess\n out.q[0] += flu.outflow"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates the actual remoterelease sequence |dam_outlets. S|.", "response": "def pass_actualremoterelease_v1(self):\n \"\"\"Update the outlet link sequence |dam_outlets.S|.\"\"\"\n flu = self.sequences.fluxes.fastaccess\n out = self.sequences.outlets.fastaccess\n out.s[0] += flu.actualremoterelease"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nupdating the actual remoterelieve sequence |dam_outlets. R|.", "response": "def pass_actualremoterelieve_v1(self):\n \"\"\"Update the outlet link sequence |dam_outlets.R|.\"\"\"\n flu = self.sequences.fluxes.fastaccess\n out = self.sequences.outlets.fastaccess\n out.r[0] += flu.actualremoterelieve"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pass_missingremoterelease_v1(self):\n flu = self.sequences.fluxes.fastaccess\n sen = self.sequences.senders.fastaccess\n sen.d[0] += flu.missingremoterelease", "response": "Update the outlet link sequence |dam_senders. D|."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate the outlet link sequence |dam_outlets. R|.", "response": "def pass_allowedremoterelieve_v1(self):\n \"\"\"Update the outlet link sequence |dam_outlets.R|.\"\"\"\n flu = self.sequences.fluxes.fastaccess\n sen = self.sequences.senders.fastaccess\n sen.r[0] += flu.allowedremoterelieve"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating the outlet link sequence |dam_outlets. S|.", "response": "def pass_requiredremotesupply_v1(self):\n \"\"\"Update the outlet link sequence |dam_outlets.S|.\"\"\"\n flu = self.sequences.fluxes.fastaccess\n sen = self.sequences.senders.fastaccess\n sen.s[0] += flu.requiredremotesupply"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_loggedoutflow_v1(self):\n con = self.parameters.control.fastaccess\n flu = self.sequences.fluxes.fastaccess\n log = self.sequences.logs.fastaccess\n for idx in range(con.nmblogentries-1, 0, -1):\n log.loggedoutflow[idx] = log.loggedoutflow[idx-1]\n log.loggedoutflow[0] = flu.outflow", "response": "Update the logged outflow sequence with the new entry of discharge at a cross section far downstream."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef turningpoint(self):\n coefs = self.coefs\n old_dc = coefs[1]-coefs[0]\n for idx in range(self.order-2):\n new_dc = coefs[idx+2]-coefs[idx+1]\n if (old_dc < 0.) and (new_dc > old_dc):\n return idx, coefs[idx]\n old_dc = new_dc\n raise RuntimeError(\n 'Not able to detect a turning point in the impulse response '\n 'defined by the MA coefficients %s.'\n % objecttools.repr_values(coefs))", "response": "Return the index and value tuple of the recession part of the instantaneous unit hydrograph."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndetermine the AR coefficients.", "response": "def update_ar_coefs(self):\n \"\"\"Determine the AR coefficients.\n\n The number of AR coefficients is subsequently increased until the\n required precision |ARMA.max_rel_rmse| is reached. Otherwise,\n a |RuntimeError| is raised.\n \"\"\"\n del self.ar_coefs\n for ar_order in range(1, self.effective_max_ar_order+1):\n self.calc_all_ar_coefs(ar_order, self.ma)\n if self._rel_rmse < self.max_rel_rmse:\n break\n else:\n with hydpy.pub.options.reprdigits(12):\n raise RuntimeError(\n f'Method `update_ar_coefs` is not able to determine '\n f'the AR coefficients of the ARMA model with the desired '\n f'accuracy. You can either set the tolerance value '\n f'`max_rel_rmse` to a higher value or increase the '\n f'allowed `max_ar_order`. An accuracy of `'\n f'{objecttools.repr_(self._rel_rmse)}` has been reached '\n f'using `{self.effective_max_ar_order}` coefficients.')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsumming of the absolute deviations between the central moments of the instantaneous unit hydrograph and the ARMA approximation.", "response": "def dev_moments(self):\n \"\"\"Sum of the absolute deviations between the central moments of the\n instantaneous unit hydrograph and the ARMA approximation.\"\"\"\n return numpy.sum(numpy.abs(self.moments-self.ma.moments))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmultiplying all coefficients by the same factor so that their sum becomes one.", "response": "def norm_coefs(self):\n \"\"\"Multiply all coefficients by the same factor, so that their sum\n becomes one.\"\"\"\n sum_coefs = self.sum_coefs\n self.ar_coefs /= sum_coefs\n self.ma_coefs /= sum_coefs"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the AR coefficients of all the entries in the current set of all the entries in the set of all the entries in the set of all the entries in the set of all the entries in the set.", "response": "def calc_all_ar_coefs(self, ar_order, ma_model):\n \"\"\"Determine the AR coeffcients based on a least squares approach.\n\n The argument `ar_order` defines the number of AR coefficients to be\n determined. The argument `ma_order` defines a pure |MA| model.\n The least squares approach is applied on all those coefficents of the\n pure MA model, which are associated with the part of the recession\n curve behind its turning point.\n\n The attribute |ARMA.rel_rmse| is updated with the resulting\n relative root mean square error.\n \"\"\"\n turning_idx, _ = ma_model.turningpoint\n values = ma_model.coefs[turning_idx:]\n self.ar_coefs, residuals = numpy.linalg.lstsq(\n self.get_a(values, ar_order),\n self.get_b(values, ar_order),\n rcond=-1)[:2]\n if len(residuals) == 1:\n self._rel_rmse = numpy.sqrt(residuals[0])/numpy.sum(values)\n else:\n self._rel_rmse = 0."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_a(values, n):\n m = len(values)-n\n a = numpy.empty((m, n), dtype=float)\n for i in range(m):\n i0 = i-1 if i > 0 else None\n i1 = i+n-1\n a[i] = values[i1:i0:-1]\n return numpy.array(a)", "response": "Extract the independent variables of the given values and return\n them as a matrix with n columns."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update_ma_coefs(self):\n self.ma_coefs = []\n for ma_order in range(1, self.ma.order+1):\n self.calc_next_ma_coef(ma_order, self.ma)\n if self.dev_coefs < self.max_dev_coefs:\n self.norm_coefs()\n break\n else:\n with hydpy.pub.options.reprdigits(12):\n raise RuntimeError(\n f'Method `update_ma_coefs` is not able to determine the '\n f'MA coefficients of the ARMA model with the desired '\n f'accuracy. You can set the tolerance value '\n f'\u00b4max_dev_coefs` to a higher value. An accuracy of '\n f'`{objecttools.repr_(self.dev_coefs)}` has been reached '\n f'using `{self.ma.order}` MA coefficients.')\n if numpy.min(self.response) < 0.:\n warnings.warn(\n 'Note that the smallest response to a standard impulse of the '\n 'determined ARMA model is negative (`%s`).'\n % objecttools.repr_(numpy.min(self.response)))", "response": "Determine the MA coefficients."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_next_ma_coef(self, ma_order, ma_model):\n idx = ma_order-1\n coef = ma_model.coefs[idx]\n for jdx, ar_coef in enumerate(self.ar_coefs):\n zdx = idx-jdx-1\n if zdx >= 0:\n coef -= ar_coef*ma_model.coefs[zdx]\n self.ma_coefs = numpy.concatenate((self.ma_coefs, [coef]))", "response": "Calculates the next MA coefficient for the given MA model based on its AR coefficients and the MA ordinates of the given MA model."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the response to a standard dt impulse.", "response": "def response(self):\n \"\"\"Return the response to a standard dt impulse.\"\"\"\n values = []\n sum_values = 0.\n ma_coefs = self.ma_coefs\n ar_coefs = self.ar_coefs\n ma_order = self.ma_order\n for idx in range(len(self.ma.delays)):\n value = 0.\n if idx < ma_order:\n value += ma_coefs[idx]\n for jdx, ar_coef in enumerate(ar_coefs):\n zdx = idx-jdx-1\n if zdx >= 0:\n value += ar_coef*values[zdx]\n values.append(value)\n sum_values += value\n return numpy.array(values)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef method_header(method_name, nogil=False, idx_as_arg=False):\n if not config.FASTCYTHON:\n nogil = False\n header = 'cpdef inline void %s(self' % method_name\n header += ', int idx)' if idx_as_arg else ')'\n header += ' nogil:' if nogil else ':'\n return header", "response": "Returns the Cython method header for methods without arguments except\n self."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add(self, indent, line):\n if isinstance(line, str):\n list.append(self, indent*4*' ' + line)\n else:\n for subline in line:\n list.append(self, indent*4*' ' + subline)", "response": "Appends the given text line with prefixed spaces in accordance with\n the given number of indentation levels."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nname of the compiled module.", "response": "def pyname(self):\n \"\"\"Name of the compiled module.\"\"\"\n if self.pymodule.endswith('__init__'):\n return self.pymodule.split('.')[-2]\n else:\n return self.pymodule.split('.')[-1]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pyxwriter(self):\n model = self.Model()\n if hasattr(self, 'Parameters'):\n model.parameters = self.Parameters(vars(self))\n else:\n model.parameters = parametertools.Parameters(vars(self))\n if hasattr(self, 'Sequences'):\n model.sequences = self.Sequences(model=model, **vars(self))\n else:\n model.sequences = sequencetools.Sequences(model=model,\n **vars(self))\n return PyxWriter(self, model, self.pyxfilepath)", "response": "Update the pyx file."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef outdated(self):\n if hydpy.pub.options.forcecompiling:\n return True\n if os.path.split(hydpy.__path__[0])[-2].endswith('-packages'):\n return False\n if not os.path.exists(self.dllfilepath):\n return True\n cydate = os.stat(self.dllfilepath).st_mtime\n for pysourcefile in self.pysourcefiles:\n pydate = os.stat(pysourcefile).st_mtime\n if pydate > cydate:\n return True\n return False", "response": "True if at least one of the pysourcefiles is newer than the compiled file under |Cythonizer. pyxfilepath| False otherwise."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compile_(self):\n from Cython import Build\n argv = copy.deepcopy(sys.argv)\n sys.argv = [sys.argv[0], 'build_ext', '--build-lib='+self.buildpath]\n exc_modules = [\n distutils.extension.Extension(\n 'hydpy.cythons.autogen.'+self.cyname,\n [self.pyxfilepath], extra_compile_args=['-O2'])]\n distutils.core.setup(ext_modules=Build.cythonize(exc_modules),\n include_dirs=[numpy.get_include()])\n sys.argv = argv", "response": "Translate cython code to C code and compile it."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef move_dll(self):\n dirinfos = os.walk(self.buildpath)\n next(dirinfos)\n system_dependent_filename = None\n for dirinfo in dirinfos:\n for filename in dirinfo[2]:\n if (filename.startswith(self.cyname) and\n filename.endswith(dllextension)):\n system_dependent_filename = filename\n break\n if system_dependent_filename:\n try:\n shutil.move(os.path.join(dirinfo[0],\n system_dependent_filename),\n os.path.join(self.cydirpath,\n self.cyname+dllextension))\n break\n except BaseException:\n prefix = ('After trying to cythonize module %s, when '\n 'trying to move the final cython module %s '\n 'from directory %s to directory %s'\n % (self.pyname, system_dependent_filename,\n self.buildpath, self.cydirpath))\n suffix = ('A likely error cause is that the cython module '\n '%s does already exist in this directory and is '\n 'currently blocked by another Python process. '\n 'Maybe it helps to close all Python processes '\n 'and restart the cyhonization afterwards.'\n % self.cyname+dllextension)\n objecttools.augment_excmessage(prefix, suffix)\n else:\n raise IOError('After trying to cythonize module %s, the resulting '\n 'file %s could neither be found in directory %s nor '\n 'its subdirectories. The distul report should tell '\n 'whether the file has been stored somewhere else,'\n 'is named somehow else, or could not be build at '\n 'all.' % self.buildpath)", "response": "Try to find the resulting dll file and to move it into the cythons package."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef parameters(self):\n lines = Lines()\n lines.add(0, '@cython.final')\n lines.add(0, 'cdef class Parameters(object):')\n for subpars in self.model.parameters:\n if subpars:\n lines.add(1, 'cdef public %s %s'\n % (objecttools.classname(subpars), subpars.name))\n for subpars in self.model.parameters:\n if subpars:\n print(' - %s' % subpars.name)\n lines.add(0, '@cython.final')\n lines.add(0, 'cdef class %s(object):'\n % objecttools.classname(subpars))\n for par in subpars:\n try:\n ctype = TYPE2STR[par.TYPE] + NDIM2STR[par.NDIM]\n except KeyError:\n ctype = par.TYPE + NDIM2STR[par.NDIM]\n lines.add(1, 'cdef public %s %s' % (ctype, par.name))\n return lines", "response": "Return a list of lines that define the parameters of the current object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of class Sequences.", "response": "def sequences(self):\n \"\"\"Sequence declaration lines.\"\"\"\n lines = Lines()\n lines.add(0, '@cython.final')\n lines.add(0, 'cdef class Sequences(object):')\n for subseqs in self.model.sequences:\n lines.add(1, 'cdef public %s %s'\n % (objecttools.classname(subseqs), subseqs.name))\n if getattr(self.model.sequences, 'states', None) is not None:\n lines.add(1, 'cdef public StateSequences old_states')\n lines.add(1, 'cdef public StateSequences new_states')\n for subseqs in self.model.sequences:\n print(' - %s' % subseqs.name)\n lines.add(0, '@cython.final')\n lines.add(0, 'cdef class %s(object):'\n % objecttools.classname(subseqs))\n for seq in subseqs:\n ctype = 'double' + NDIM2STR[seq.NDIM]\n if isinstance(subseqs, sequencetools.LinkSequences):\n if seq.NDIM == 0:\n lines.add(1, 'cdef double *%s' % seq.name)\n elif seq.NDIM == 1:\n lines.add(1, 'cdef double **%s' % seq.name)\n lines.add(1, 'cdef public int len_%s' % seq.name)\n else:\n lines.add(1, 'cdef public %s %s' % (ctype, seq.name))\n lines.add(1, 'cdef public int _%s_ndim' % seq.name)\n lines.add(1, 'cdef public int _%s_length' % seq.name)\n for idx in range(seq.NDIM):\n lines.add(1, 'cdef public int _%s_length_%d'\n % (seq.name, idx))\n if seq.NUMERIC:\n ctype_numeric = 'double' + NDIM2STR[seq.NDIM+1]\n lines.add(1, 'cdef public %s _%s_points'\n % (ctype_numeric, seq.name))\n lines.add(1, 'cdef public %s _%s_results'\n % (ctype_numeric, seq.name))\n if isinstance(subseqs, sequencetools.FluxSequences):\n lines.add(1, 'cdef public %s _%s_integrals'\n % (ctype_numeric, seq.name))\n lines.add(1, 'cdef public %s _%s_sum'\n % (ctype, seq.name))\n if isinstance(subseqs, sequencetools.IOSequences):\n lines.extend(self.iosequence(seq))\n if isinstance(subseqs, sequencetools.InputSequences):\n lines.extend(self.load_data(subseqs))\n if isinstance(subseqs, sequencetools.IOSequences):\n lines.extend(self.open_files(subseqs))\n lines.extend(self.close_files(subseqs))\n if not isinstance(subseqs, sequencetools.InputSequence):\n lines.extend(self.save_data(subseqs))\n if isinstance(subseqs, sequencetools.LinkSequences):\n lines.extend(self.set_pointer(subseqs))\n return lines"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a |Lines| object containing the |Reserved| and |Reserved| declaration lines for the given |IOSequence| object.", "response": "def iosequence(seq):\n \"\"\"Special declaration lines for the given |IOSequence| object.\n \"\"\"\n lines = Lines()\n lines.add(1, 'cdef public bint _%s_diskflag' % seq.name)\n lines.add(1, 'cdef public str _%s_path' % seq.name)\n lines.add(1, 'cdef FILE *_%s_file' % seq.name)\n lines.add(1, 'cdef public bint _%s_ramflag' % seq.name)\n ctype = 'double' + NDIM2STR[seq.NDIM+1]\n lines.add(1, 'cdef public %s _%s_array' % (ctype, seq.name))\n return lines"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_pointer(self, subseqs):\n lines = Lines()\n for seq in subseqs:\n if seq.NDIM == 0:\n lines.extend(self.set_pointer0d(subseqs))\n break\n for seq in subseqs:\n if seq.NDIM == 1:\n lines.extend(self.alloc(subseqs))\n lines.extend(self.dealloc(subseqs))\n lines.extend(self.set_pointer1d(subseqs))\n break\n return lines", "response": "Set_pointer functions for link sequences."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nallocating memory for 1 - dimensional link sequences.", "response": "def alloc(subseqs):\n \"\"\"Allocate memory for 1-dimensional link sequences.\"\"\"\n print(' . setlength')\n lines = Lines()\n lines.add(1, 'cpdef inline alloc(self, name, int length):')\n for seq in subseqs:\n lines.add(2, 'if name == \"%s\":' % seq.name)\n lines.add(3, 'self._%s_length_0 = length' % seq.name)\n lines.add(3, 'self.%s = '\n 'PyMem_Malloc(length * sizeof(double*))' % seq.name)\n return lines"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_pointer1d(subseqs):\n print(' . set_pointer1d')\n lines = Lines()\n lines.add(1, 'cpdef inline set_pointer1d'\n '(self, str name, pointerutils.PDouble value, int idx):')\n for seq in subseqs:\n lines.add(2, 'if name == \"%s\":' % seq.name)\n lines.add(3, 'self.%s[idx] = value.p_value' % seq.name)\n return lines", "response": "Set_pointer function for 1 - dimensional link sequences."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef numericalparameters(self):\n lines = Lines()\n if self.model.NUMERICAL:\n lines.add(0, '@cython.final')\n lines.add(0, 'cdef class NumConsts(object):')\n for name in ('nmb_methods', 'nmb_stages'):\n lines.add(1, 'cdef public %s %s' % (TYPE2STR[int], name))\n for name in ('dt_increase', 'dt_decrease'):\n lines.add(1, 'cdef public %s %s' % (TYPE2STR[float], name))\n lines.add(1, 'cdef public configutils.Config pub')\n lines.add(1, 'cdef public double[:, :, :] a_coefs')\n lines.add(0, 'cdef class NumVars(object):')\n for name in ('nmb_calls', 'idx_method', 'idx_stage'):\n lines.add(1, 'cdef public %s %s' % (TYPE2STR[int], name))\n for name in ('t0', 't1', 'dt', 'dt_est',\n 'error', 'last_error', 'extrapolated_error'):\n lines.add(1, 'cdef public %s %s' % (TYPE2STR[float], name))\n lines.add(1, 'cdef public %s f0_ready' % TYPE2STR[bool])\n return lines", "response": "Numeric parameter declaration lines."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef modeldeclarations(self):\n lines = Lines()\n lines.add(0, '@cython.final')\n lines.add(0, 'cdef class Model(object):')\n lines.add(1, 'cdef public int idx_sim')\n lines.add(1, 'cdef public Parameters parameters')\n lines.add(1, 'cdef public Sequences sequences')\n if hasattr(self.model, 'numconsts'):\n lines.add(1, 'cdef public NumConsts numconsts')\n if hasattr(self.model, 'numvars'):\n lines.add(1, 'cdef public NumVars numvars')\n return lines", "response": "Attribute declarations of the model class."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef modelstandardfunctions(self):\n lines = Lines()\n lines.extend(self.doit)\n lines.extend(self.iofunctions)\n lines.extend(self.new2old)\n lines.extend(self.run)\n lines.extend(self.update_inlets)\n lines.extend(self.update_outlets)\n lines.extend(self.update_receivers)\n lines.extend(self.update_senders)\n return lines", "response": "Standard functions of the model class."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef modelnumericfunctions(self):\n lines = Lines()\n lines.extend(self.solve)\n lines.extend(self.calculate_single_terms)\n lines.extend(self.calculate_full_terms)\n lines.extend(self.get_point_states)\n lines.extend(self.set_point_states)\n lines.extend(self.set_result_states)\n lines.extend(self.get_sum_fluxes)\n lines.extend(self.set_point_fluxes)\n lines.extend(self.set_result_fluxes)\n lines.extend(self.integrate_fluxes)\n lines.extend(self.reset_sum_fluxes)\n lines.extend(self.addup_fluxes)\n lines.extend(self.calculate_error)\n lines.extend(self.extrapolate_error)\n return lines", "response": "Return a list of numeric functions of the model class."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndoing ( most of ) it function of the model class.", "response": "def doit(self):\n \"\"\"Do (most of) it function of the model class.\"\"\"\n print(' . doit')\n lines = Lines()\n lines.add(1, 'cpdef inline void doit(self, int idx) %s:' % _nogil)\n lines.add(2, 'self.idx_sim = idx')\n if getattr(self.model.sequences, 'inputs', None) is not None:\n lines.add(2, 'self.load_data()')\n if self.model.INLET_METHODS:\n lines.add(2, 'self.update_inlets()')\n if hasattr(self.model, 'solve'):\n lines.add(2, 'self.solve()')\n else:\n lines.add(2, 'self.run()')\n if getattr(self.model.sequences, 'states', None) is not None:\n lines.add(2, 'self.new2old()')\n if self.model.OUTLET_METHODS:\n lines.add(2, 'self.update_outlets()')\n return lines"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninputs and output functions of the model class.", "response": "def iofunctions(self):\n \"\"\"Input/output functions of the model class.\"\"\"\n lines = Lines()\n for func in ('open_files', 'close_files', 'load_data', 'save_data'):\n if ((func == 'load_data') and\n (getattr(self.model.sequences, 'inputs', None) is None)):\n continue\n if ((func == 'save_data') and\n ((getattr(self.model.sequences, 'fluxes', None) is None) and\n (getattr(self.model.sequences, 'states', None) is None))):\n continue\n print(' . %s' % func)\n nogil = func in ('load_data', 'save_data')\n idx_as_arg = func == 'save_data'\n lines.add(1, method_header(\n func, nogil=nogil, idx_as_arg=idx_as_arg))\n for subseqs in self.model.sequences:\n if func == 'load_data':\n applyfuncs = ('inputs',)\n elif func == 'save_data':\n applyfuncs = ('fluxes', 'states')\n else:\n applyfuncs = ('inputs', 'fluxes', 'states')\n if subseqs.name in applyfuncs:\n if func == 'close_files':\n lines.add(2, 'self.sequences.%s.%s()'\n % (subseqs.name, func))\n else:\n lines.add(2, 'self.sequences.%s.%s(self.idx_sim)'\n % (subseqs.name, func))\n return lines"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlining of model method with the same name.", "response": "def calculate_single_terms(self):\n \"\"\"Lines of model method with the same name.\"\"\"\n lines = self._call_methods('calculate_single_terms',\n self.model.PART_ODE_METHODS)\n if lines:\n lines.insert(1, (' self.numvars.nmb_calls ='\n 'self.numvars.nmb_calls+1'))\n return lines"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef listofmodeluserfunctions(self):\n lines = []\n for (name, member) in vars(self.model.__class__).items():\n if (inspect.isfunction(member) and\n (name not in ('run', 'new2old')) and\n ('fastaccess' in inspect.getsource(member))):\n lines.append((name, member))\n run = vars(self.model.__class__).get('run')\n if run is not None:\n lines.append(('run', run))\n for (name, member) in vars(self.model).items():\n if (inspect.ismethod(member) and\n ('fastaccess' in inspect.getsource(member))):\n lines.append((name, member))\n return lines", "response": "List of user functions of the model class."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclean code lines. Implemented cleanups: * eventually remove method version * remove docstrings * remove comments * remove empty lines * remove line brackes within brackets * replace `modelutils` with nothing * remove complete lines containing `fastaccess` * replace shortcuts with complete references", "response": "def cleanlines(self):\n \"\"\"Cleaned code lines.\n\n Implemented cleanups:\n * eventually remove method version\n * remove docstrings\n * remove comments\n * remove empty lines\n * remove line brackes within brackets\n * replace `modelutils` with nothing\n * remove complete lines containing `fastaccess`\n * replace shortcuts with complete references\n \"\"\"\n code = inspect.getsource(self.func)\n code = '\\n'.join(code.split('\"\"\"')[::2])\n code = code.replace('modelutils.', '')\n for (name, shortcut) in zip(self.collectornames,\n self.collectorshortcuts):\n code = code.replace('%s.' % shortcut, 'self.%s.' % name)\n code = self.remove_linebreaks_within_equations(code)\n lines = code.splitlines()\n self.remove_imath_operators(lines)\n lines[0] = 'def %s(self):' % self.funcname\n lines = [l.split('#')[0] for l in lines]\n lines = [l for l in lines if 'fastaccess' not in l]\n lines = [l.rstrip() for l in lines if l.rstrip()]\n return Lines(*lines)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nremoves mathematical expressions that require Pythons global interpreter locking mechanism.", "response": "def remove_imath_operators(lines):\n \"\"\"Remove mathematical expressions that require Pythons global\n interpreter locking mechanism.\n\n This is not a exhaustive test, but shows how the method works:\n\n >>> lines = [' x += 1*1']\n >>> from hydpy.cythons.modelutils import FuncConverter\n >>> FuncConverter.remove_imath_operators(lines)\n >>> lines\n [' x = x + (1*1)']\n \"\"\"\n for idx, line in enumerate(lines):\n for operator in ('+=', '-=', '**=', '*=', '//=', '/=', '%='):\n sublines = line.split(operator)\n if len(sublines) > 1:\n indent = line.count(' ') - line.lstrip().count(' ')\n sublines = [sl.strip() for sl in sublines]\n line = ('%s%s = %s %s (%s)'\n % (indent*' ', sublines[0], sublines[0],\n operator[:-1], sublines[1]))\n lines[idx] = line"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef calc_smoothpar_logistic2(metapar):\n if metapar <= 0.:\n return 0.\n return optimize.newton(_error_smoothpar_logistic2,\n .3 * metapar**.84,\n _smooth_logistic2_derivative,\n args=(metapar,))", "response": "Return the smoothing parameter corresponding to the given meta parameter when using |smooth_logistic2|."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a |Date| instance based on a |numpy. ndarray| containing the date information of the first entry of the given |numpy. ndarray|.", "response": "def from_array(cls, array):\n \"\"\"Return a |Date| instance based on date information (year,\n month, day, hour, minute, second) stored as the first entries of\n the successive rows of a |numpy.ndarray|.\n\n >>> from hydpy import Date\n >>> import numpy\n >>> array1d = numpy.array([1992, 10, 8, 15, 15, 42, 999])\n >>> Date.from_array(array1d)\n Date('1992-10-08 15:15:42')\n\n >>> array3d = numpy.zeros((7, 2, 2))\n >>> array3d[:, 0, 0] = array1d\n >>> Date.from_array(array3d)\n Date('1992-10-08 15:15:42')\n\n .. note::\n\n The date defined by the given |numpy.ndarray| cannot\n include any time zone information and corresponds to\n |Options.utcoffset|, which defaults to UTC+01:00.\n \"\"\"\n intarray = numpy.array(array, dtype=int)\n for dummy in range(1, array.ndim):\n intarray = intarray[:, 0]\n return cls(datetime.datetime(*intarray[:6]))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a 1 - dimensional |numpy| |numpy. ndarray| containing the actual date and time zone information.", "response": "def to_array(self):\n \"\"\"Return a 1-dimensional |numpy| |numpy.ndarray| with six entries\n defining the actual date (year, month, day, hour, minute, second).\n\n >>> from hydpy import Date\n >>> Date('1992-10-8 15:15:42').to_array()\n array([ 1992., 10., 8., 15., 15., 42.])\n\n .. note::\n\n The date defined by the returned |numpy.ndarray| does not\n include any time zone information and corresponds to\n |Options.utcoffset|, which defaults to UTC+01:00.\n \"\"\"\n return numpy.array([self.year, self.month, self.day, self.hour,\n self.minute, self.second], dtype=float)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef from_cfunits(cls, units) -> 'Date':\n try:\n string = units[units.find('since')+6:]\n idx = string.find('.')\n if idx != -1:\n jdx = None\n for jdx, char in enumerate(string[idx+1:]):\n if not char.isnumeric():\n break\n if char != '0':\n raise ValueError(\n 'No other decimal fraction of a second '\n 'than \"0\" allowed.')\n else:\n if jdx is None:\n jdx = idx+1\n else:\n jdx += 1\n string = f'{string[:idx]}{string[idx+jdx+1:]}'\n return cls(string)\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to parse the date of the NetCDF-CF \"units\" '\n f'string `{units}`')", "response": "Return a |Date| object representing the reference date of the current NetCDF - CF entry with the given units string."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_cfunits(self, unit='hours', utcoffset=None):\n if utcoffset is None:\n utcoffset = hydpy.pub.options.utcoffset\n string = self.to_string('iso2', utcoffset)\n string = ' '.join((string[:-6], string[-6:]))\n return f'{unit} since {string}'", "response": "Return a string agreeing with NetCDF - CF conventions."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _set_thing(self, thing, value):\n try:\n value = int(value)\n except (TypeError, ValueError):\n raise TypeError(\n f'Changing the {thing} of a `Date` instance is only '\n f'allowed via numbers, but the given value `{value}` '\n f'is of type `{type(value)}` instead.')\n kwargs = {}\n for unit in ('year', 'month', 'day', 'hour', 'minute', 'second'):\n kwargs[unit] = getattr(self, unit)\n kwargs[thing] = value\n self.datetime = datetime.datetime(**kwargs)", "response": "Convenience method for _set_year _set_month..."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef to_string(self, style=None, utcoffset=None):\n if not style:\n style = self.style\n if utcoffset is None:\n string = ''\n date = self.datetime\n else:\n sign = '+' if utcoffset >= 0 else '-'\n hours = abs(utcoffset // 60)\n minutes = abs(utcoffset % 60)\n string = f'{sign}{hours:02d}:{minutes:02d}'\n offset = utcoffset-hydpy.pub.options.utcoffset\n date = self.datetime + datetime.timedelta(minutes=offset)\n return date.strftime(self._formatstrings[style]) + string", "response": "Return a |str| object representing the actual date in\n accordance with the given style and the eventually given\n UTC offset."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a |Period| instance based on a given number of seconds.", "response": "def fromseconds(cls, seconds):\n \"\"\"Return a |Period| instance based on a given number of seconds.\"\"\"\n try:\n seconds = int(seconds)\n except TypeError:\n seconds = int(seconds.flatten()[0])\n return cls(datetime.timedelta(0, int(seconds)))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nguess the unit of the period as the largest one which results in an integer duration.", "response": "def _guessunit(self):\n \"\"\"Guess the unit of the period as the largest one, which results in\n an integer duration.\n \"\"\"\n if not self.days % 1:\n return 'd'\n elif not self.hours % 1:\n return 'h'\n elif not self.minutes % 1:\n return 'm'\n elif not self.seconds % 1:\n return 's'\n else:\n raise ValueError(\n 'The stepsize is not a multiple of one '\n 'second, which is not allowed.')"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a |Timegrid| instance based on two date and one period getTaxID from a given |numpy. ndarray| object.", "response": "def from_array(cls, array):\n \"\"\"Returns a |Timegrid| instance based on two date and one period\n information stored in the first 13 rows of a |numpy.ndarray| object.\n \"\"\"\n try:\n return cls(Date.from_array(array[:6]),\n Date.from_array(array[6:12]),\n Period.fromseconds(array[12]))\n except IndexError:\n raise IndexError(\n f'To define a Timegrid instance via an array, 13 '\n f'numbers are required. However, the given array '\n f'consist of {len(array)} entries/rows only.')"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a 1 - dimensional |numpy| |numpy. ndarray| with thirteen entries firstly defining the start date secondly defining the end date thirdly defining the step size in seconds.", "response": "def to_array(self):\n \"\"\"Returns a 1-dimensional |numpy| |numpy.ndarray| with thirteen\n entries first defining the start date, secondly defining the end\n date and thirdly the step size in seconds.\n \"\"\"\n values = numpy.empty(13, dtype=float)\n values[:6] = self.firstdate.to_array()\n values[6:12] = self.lastdate.to_array()\n values[12] = self.stepsize.seconds\n return values"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a |Timegrid| object representing the given starting timepoints in relation to the given refdate.", "response": "def from_timepoints(cls, timepoints, refdate, unit='hours'):\n \"\"\"Return a |Timegrid| object representing the given starting\n `timepoints` in relation to the given `refdate`.\n\n The following examples identical with the ones of\n |Timegrid.to_timepoints| but reversed.\n\n At least two given time points must be increasing and\n equidistant. By default, they are assumed in hours since\n the given reference date:\n\n >>> from hydpy import Timegrid\n >>> Timegrid.from_timepoints(\n ... [0.0, 6.0, 12.0, 18.0], '01.01.2000')\n Timegrid('01.01.2000 00:00:00',\n '02.01.2000 00:00:00',\n '6h')\n >>> Timegrid.from_timepoints(\n ... [24.0, 30.0, 36.0, 42.0], '1999-12-31')\n Timegrid('2000-01-01 00:00:00',\n '2000-01-02 00:00:00',\n '6h')\n\n Other time units (`days` or `min`) must be passed explicitely\n (only the first character counts):\n\n >>> Timegrid.from_timepoints(\n ... [0.0, 0.25, 0.5, 0.75], '01.01.2000', unit='d')\n Timegrid('01.01.2000 00:00:00',\n '02.01.2000 00:00:00',\n '6h')\n >>> Timegrid.from_timepoints(\n ... [1.0, 1.25, 1.5, 1.75], '1999-12-31', unit='day')\n Timegrid('2000-01-01 00:00:00',\n '2000-01-02 00:00:00',\n '6h')\n \"\"\"\n refdate = Date(refdate)\n unit = Period.from_cfunits(unit)\n delta = timepoints[1]-timepoints[0]\n firstdate = refdate+timepoints[0]*unit\n lastdate = refdate+(timepoints[-1]+delta)*unit\n stepsize = (lastdate-firstdate)/len(timepoints)\n return cls(firstdate, lastdate, stepsize)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn an |numpy. ndarray| representing the starting time points of the |Timegrid| object.", "response": "def to_timepoints(self, unit='hours', offset=None):\n \"\"\"Return an |numpy.ndarray| representing the starting time points\n of the |Timegrid| object.\n\n The following examples identical with the ones of\n |Timegrid.from_timepoints| but reversed.\n\n By default, the time points are given in hours:\n\n >>> from hydpy import Timegrid\n >>> timegrid = Timegrid('2000-01-01', '2000-01-02', '6h')\n >>> timegrid.to_timepoints()\n array([ 0., 6., 12., 18.])\n\n Other time units (`days` or `min`) can be defined (only the first\n character counts):\n\n >>> timegrid.to_timepoints(unit='d')\n array([ 0. , 0.25, 0.5 , 0.75])\n\n Additionally, one can pass an `offset` that must be of type |int|\n or an valid |Period| initialization argument:\n\n >>> timegrid.to_timepoints(offset=24)\n array([ 24., 30., 36., 42.])\n >>> timegrid.to_timepoints(offset='1d')\n array([ 24., 30., 36., 42.])\n >>> timegrid.to_timepoints(unit='day', offset='1d')\n array([ 1. , 1.25, 1.5 , 1.75])\n \"\"\"\n unit = Period.from_cfunits(unit)\n if offset is None:\n offset = 0.\n else:\n try:\n offset = Period(offset)/unit\n except TypeError:\n offset = offset\n step = self.stepsize/unit\n nmb = len(self)\n variable = numpy.linspace(offset, offset+step*(nmb-1), nmb)\n return variable"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef array2series(self, array):\n try:\n array = numpy.array(array, dtype=float)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to prefix timegrid information to the '\n 'given array')\n if len(array) != len(self):\n raise ValueError(\n f'When converting an array to a sequence, the lengths of the '\n f'timegrid and the given array must be equal, but the length '\n f'of the timegrid object is `{len(self)}` and the length of '\n f'the array object is `{len(array)}`.')\n shape = list(array.shape)\n shape[0] += 13\n series = numpy.full(shape, numpy.nan)\n slices = [slice(0, 13)]\n subshape = [13]\n for dummy in range(1, series.ndim):\n slices.append(slice(0, 1))\n subshape.append(1)\n series[tuple(slices)] = self.to_array().reshape(subshape)\n series[13:] = array\n return series", "response": "Given an array of timegrid objects and a timegrid object return a new array of timegrid objects."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef verify(self):\n if self.firstdate >= self.lastdate:\n raise ValueError(\n f'Unplausible timegrid. The first given date '\n f'{self.firstdate}, the second given date is {self.lastdate}.')\n if (self.lastdate-self.firstdate) % self.stepsize:\n raise ValueError(\n f'Unplausible timegrid. The period span between the given '\n f'dates {self.firstdate} and {self.lastdate} is not '\n f'a multiple of the given step size {self.stepsize}.')", "response": "Raise an |ValueError| if the dates or step size of the time\n frame are inconsistent."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a |repr| string with an prefixed assignement.", "response": "def assignrepr(self, prefix, style=None, utcoffset=None):\n \"\"\"Return a |repr| string with an prefixed assignement.\n\n Without option arguments given, printing the returned string\n looks like:\n\n >>> from hydpy import Timegrid\n >>> timegrid = Timegrid('1996-11-01 00:00:00',\n ... '1997-11-01 00:00:00',\n ... '1d')\n >>> print(timegrid.assignrepr(prefix='timegrid = '))\n timegrid = Timegrid('1996-11-01 00:00:00',\n '1997-11-01 00:00:00',\n '1d')\n\n The optional arguments are passed to method |Date.to_repr|\n without any modifications:\n\n >>> print(timegrid.assignrepr(\n ... prefix='', style='iso1', utcoffset=120))\n Timegrid('1996-11-01T01:00:00+02:00',\n '1997-11-01T01:00:00+02:00',\n '1d')\n \"\"\"\n skip = len(prefix) + 9\n blanks = ' ' * skip\n return (f\"{prefix}Timegrid('\"\n f\"{self.firstdate.to_string(style, utcoffset)}',\\n\"\n f\"{blanks}'{self.lastdate.to_string(style, utcoffset)}',\\n\"\n f\"{blanks}'{str(self.stepsize)}')\")"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef verify(self):\n self.init.verify()\n self.sim.verify()\n if self.init.firstdate > self.sim.firstdate:\n raise ValueError(\n f'The first date of the initialisation period '\n f'({self.init.firstdate}) must not be later '\n f'than the first date of the simulation period '\n f'({self.sim.firstdate}).')\n elif self.init.lastdate < self.sim.lastdate:\n raise ValueError(\n f'The last date of the initialisation period '\n f'({self.init.lastdate}) must not be earlier '\n f'than the last date of the simulation period '\n f'({self.sim.lastdate}).')\n elif self.init.stepsize != self.sim.stepsize:\n raise ValueError(\n f'The initialization stepsize ({self.init.stepsize}) '\n f'must be identical with the simulation stepsize '\n f'({self.sim.stepsize}).')\n else:\n try:\n self.init[self.sim.firstdate]\n except ValueError:\n raise ValueError(\n 'The simulation time grid is not properly '\n 'alligned on the initialization time grid.')", "response": "Raise an |ValueError| it the different time grids are not inconsistent."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a |repr| string with a prefixed assignment.", "response": "def assignrepr(self, prefix):\n \"\"\"Return a |repr| string with a prefixed assignment.\"\"\"\n caller = 'Timegrids('\n blanks = ' ' * (len(prefix) + len(caller))\n prefix = f'{prefix}{caller}'\n lines = [f'{self.init.assignrepr(prefix)},']\n if self.sim != self.init:\n lines.append(f'{self.sim.assignrepr(blanks)},')\n lines[-1] = lines[-1][:-1] + ')'\n return '\\n'.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\namounts of time passed since the beginning of the year.", "response": "def seconds_passed(self):\n \"\"\"Amount of time passed in seconds since the beginning of the year.\n\n In the first example, the year is only one minute and thirty seconds\n old:\n\n >>> from hydpy.core.timetools import TOY\n >>> TOY('1_1_0_1_30').seconds_passed\n 90\n\n The second example shows that the 29th February is generally included:\n\n >>> TOY('3').seconds_passed\n 5184000\n \"\"\"\n return int((Date(self).datetime -\n self._STARTDATE.datetime).total_seconds())"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the number of seconds left in the year.", "response": "def seconds_left(self):\n \"\"\"Remaining part of the year in seconds.\n\n In the first example, only one minute and thirty seconds of the year\n remain:\n\n >>> from hydpy.core.timetools import TOY\n >>> TOY('12_31_23_58_30').seconds_left\n 90\n\n The second example shows that the 29th February is generally included:\n\n >>> TOY('2').seconds_left\n 28944000\n \"\"\"\n return int((self._ENDDATE.datetime -\n Date(self).datetime).total_seconds())"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a |Timegrid| object defining the central time points for the given simulation step.", "response": "def centred_timegrid(cls, simulationstep):\n \"\"\"Return a |Timegrid| object defining the central time points\n of the year 2000 for the given simulation step.\n\n >>> from hydpy.core.timetools import TOY\n >>> TOY.centred_timegrid('1d')\n Timegrid('2000-01-01 12:00:00',\n '2001-01-01 12:00:00',\n '1d')\n \"\"\"\n simulationstep = Period(simulationstep)\n return Timegrid(\n cls._STARTDATE+simulationstep/2,\n cls._ENDDATE+simulationstep/2,\n simulationstep)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the class name of the given object or class.", "response": "def classname(self):\n \"\"\"Return the class name of the given instance object or class.\n\n >>> from hydpy.core.objecttools import classname\n >>> from hydpy import pub\n >>> print(classname(float))\n float\n >>> print(classname(pub.options))\n Options\n \"\"\"\n if inspect.isclass(self):\n string = str(self)\n else:\n string = str(type(self))\n try:\n string = string.split(\"'\")[1]\n except IndexError:\n pass\n return string.split('.')[-1]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef name(self):\n cls = type(self)\n try:\n return cls.__dict__['_name']\n except KeyError:\n setattr(cls, '_name', instancename(self))\n return cls.__dict__['_name']", "response": "Name of the class of the given instance in lower case letters."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef valid_variable_identifier(string):\n string = str(string)\n try:\n exec('%s = None' % string)\n if string in dir(builtins):\n raise SyntaxError()\n except SyntaxError:\n raise ValueError(\n 'The given name string `%s` does not define a valid variable '\n 'identifier. Valid identifiers do not contain characters like '\n '`-` or empty spaces, do not start with numbers, cannot be '\n 'mistaken with Python built-ins like `for`...)'\n % string)", "response": "Raises |ValueError| if the given string is not a valid Python\n identifier."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef augment_excmessage(prefix=None, suffix=None) -> NoReturn:\n exc_old = sys.exc_info()[1]\n message = str(exc_old)\n if prefix is not None:\n message = f'{prefix}, the following error occurred: {message}'\n if suffix is not None:\n message = f'{message} {suffix}'\n try:\n exc_new = type(exc_old)(message)\n except BaseException:\n exc_name = str(type(exc_old)).split(\"'\")[1]\n exc_type = type(exc_name, (BaseException,), {})\n exc_type.__module = exc_old.__module__\n raise exc_type(message) from exc_old\n raise exc_new from exc_old", "response": "Augment an exception message with additional information while keeping the original traceback."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef excmessage_decorator(description) -> Callable:\n @wrapt.decorator\n def wrapper(wrapped, instance, args, kwargs):\n \"\"\"Apply |augment_excmessage| when the wrapped function fails.\"\"\"\n # pylint: disable=unused-argument\n try:\n return wrapped(*args, **kwargs)\n except BaseException:\n info = kwargs.copy()\n info['self'] = instance\n argnames = inspect.getfullargspec(wrapped).args\n if argnames[0] == 'self':\n argnames = argnames[1:]\n for argname, arg in zip(argnames, args):\n info[argname] = arg\n for argname in argnames:\n if argname not in info:\n info[argname] = '?'\n message = eval(\n f\"f'While trying to {description}'\", globals(), info)\n augment_excmessage(message)\n return wrapper", "response": "A decorator to apply an exception message to a non - numbered sequence of exceptions."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprint the given values in multiple lines with a certain maximum width.", "response": "def print_values(values, width=70):\n \"\"\"Print the given values in multiple lines with a certain maximum width.\n\n By default, each line contains at most 70 characters:\n\n >>> from hydpy import print_values\n >>> print_values(range(21))\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,\n 20\n\n You can change this default behaviour by passing an alternative\n number of characters:\n\n >>> print_values(range(21), width=30)\n 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,\n 10, 11, 12, 13, 14, 15, 16,\n 17, 18, 19, 20\n \"\"\"\n for line in textwrap.wrap(repr_values(values), width=width):\n print(line)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef assignrepr_values(values, prefix, width=None, _fakeend=0):\n ellipsis_ = hydpy.pub.options.ellipsis\n if (ellipsis_ > 0) and (len(values) > 2*ellipsis_):\n string = (repr_values(values[:ellipsis_]) +\n ', ...,' +\n repr_values(values[-ellipsis_:]))\n else:\n string = repr_values(values)\n blanks = ' '*len(prefix)\n if width is None:\n wrapped = [string]\n _fakeend = 0\n else:\n width -= len(prefix)\n wrapped = textwrap.wrap(string+'_'*_fakeend, width)\n if not wrapped:\n wrapped = ['']\n lines = []\n for (idx, line) in enumerate(wrapped):\n if idx == 0:\n lines.append('%s%s' % (prefix, line))\n else:\n lines.append('%s%s' % (blanks, line))\n string = '\\n'.join(lines)\n return string[:len(string)-_fakeend]", "response": "Return a prefixed and properly aligned string representation of the given values using function |repr|."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a prefixed and properly aligned string representation of the given 2 - dimensional value matrix using function |repr|.", "response": "def assignrepr_values2(values, prefix):\n \"\"\"Return a prefixed and properly aligned string representation\n of the given 2-dimensional value matrix using function |repr|.\n\n >>> from hydpy.core.objecttools import assignrepr_values2\n >>> import numpy\n >>> print(assignrepr_values2(numpy.eye(3), 'test(') + ')')\n test(1.0, 0.0, 0.0,\n 0.0, 1.0, 0.0,\n 0.0, 0.0, 1.0)\n\n Functions |assignrepr_values2| works also on empty iterables:\n\n >>> print(assignrepr_values2([[]], 'test(') + ')')\n test()\n \"\"\"\n lines = []\n blanks = ' '*len(prefix)\n for (idx, subvalues) in enumerate(values):\n if idx == 0:\n lines.append('%s%s,' % (prefix, repr_values(subvalues)))\n else:\n lines.append('%s%s,' % (blanks, repr_values(subvalues)))\n lines[-1] = lines[-1][:-1]\n return '\\n'.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a prefixed and properly aligned bracketed string representation of the given 2 - dimensional value matrix using function |repr|.", "response": "def _assignrepr_bracketed2(assignrepr_bracketed1, values, prefix, width=None):\n \"\"\"Return a prefixed, wrapped and properly aligned bracketed string\n representation of the given 2-dimensional value matrix using function\n |repr|.\"\"\"\n brackets = getattr(assignrepr_bracketed1, '_brackets')\n prefix += brackets[0]\n lines = []\n blanks = ' '*len(prefix)\n for (idx, subvalues) in enumerate(values):\n if idx == 0:\n lines.append(assignrepr_bracketed1(subvalues, prefix, width))\n else:\n lines.append(assignrepr_bracketed1(subvalues, blanks, width))\n lines[-1] += ','\n if (len(values) > 1) or (brackets != '()'):\n lines[-1] = lines[-1][:-1]\n lines[-1] += brackets[1]\n return '\\n'.join(lines)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nprint values with a maximum number of digits in doctests.", "response": "def round_(values, decimals=None, width=0,\n lfill=None, rfill=None, **kwargs):\n \"\"\"Prints values with a maximum number of digits in doctests.\n\n See the documentation on function |repr| for more details. And\n note thate the option keyword arguments are passed to the print function.\n\n Usually one would apply function |round_| on a single or a vector\n of numbers:\n\n >>> from hydpy import round_\n >>> round_(1./3., decimals=6)\n 0.333333\n >>> round_((1./2., 1./3., 1./4.), decimals=4)\n 0.5, 0.3333, 0.25\n\n Additionally, one can supply a `width` and a `rfill` argument:\n >>> round_(1.0, width=6, rfill='0')\n 1.0000\n\n Alternatively, one can use the `lfill` arguments, which\n might e.g. be usefull for aligning different strings:\n\n >>> round_('test', width=6, lfill='_')\n __test\n\n Using both the `lfill` and the `rfill` argument raises an error:\n\n >>> round_(1.0, lfill='_', rfill='0')\n Traceback (most recent call last):\n ...\n ValueError: For function `round_` values are passed for both \\\narguments `lfill` and `rfill`. This is not allowed.\n \"\"\"\n if decimals is None:\n decimals = hydpy.pub.options.reprdigits\n with hydpy.pub.options.reprdigits(decimals):\n if isinstance(values, abctools.IterableNonStringABC):\n string = repr_values(values)\n else:\n string = repr_(values)\n if (lfill is not None) and (rfill is not None):\n raise ValueError(\n 'For function `round_` values are passed for both arguments '\n '`lfill` and `rfill`. This is not allowed.')\n if (lfill is not None) or (rfill is not None):\n width = max(width, len(string))\n if lfill is not None:\n string = string.rjust(width, lfill)\n else:\n string = string.ljust(width, rfill)\n print(string, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef extract(values, types, skip=False):\n if isinstance(values, types):\n yield values\n elif skip and (values is None):\n return\n else:\n try:\n for value in values:\n for subvalue in extract(value, types, skip):\n yield subvalue\n except TypeError as exc:\n if exc.args[0].startswith('The given value'):\n raise exc\n else:\n raise TypeError(\n f'The given value `{repr(values)}` is neither iterable '\n f'nor an instance of the following classes: '\n f'{enumeration(types, converter=instancename)}.')", "response": "Return a generator that extracts certain objects from values."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef enumeration(values, converter=str, default=''):\n values = tuple(converter(value) for value in values)\n if not values:\n return default\n if len(values) == 1:\n return values[0]\n if len(values) == 2:\n return ' and '.join(values)\n return ', and '.join((', '.join(values[:-1]), values[-1]))", "response": "Return an enumeration string based on the given values."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntrim upper values in accordance with IC \\\\ leq ICMAX.", "response": "def trim(self, lower=None, upper=None):\n \"\"\"Trim upper values in accordance with :math:`IC \\\\leq ICMAX`.\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(5)\n >>> icmax(2.0)\n >>> states.ic(-1.0, 0.0, 1.0, 2.0, 3.0)\n >>> states.ic\n ic(0.0, 0.0, 1.0, 2.0, 2.0)\n \"\"\"\n if upper is None:\n control = self.subseqs.seqs.model.parameters.control\n upper = control.icmax\n hland_sequences.State1DSequence.trim(self, lower, upper)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ntrims values in accordance with WC \\\\ leq WHC \\\\ cdot SP.", "response": "def trim(self, lower=None, upper=None):\n \"\"\"Trim values in accordance with :math:`WC \\\\leq WHC \\\\cdot SP`.\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(7)\n >>> whc(0.1)\n >>> states.wc.values = -1.0, 0.0, 1.0, -1.0, 0.0, 0.5, 1.0\n >>> states.sp(-1., 0., 0., 5., 5., 5., 5.)\n >>> states.sp\n sp(0.0, 0.0, 10.0, 5.0, 5.0, 5.0, 10.0)\n \"\"\"\n whc = self.subseqs.seqs.model.parameters.control.whc\n wc = self.subseqs.wc\n if lower is None:\n if wc.values is not None:\n with numpy.errstate(divide='ignore', invalid='ignore'):\n lower = numpy.clip(wc.values / whc.values, 0., numpy.inf)\n else:\n lower = 0.\n hland_sequences.State1DSequence.trim(self, lower, upper)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntrimming values in accordance with WC \\\\ leq WHC \\\\ cdot SP.", "response": "def trim(self, lower=None, upper=None):\n \"\"\"Trim values in accordance with :math:`WC \\\\leq WHC \\\\cdot SP`.\n\n >>> from hydpy.models.hland import *\n >>> parameterstep('1d')\n >>> nmbzones(7)\n >>> whc(0.1)\n >>> states.sp = 0.0, 0.0, 0.0, 5.0, 5.0, 5.0, 5.0\n >>> states.wc(-1.0, 0.0, 1.0, -1.0, 0.0, 0.5, 1.0)\n >>> states.wc\n wc(0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.5)\n \"\"\"\n whc = self.subseqs.seqs.model.parameters.control.whc\n sp = self.subseqs.sp\n if (upper is None) and (sp.values is not None):\n upper = whc*sp\n hland_sequences.State1DSequence.trim(self, lower, upper)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef trim(self, lower=None, upper=None):\n if upper is None:\n control = self.subseqs.seqs.model.parameters.control\n if not any(control.zonetype.values == ILAKE):\n lower = 0.\n sequencetools.StateSequence.trim(self, lower, upper)", "response": "Trim negative value whenever there is no internal lake within\n the respective subbasin."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef load_data(self, idx):\n for subseqs in self:\n if isinstance(subseqs, abctools.InputSequencesABC):\n subseqs.load_data(idx)", "response": "Call method |InputSequences. load_data| of all handled\n |InputSequencesABC| objects."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef save_data(self, idx):\n for subseqs in self:\n if isinstance(subseqs, abctools.OutputSequencesABC):\n subseqs.save_data(idx)", "response": "Call method save_data| of all handled |IOSequences| objects registered under |OutputSequencesABC|."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef conditions(self) -> Dict[str, Dict[str, Union[float, numpy.ndarray]]]:\n conditions = {}\n for subname in NAMES_CONDITIONSEQUENCES:\n subseqs = getattr(self, subname, ())\n subconditions = {seq.name: copy.deepcopy(seq.values)\n for seq in subseqs}\n if subconditions:\n conditions[subname] = subconditions\n return conditions", "response": "Nested dictionary containing the values of all condition sequences."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_conditions(self, filename=None):\n if self.hasconditions:\n if not filename:\n filename = self._conditiondefaultfilename\n namespace = locals()\n for seq in self.conditionsequences:\n namespace[seq.name] = seq\n namespace['model'] = self\n code = hydpy.pub.conditionmanager.load_file(filename)\n try:\n # ToDo: raises an escape sequence deprecation sometimes\n # ToDo: use runpy instead?\n # ToDo: Move functionality to filetools.py?\n exec(code)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to gather initial conditions of element %s'\n % objecttools.devicename(self))", "response": "Load the initial conditions from a file and assign them to the state sequence and log sequence objects handled by the actual |Sequences| object."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsaving the actual conditions of the state sequence and or log sequence objects in a file.", "response": "def save_conditions(self, filename=None):\n \"\"\"Query the actual conditions of the |StateSequence| and/or\n |LogSequence| objects handled by the actual |Sequences| object and\n write them into a initial condition file.\n\n If no filename or dirname is passed, the ones defined by the\n |ConditionManager| stored in module |pub| are used.\n \"\"\"\n if self.hasconditions:\n if filename is None:\n filename = self._conditiondefaultfilename\n con = hydpy.pub.controlmanager\n lines = ['# -*- coding: utf-8 -*-\\n\\n',\n 'from hydpy.models.%s import *\\n\\n' % self.model,\n 'controlcheck(projectdir=\"%s\", controldir=\"%s\")\\n\\n'\n % (con.projectdir, con.currentdir)]\n for seq in self.conditionsequences:\n lines.append(repr(seq) + '\\n')\n hydpy.pub.conditionmanager.save_file(filename, ''.join(lines))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmove internal data from disk to RAM.", "response": "def disk2ram(self):\n \"\"\"Move internal data from disk to RAM.\"\"\"\n values = self.series\n self.deactivate_disk()\n self.ramflag = True\n self.__set_array(values)\n self.update_fastaccess()"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmoves internal data from RAM to disk.", "response": "def ram2disk(self):\n \"\"\"Move internal data from RAM to disk.\"\"\"\n values = self.series\n self.deactivate_ram()\n self.diskflag = True\n self._save_int(values)\n self.update_fastaccess()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef seriesshape(self):\n seriesshape = [len(hydpy.pub.timegrids.init)]\n seriesshape.extend(self.shape)\n return tuple(seriesshape)", "response": "Shape of the whole time series."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nshaping of the array of temporary values required for the numerical solver actually being selected.", "response": "def numericshape(self):\n \"\"\"Shape of the array of temporary values required for the numerical\n solver actually being selected.\"\"\"\n try:\n numericshape = [self.subseqs.seqs.model.numconsts.nmb_stages]\n except AttributeError:\n objecttools.augment_excmessage(\n 'The `numericshape` of a sequence like `%s` depends on the '\n 'configuration of the actual integration algorithm. '\n 'While trying to query the required configuration data '\n '`nmb_stages` of the model associated with element `%s`'\n % (self.name, objecttools.devicename(self)))\n # noinspection PyUnboundLocalVariable\n numericshape.extend(self.shape)\n return tuple(numericshape)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef series(self) -> InfoArray:\n if self.diskflag:\n array = self._load_int()\n elif self.ramflag:\n array = self.__get_array()\n else:\n raise AttributeError(\n f'Sequence {objecttools.devicephrase(self)} is not requested '\n f'to make any internal data available to the user.')\n return InfoArray(array, info={'type': 'unmodified'})", "response": "Internal time series data within an |numpy. ndarray|."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef load_ext(self):\n try:\n sequencemanager = hydpy.pub.sequencemanager\n except AttributeError:\n raise RuntimeError(\n 'The time series of sequence %s cannot be loaded. Firstly, '\n 'you have to prepare `pub.sequencemanager` correctly.'\n % objecttools.devicephrase(self))\n sequencemanager.load_file(self)", "response": "Read the internal data from an external data file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadjust a short time series to a longer timegrid.", "response": "def adjust_short_series(self, timegrid, values):\n \"\"\"Adjust a short time series to a longer timegrid.\n\n Normally, time series data to be read from a external data files\n should span (at least) the whole initialization time period of a\n HydPy project. However, for some variables which are only used\n for comparison (e.g. observed runoff used for calibration),\n incomplete time series might also be helpful. This method it\n thought for adjusting such incomplete series to the public\n initialization time grid stored in module |pub|. It is\n automatically called in method |IOSequence.adjust_series| when\n necessary provided that the option |Options.checkseries| is\n disabled.\n\n Assume the initialization time period of a HydPy project spans\n five day:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2000.01.10', '2000.01.15', '1d'\n\n Prepare a node series object for observational data:\n\n >>> from hydpy.core.sequencetools import Obs\n >>> obs = Obs(None)\n\n Prepare a test function that expects the timegrid of the\n data and the data itself, which returns the ajdusted array by\n means of calling method |IOSequence.adjust_short_series|:\n\n >>> import numpy\n >>> def test(timegrid):\n ... values = numpy.ones(len(timegrid))\n ... return obs.adjust_short_series(timegrid, values)\n\n The following calls to the test function shows the arrays\n returned for different kinds of misalignments:\n\n >>> from hydpy import Timegrid\n >>> test(Timegrid('2000.01.05', '2000.01.20', '1d'))\n array([ 1., 1., 1., 1., 1.])\n >>> test(Timegrid('2000.01.12', '2000.01.15', '1d'))\n array([ nan, nan, 1., 1., 1.])\n >>> test(Timegrid('2000.01.12', '2000.01.17', '1d'))\n array([ nan, nan, 1., 1., 1.])\n >>> test(Timegrid('2000.01.10', '2000.01.13', '1d'))\n array([ 1., 1., 1., nan, nan])\n >>> test(Timegrid('2000.01.08', '2000.01.13', '1d'))\n array([ 1., 1., 1., nan, nan])\n >>> test(Timegrid('2000.01.12', '2000.01.13', '1d'))\n array([ nan, nan, 1., nan, nan])\n >>> test(Timegrid('2000.01.05', '2000.01.10', '1d'))\n array([ nan, nan, nan, nan, nan])\n >>> test(Timegrid('2000.01.05', '2000.01.08', '1d'))\n array([ nan, nan, nan, nan, nan])\n >>> test(Timegrid('2000.01.15', '2000.01.18', '1d'))\n array([ nan, nan, nan, nan, nan])\n >>> test(Timegrid('2000.01.16', '2000.01.18', '1d'))\n array([ nan, nan, nan, nan, nan])\n\n Through enabling option |Options.usedefaultvalues| the missing\n values are initialised with zero instead of nan:\n\n >>> with pub.options.usedefaultvalues(True):\n ... test(Timegrid('2000.01.12', '2000.01.17', '1d'))\n array([ 0., 0., 1., 1., 1.])\n \"\"\"\n idxs = [timegrid[hydpy.pub.timegrids.init.firstdate],\n timegrid[hydpy.pub.timegrids.init.lastdate]]\n valcopy = values\n values = numpy.full(self.seriesshape, self.initinfo[0])\n len_ = len(valcopy)\n jdxs = []\n for idx in idxs:\n if idx < 0:\n jdxs.append(0)\n elif idx <= len_:\n jdxs.append(idx)\n else:\n jdxs.append(len_)\n valcopy = valcopy[jdxs[0]:jdxs[1]]\n zdx1 = max(-idxs[0], 0)\n zdx2 = zdx1+jdxs[1]-jdxs[0]\n values[zdx1:zdx2] = valcopy\n return values"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef check_completeness(self):\n if hydpy.pub.options.checkseries:\n isnan = numpy.isnan(self.series)\n if numpy.any(isnan):\n nmb = numpy.sum(isnan)\n valuestring = 'value' if nmb == 1 else 'values'\n raise RuntimeError(\n f'The series array of sequence '\n f'{objecttools.devicephrase(self)} contains '\n f'{nmb} nan {valuestring}.')", "response": "Raise a |RuntimeError| if the |IOSequence. series| contains at least one |numpy. nan| value."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_ext(self):\n try:\n sequencemanager = hydpy.pub.sequencemanager\n except AttributeError:\n raise RuntimeError(\n 'The time series of sequence %s cannot be saved. Firstly,'\n 'you have to prepare `pub.sequencemanager` correctly.'\n % objecttools.devicephrase(self))\n sequencemanager.save_file(self)", "response": "Write the internal data into an external data file."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _load_int(self):\n values = numpy.fromfile(self.filepath_int)\n if self.NDIM > 0:\n values = values.reshape(self.seriesshape)\n return values", "response": "Load internal data from file and return it."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\naveraging the actual time series of the |Variable| object for all time points. Method |IOSequence.average_series| works similarly as method |Variable.average_values| of class |Variable|, from which we borrow some examples. However, firstly, we have to prepare a |Timegrids| object to define the |IOSequence.series| length: >>> from hydpy import pub >>> pub.timegrids = '2000-01-01', '2000-01-04', '1d' As shown for method |Variable.average_values|, for 0-dimensional |IOSequence| objects the result of |IOSequence.average_series| equals |IOSequence.series| itself: >>> from hydpy.core.sequencetools import IOSequence >>> class SoilMoisture(IOSequence): ... NDIM = 0 >>> sm = SoilMoisture(None) >>> sm.activate_ram() >>> import numpy >>> sm.series = numpy.array([190.0, 200.0, 210.0]) >>> sm.average_series() InfoArray([ 190., 200., 210.]) For |IOSequence| objects with an increased dimensionality, a weighting parameter is required, again: >>> SoilMoisture.NDIM = 1 >>> sm.shape = 3 >>> sm.activate_ram() >>> sm.series = ( ... [190.0, 390.0, 490.0], ... [200.0, 400.0, 500.0], ... [210.0, 410.0, 510.0]) >>> from hydpy.core.parametertools import Parameter >>> class Area(Parameter): ... NDIM = 1 ... shape = (3,) ... value = numpy.array([1.0, 1.0, 2.0]) >>> area = Area(None) >>> SoilMoisture.refweights = property(lambda self: area) >>> sm.average_series() InfoArray([ 390., 400., 410.]) The documentation on method |Variable.average_values| provides many examples on how to use different masks in different ways. Here we restrict ourselves to the first example, where a new mask enforces that |IOSequence.average_series| takes only the first two columns of the `series` into account: >>> from hydpy.core.masktools import DefaultMask >>> class Soil(DefaultMask): ... @classmethod ... def new(cls, variable, **kwargs): ... return cls.array2mask([True, True, False]) >>> SoilMoisture.mask = Soil() >>> sm.average_series() InfoArray([ 290., 300., 310.])", "response": "def average_series(self, *args, **kwargs) -> InfoArray:\n \"\"\"Average the actual time series of the |Variable| object for all\n time points.\n\n Method |IOSequence.average_series| works similarly as method\n |Variable.average_values| of class |Variable|, from which we\n borrow some examples. However, firstly, we have to prepare a\n |Timegrids| object to define the |IOSequence.series| length:\n\n >>> from hydpy import pub\n >>> pub.timegrids = '2000-01-01', '2000-01-04', '1d'\n\n As shown for method |Variable.average_values|, for 0-dimensional\n |IOSequence| objects the result of |IOSequence.average_series|\n equals |IOSequence.series| itself:\n\n >>> from hydpy.core.sequencetools import IOSequence\n >>> class SoilMoisture(IOSequence):\n ... NDIM = 0\n >>> sm = SoilMoisture(None)\n >>> sm.activate_ram()\n >>> import numpy\n >>> sm.series = numpy.array([190.0, 200.0, 210.0])\n >>> sm.average_series()\n InfoArray([ 190., 200., 210.])\n\n For |IOSequence| objects with an increased dimensionality, a\n weighting parameter is required, again:\n\n >>> SoilMoisture.NDIM = 1\n >>> sm.shape = 3\n >>> sm.activate_ram()\n >>> sm.series = (\n ... [190.0, 390.0, 490.0],\n ... [200.0, 400.0, 500.0],\n ... [210.0, 410.0, 510.0])\n >>> from hydpy.core.parametertools import Parameter\n >>> class Area(Parameter):\n ... NDIM = 1\n ... shape = (3,)\n ... value = numpy.array([1.0, 1.0, 2.0])\n >>> area = Area(None)\n >>> SoilMoisture.refweights = property(lambda self: area)\n >>> sm.average_series()\n InfoArray([ 390., 400., 410.])\n\n The documentation on method |Variable.average_values| provides\n many examples on how to use different masks in different ways.\n Here we restrict ourselves to the first example, where a new\n mask enforces that |IOSequence.average_series| takes only the\n first two columns of the `series` into account:\n\n >>> from hydpy.core.masktools import DefaultMask\n >>> class Soil(DefaultMask):\n ... @classmethod\n ... def new(cls, variable, **kwargs):\n ... return cls.array2mask([True, True, False])\n >>> SoilMoisture.mask = Soil()\n >>> sm.average_series()\n InfoArray([ 290., 300., 310.])\n \"\"\"\n try:\n if not self.NDIM:\n array = self.series\n else:\n mask = self.get_submask(*args, **kwargs)\n if numpy.any(mask):\n weights = self.refweights[mask]\n weights /= numpy.sum(weights)\n series = self.series[:, mask]\n axes = tuple(range(1, self.NDIM+1))\n array = numpy.sum(weights*series, axis=axes)\n else:\n return numpy.nan\n return InfoArray(array, info={'type': 'mean'})\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to calculate the mean value of '\n 'the internal time series of sequence %s'\n % objecttools.devicephrase(self))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\naggregate time series data based on the actual |FluxSequence.aggregation_ext| attribute of |IOSequence| subclasses. We prepare some nodes and elements with the help of method |prepare_io_example_1| and select a 1-dimensional flux sequence of type |lland_fluxes.NKor| as an example: >>> from hydpy.core.examples import prepare_io_example_1 >>> nodes, elements = prepare_io_example_1() >>> seq = elements.element3.model.sequences.fluxes.nkor If no |FluxSequence.aggregation_ext| is `none`, the original time series values are returned: >>> seq.aggregation_ext 'none' >>> seq.aggregate_series() InfoArray([[ 24., 25., 26.], [ 27., 28., 29.], [ 30., 31., 32.], [ 33., 34., 35.]]) If no |FluxSequence.aggregation_ext| is `mean`, function |IOSequence.aggregate_series| is called: >>> seq.aggregation_ext = 'mean' >>> seq.aggregate_series() InfoArray([ 25., 28., 31., 34.]) In case the state of the sequence is invalid: >>> seq.aggregation_ext = 'nonexistent' >>> seq.aggregate_series() Traceback (most recent call last): ... RuntimeError: Unknown aggregation mode `nonexistent` for \\ sequence `nkor` of element `element3`. The following technical test confirms that all potential positional and keyword arguments are passed properly: >>> seq.aggregation_ext = 'mean' >>> from unittest import mock >>> seq.average_series = mock.MagicMock() >>> _ = seq.aggregate_series(1, x=2) >>> seq.average_series.assert_called_with(1, x=2)", "response": "def aggregate_series(self, *args, **kwargs) -> InfoArray:\n \"\"\"Aggregates time series data based on the actual\n |FluxSequence.aggregation_ext| attribute of |IOSequence|\n subclasses.\n\n We prepare some nodes and elements with the help of\n method |prepare_io_example_1| and select a 1-dimensional\n flux sequence of type |lland_fluxes.NKor| as an example:\n\n >>> from hydpy.core.examples import prepare_io_example_1\n >>> nodes, elements = prepare_io_example_1()\n >>> seq = elements.element3.model.sequences.fluxes.nkor\n\n If no |FluxSequence.aggregation_ext| is `none`, the\n original time series values are returned:\n\n >>> seq.aggregation_ext\n 'none'\n >>> seq.aggregate_series()\n InfoArray([[ 24., 25., 26.],\n [ 27., 28., 29.],\n [ 30., 31., 32.],\n [ 33., 34., 35.]])\n\n If no |FluxSequence.aggregation_ext| is `mean`, function\n |IOSequence.aggregate_series| is called:\n\n >>> seq.aggregation_ext = 'mean'\n >>> seq.aggregate_series()\n InfoArray([ 25., 28., 31., 34.])\n\n In case the state of the sequence is invalid:\n\n >>> seq.aggregation_ext = 'nonexistent'\n >>> seq.aggregate_series()\n Traceback (most recent call last):\n ...\n RuntimeError: Unknown aggregation mode `nonexistent` for \\\nsequence `nkor` of element `element3`.\n\n The following technical test confirms that all potential\n positional and keyword arguments are passed properly:\n >>> seq.aggregation_ext = 'mean'\n\n >>> from unittest import mock\n >>> seq.average_series = mock.MagicMock()\n >>> _ = seq.aggregate_series(1, x=2)\n >>> seq.average_series.assert_called_with(1, x=2)\n \"\"\"\n mode = self.aggregation_ext\n if mode == 'none':\n return self.series\n elif mode == 'mean':\n return self.average_series(*args, **kwargs)\n else:\n raise RuntimeError(\n 'Unknown aggregation mode `%s` for sequence %s.'\n % (mode, objecttools.devicephrase(self)))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef old(self):\n value = getattr(self.fastaccess_old, self.name, None)\n if value is None:\n raise RuntimeError(\n 'No value/values of sequence %s has/have '\n 'not been defined so far.'\n % objecttools.elementphrase(self))\n else:\n if self.NDIM:\n value = numpy.asarray(value)\n return value", "response": "Assess to the state value at beginning of the time step which\n has been processed most recently."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading the external data of the time series of the given class.", "response": "def load_ext(self):\n \"\"\"Read time series data like method |IOSequence.load_ext| of class\n |IOSequence|, but with special handling of missing data.\n\n The method's \"special handling\" is to convert errors to warnings.\n We explain the reasons in the documentation on method |Obs.load_ext|\n of class |Obs|, from which we borrow the following examples.\n The only differences are that method |Sim.load_ext| of class |Sim|\n does not disable property |IOSequence.memoryflag| and uses option\n |Options.warnmissingsimfile| instead of |Options.warnmissingobsfile|:\n\n >>> from hydpy.core.examples import prepare_full_example_1\n >>> prepare_full_example_1()\n >>> from hydpy import HydPy, pub, TestIO\n >>> hp = HydPy('LahnH')\n >>> pub.timegrids = '1996-01-01', '1996-01-06', '1d'\n >>> with TestIO():\n ... hp.prepare_network()\n ... hp.init_models()\n ... hp.prepare_simseries()\n >>> sim = hp.nodes.dill.sequences.sim\n >>> with TestIO():\n ... sim.load_ext() # doctest: +ELLIPSIS\n Traceback (most recent call last):\n ...\n UserWarning: While trying to load the external data of sequence \\\n`sim` of node `dill`, the following error occurred: [Errno 2] No such file \\\nor directory: '...dill_sim_q.asc'\n >>> sim.series\n InfoArray([ nan, nan, nan, nan, nan])\n\n >>> sim.series = 1.0\n >>> with TestIO():\n ... sim.save_ext()\n >>> sim.series = 0.0\n >>> with TestIO():\n ... sim.load_ext()\n >>> sim.series\n InfoArray([ 1., 1., 1., 1., 1.])\n\n >>> import numpy\n >>> sim.series[2] = numpy.nan\n >>> with TestIO():\n ... pub.sequencemanager.nodeoverwrite = True\n ... sim.save_ext()\n >>> with TestIO():\n ... sim.load_ext()\n Traceback (most recent call last):\n ...\n UserWarning: While trying to load the external data of sequence `sim` \\\nof node `dill`, the following error occurred: The series array of sequence \\\n`sim` of node `dill` contains 1 nan value.\n >>> sim.series\n InfoArray([ 1., 1., nan, 1., 1.])\n\n >>> sim.series = 0.0\n >>> with TestIO():\n ... with pub.options.warnmissingsimfile(False):\n ... sim.load_ext()\n >>> sim.series\n InfoArray([ 1., 1., nan, 1., 1.])\n \"\"\"\n try:\n super().load_ext()\n except BaseException:\n if hydpy.pub.options.warnmissingsimfile:\n warnings.warn(str(sys.exc_info()[1]))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread external time series data from file.", "response": "def load_ext(self):\n \"\"\"Read time series data like method |IOSequence.load_ext| of class\n |IOSequence|, but with special handling of missing data.\n\n When reading incomplete time series data, *HydPy* usually raises\n a |RuntimeError| to prevent from performing erroneous calculations.\n For instance, this makes sense for meteorological input data, being a\n definite requirement for hydrological simulations. However, the\n same often does not hold for the time series of |Obs| sequences,\n e.g. representing measured discharge. Measured discharge is often\n handled as an optional input value, or even used for comparison\n purposes only.\n\n According to this reasoning, *HydPy* raises (at most) a |UserWarning|\n in case of missing or incomplete external time series data of |Obs|\n sequences. The following examples show this based on the `LahnH`\n project, mainly focussing on the |Obs| sequence of node `dill`,\n which is ready for handling time series data at the end of the\n following steps:\n\n >>> from hydpy.core.examples import prepare_full_example_1\n >>> prepare_full_example_1()\n >>> from hydpy import HydPy, pub, TestIO\n >>> hp = HydPy('LahnH')\n >>> pub.timegrids = '1996-01-01', '1996-01-06', '1d'\n >>> with TestIO():\n ... hp.prepare_network()\n ... hp.init_models()\n ... hp.prepare_obsseries()\n >>> obs = hp.nodes.dill.sequences.obs\n >>> obs.ramflag\n True\n\n Trying to read non-existing data raises the following warning\n and disables the sequence's ability to handle time series data:\n\n >>> with TestIO():\n ... hp.load_obsseries() # doctest: +ELLIPSIS\n Traceback (most recent call last):\n ...\n UserWarning: The `memory flag` of sequence `obs` of node `dill` had \\\nto be set to `False` due to the following problem: While trying to load the \\\nexternal data of sequence `obs` of node `dill`, the following error occurred: \\\n[Errno 2] No such file or directory: '...dill_obs_q.asc'\n >>> obs.ramflag\n False\n\n After writing a complete external data fine, everything works fine:\n\n >>> obs.activate_ram()\n >>> obs.series = 1.0\n >>> with TestIO():\n ... obs.save_ext()\n >>> obs.series = 0.0\n >>> with TestIO():\n ... obs.load_ext()\n >>> obs.series\n InfoArray([ 1., 1., 1., 1., 1.])\n\n Reading incomplete data also results in a warning message, but does\n not disable the |IOSequence.memoryflag|:\n\n >>> import numpy\n >>> obs.series[2] = numpy.nan\n >>> with TestIO():\n ... pub.sequencemanager.nodeoverwrite = True\n ... obs.save_ext()\n >>> with TestIO():\n ... obs.load_ext()\n Traceback (most recent call last):\n ...\n UserWarning: While trying to load the external data of sequence `obs` \\\nof node `dill`, the following error occurred: The series array of sequence \\\n`obs` of node `dill` contains 1 nan value.\n >>> obs.memoryflag\n True\n\n Option |Options.warnmissingobsfile| allows disabling the warning\n messages without altering the functionalities described above:\n\n >>> hp.prepare_obsseries()\n >>> with TestIO():\n ... with pub.options.warnmissingobsfile(False):\n ... hp.load_obsseries()\n >>> obs.series\n InfoArray([ 1., 1., nan, 1., 1.])\n >>> hp.nodes.lahn_1.sequences.obs.memoryflag\n False\n \"\"\"\n try:\n super().load_ext()\n except OSError:\n del self.memoryflag\n if hydpy.pub.options.warnmissingobsfile:\n warnings.warn(\n f'The `memory flag` of sequence '\n f'{objecttools.nodephrase(self)} had to be set to `False` '\n f'due to the following problem: {sys.exc_info()[1]}')\n except BaseException:\n if hydpy.pub.options.warnmissingobsfile:\n warnings.warn(str(sys.exc_info()[1]))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef open_files(self, idx):\n for name in self:\n if getattr(self, '_%s_diskflag' % name):\n path = getattr(self, '_%s_path' % name)\n file_ = open(path, 'rb+')\n ndim = getattr(self, '_%s_ndim' % name)\n position = 8*idx\n for idim in range(ndim):\n length = getattr(self, '_%s_length_%d' % (name, idim))\n position *= length\n file_.seek(position)\n setattr(self, '_%s_file' % name, file_)", "response": "Open all files with an activated disk flag."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef close_files(self):\n for name in self:\n if getattr(self, '_%s_diskflag' % name):\n file_ = getattr(self, '_%s_file' % name)\n file_.close()", "response": "Close all files with an activated disk flag."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads the internal data of all sequences. If the disk flag is activated then load from RAM.", "response": "def load_data(self, idx):\n \"\"\"Load the internal data of all sequences. Load from file if the\n corresponding disk flag is activated, otherwise load from RAM.\"\"\"\n for name in self:\n ndim = getattr(self, '_%s_ndim' % name)\n diskflag = getattr(self, '_%s_diskflag' % name)\n ramflag = getattr(self, '_%s_ramflag' % name)\n if diskflag:\n file_ = getattr(self, '_%s_file' % name)\n length_tot = 1\n shape = []\n for jdx in range(ndim):\n length = getattr(self, '_%s_length_%s' % (name, jdx))\n length_tot *= length\n shape.append(length)\n raw = file_.read(length_tot*8)\n values = struct.unpack(length_tot*'d', raw)\n if ndim:\n values = numpy.array(values).reshape(shape)\n else:\n values = values[0]\n elif ramflag:\n array = getattr(self, '_%s_array' % name)\n values = array[idx]\n if diskflag or ramflag:\n if ndim == 0:\n setattr(self, name, values)\n else:\n getattr(self, name)[:] = values"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_data(self, idx):\n for name in self:\n actual = getattr(self, name)\n diskflag = getattr(self, '_%s_diskflag' % name)\n ramflag = getattr(self, '_%s_ramflag' % name)\n if diskflag:\n file_ = getattr(self, '_%s_file' % name)\n ndim = getattr(self, '_%s_ndim' % name)\n length_tot = 1\n for jdx in range(ndim):\n length = getattr(self, '_%s_length_%s' % (name, jdx))\n length_tot *= length\n if ndim:\n raw = struct.pack(length_tot*'d', *actual.flatten())\n else:\n raw = struct.pack('d', actual)\n file_.write(raw)\n elif ramflag:\n array = getattr(self, '_%s_array' % name)\n array[idx] = actual", "response": "Save the internal data of all sequences with an activated flag. Write to file if the corresponding disk flag is activated ; store\n in working memory if the corresponding ram flag is activated."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload the next sim sequence value of the given index.", "response": "def load_simdata(self, idx: int) -> None:\n \"\"\"Load the next sim sequence value (of the given index).\"\"\"\n if self._sim_ramflag:\n self.sim[0] = self._sim_array[idx]\n elif self._sim_diskflag:\n raw = self._sim_file.read(8)\n self.sim[0] = struct.unpack('d', raw)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef save_simdata(self, idx: int) -> None:\n if self._sim_ramflag:\n self._sim_array[idx] = self.sim[0]\n elif self._sim_diskflag:\n raw = struct.pack('d', self.sim[0])\n self._sim_file.write(raw)", "response": "Save the last sim sequence value of the given index."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nload the next observation sequence value of the given index.", "response": "def load_obsdata(self, idx: int) -> None:\n \"\"\"Load the next obs sequence value (of the given index).\"\"\"\n if self._obs_ramflag:\n self.obs[0] = self._obs_array[idx]\n elif self._obs_diskflag:\n raw = self._obs_file.read(8)\n self.obs[0] = struct.unpack('d', raw)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update(self):\n control = self.subpars.pars.control\n self(control.ft*control.fhru)", "response": "Update |AbsFHRU| based on |FT| and |FHRU|."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update(self):\n con = self.subpars.pars.control\n self(con.hinz*con.lai)", "response": "Update |KInz| based on |HInz| and |LAI|."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update(self):\n con = self.subpars.pars.control\n self(con.relwb*con.nfk)", "response": "Update |WB| based on |RelWB| and |NFk|."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate |WZ| based on |RelWZ| and |NFk|.", "response": "def update(self):\n \"\"\"Update |WZ| based on |RelWZ| and |NFk|.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> nhru(2)\n >>> lnk(ACKER)\n >>> relwz(0.8)\n >>> nfk(100.0, 200.0)\n >>> derived.wz.update()\n >>> derived.wz\n wz(80.0, 160.0)\n \"\"\"\n con = self.subpars.pars.control\n self(con.relwz*con.nfk)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self):\n con = self.subpars.pars.control\n self(con.eqb*con.tind)", "response": "Update |KB| based on |EQB| and |TInd|."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates |KI1| based on |EQI1| and |TInd|.", "response": "def update(self):\n \"\"\"Update |KI1| based on |EQI1| and |TInd|.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> eqi1(5.0)\n >>> tind.value = 10.0\n >>> derived.ki1.update()\n >>> derived.ki1\n ki1(50.0)\n \"\"\"\n con = self.subpars.pars.control\n self(con.eqi1*con.tind)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate |KI2| based on |EQI2| and |TInd|.", "response": "def update(self):\n \"\"\"Update |KI2| based on |EQI2| and |TInd|.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> eqi2(1.0)\n >>> tind.value = 10.0\n >>> derived.ki2.update()\n >>> derived.ki2\n ki2(10.0)\n \"\"\"\n con = self.subpars.pars.control\n self(con.eqi2*con.tind)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nupdate |KD1| based on |EQD1| and |TInd|.", "response": "def update(self):\n \"\"\"Update |KD1| based on |EQD1| and |TInd|.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> eqd1(0.5)\n >>> tind.value = 10.0\n >>> derived.kd1.update()\n >>> derived.kd1\n kd1(5.0)\n \"\"\"\n con = self.subpars.pars.control\n self(con.eqd1*con.tind)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nupdate |KD2| based on |EQD2| and |TInd|.", "response": "def update(self):\n \"\"\"Update |KD2| based on |EQD2| and |TInd|.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> eqd2(0.1)\n >>> tind.value = 10.0\n >>> derived.kd2.update()\n >>> derived.kd2\n kd2(1.0)\n \"\"\"\n con = self.subpars.pars.control\n self(con.eqd2*con.tind)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating |QFactor| based on |FT| and the current simulation step size.", "response": "def update(self):\n \"\"\"Update |QFactor| based on |FT| and the current simulation step size.\n\n >>> from hydpy.models.lland import *\n >>> parameterstep('1d')\n >>> simulationstep('1d')\n >>> ft(10.0)\n >>> derived.qfactor.update()\n >>> derived.qfactor\n qfactor(0.115741)\n \"\"\"\n con = self.subpars.pars.control\n self(con.ft*1000./self.simulationstep.seconds)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _router_numbers(self):\n return tuple(up for up in self._up2down.keys()\n if up in self._up2down.values())", "response": "A tuple of the numbers of all routing basins."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef supplier_elements(self):\n elements = devicetools.Elements()\n for supplier in self._supplier_numbers:\n element = self._get_suppliername(supplier)\n try:\n outlet = self._get_nodename(self._up2down[supplier])\n except TypeError:\n outlet = self.last_node\n elements += devicetools.Element(element, outlets=outlet)\n return elements", "response": "A |Elements| collection of all supplying basins."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef nodes(self):\n return (\n devicetools.Nodes(\n self.node_prefix+routers for routers in self._router_numbers) +\n devicetools.Node(self.last_node))", "response": "A |Nodes| collection of all required nodes."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef selection(self):\n return selectiontools.Selection(\n self.selection_name, self.nodes, self.elements)", "response": "A |Selection| object of all supplying and routing nodes and required elements and required nodes."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef str2chars(strings) -> numpy.ndarray:\n maxlen = 0\n for name in strings:\n maxlen = max(maxlen, len(name))\n # noinspection PyTypeChecker\n chars = numpy.full(\n (len(strings), maxlen), b'', dtype='|S1')\n for idx, name in enumerate(strings):\n for jdx, char in enumerate(name):\n chars[idx, jdx] = char.encode('utf-8')\n return chars", "response": "Return |numpy. ndarray| containing the byte characters of all given strings."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef chars2str(chars) -> List[str]:\n strings = collections.deque()\n for subchars in chars:\n substrings = collections.deque()\n for char in subchars:\n if char:\n substrings.append(char.decode('utf-8'))\n else:\n substrings.append('')\n strings.append(''.join(substrings))\n return list(strings)", "response": "Inversion function of function |str2chars|."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef create_dimension(ncfile, name, length) -> None:\n try:\n ncfile.createDimension(name, length)\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to add dimension `%s` with length `%d` '\n 'to the NetCDF file `%s`'\n % (name, length, get_filepath(ncfile)))", "response": "Create a new dimension with the given name and length."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a new variable with the given name datatype and dimensions.", "response": "def create_variable(ncfile, name, datatype, dimensions) -> None:\n \"\"\"Add a new variable with the given name, datatype, and dimensions\n to the given NetCDF file.\n\n Essentially, |create_variable| just calls the equally named method\n of the NetCDF library, but adds information to possible error messages:\n\n >>> from hydpy import TestIO\n >>> from hydpy.core.netcdftools import netcdf4\n >>> with TestIO():\n ... ncfile = netcdf4.Dataset('test.nc', 'w')\n >>> from hydpy.core.netcdftools import create_variable\n >>> try:\n ... create_variable(ncfile, 'var1', 'f8', ('dim1',))\n ... except BaseException as exc:\n ... print(str(exc).strip('\"')) # doctest: +ELLIPSIS\n While trying to add variable `var1` with datatype `f8` and \\\ndimensions `('dim1',)` to the NetCDF file `test.nc`, the following error \\\noccurred: ...\n\n >>> from hydpy.core.netcdftools import create_dimension\n >>> create_dimension(ncfile, 'dim1', 5)\n >>> create_variable(ncfile, 'var1', 'f8', ('dim1',))\n >>> import numpy\n >>> numpy.array(ncfile['var1'][:])\n array([ nan, nan, nan, nan, nan])\n\n >>> ncfile.close()\n \"\"\"\n default = fillvalue if (datatype == 'f8') else None\n try:\n ncfile.createVariable(\n name, datatype, dimensions=dimensions, fill_value=default)\n ncfile[name].long_name = name\n except BaseException:\n objecttools.augment_excmessage(\n 'While trying to add variable `%s` with datatype `%s` '\n 'and dimensions `%s` to the NetCDF file `%s`'\n % (name, datatype, dimensions, get_filepath(ncfile)))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef query_variable(ncfile, name) -> netcdf4.Variable:\n try:\n return ncfile[name]\n except (IndexError, KeyError):\n raise OSError(\n 'NetCDF file `%s` does not contain variable `%s`.'\n % (get_filepath(ncfile), name))", "response": "Query the variable with the given name from the given NetCDF file."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef query_timegrid(ncfile) -> timetools.Timegrid:\n timepoints = ncfile[varmapping['timepoints']]\n refdate = timetools.Date.from_cfunits(timepoints.units)\n return timetools.Timegrid.from_timepoints(\n timepoints=timepoints[:],\n refdate=refdate,\n unit=timepoints.units.strip().split()[0])", "response": "Query the |Timegrid| defined by the given NetCDF file."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nqueries the data of the given name from the given NetCDF file.", "response": "def query_array(ncfile, name) -> numpy.ndarray:\n \"\"\"Return the data of the variable with the given name from the given\n NetCDF file.\n\n The following example shows that |query_array| returns |nan| entries\n to represent missing values even when the respective NetCDF variable\n defines a different fill value:\n\n >>> from hydpy import TestIO\n >>> from hydpy.core.netcdftools import netcdf4\n >>> from hydpy.core import netcdftools\n >>> netcdftools.fillvalue = -999.0\n >>> with TestIO():\n ... with netcdf4.Dataset('test.nc', 'w') as ncfile:\n ... netcdftools.create_dimension(ncfile, 'dim1', 5)\n ... netcdftools.create_variable(ncfile, 'var1', 'f8', ('dim1',))\n ... ncfile = netcdf4.Dataset('test.nc', 'r')\n >>> netcdftools.query_variable(ncfile, 'var1')[:].data\n array([-999., -999., -999., -999., -999.])\n >>> netcdftools.query_array(ncfile, 'var1')\n array([ nan, nan, nan, nan, nan])\n >>> import numpy\n >>> netcdftools.fillvalue = numpy.nan\n \"\"\"\n variable = query_variable(ncfile, name)\n maskedarray = variable[:]\n fillvalue_ = getattr(variable, '_FillValue', numpy.nan)\n if not numpy.isnan(fillvalue_):\n maskedarray[maskedarray.mask] = numpy.nan\n return maskedarray.data"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlogging the given |IOSequence| object to its |NetCDFFile. log| method.", "response": "def log(self, sequence, infoarray) -> None:\n \"\"\"Prepare a |NetCDFFile| object suitable for the given |IOSequence|\n object, when necessary, and pass the given arguments to its\n |NetCDFFile.log| method.\"\"\"\n if isinstance(sequence, sequencetools.ModelSequence):\n descr = sequence.descr_model\n else:\n descr = 'node'\n if self._isolate:\n descr = '%s_%s' % (descr, sequence.descr_sequence)\n if ((infoarray is not None) and\n (infoarray.info['type'] != 'unmodified')):\n descr = '%s_%s' % (descr, infoarray.info['type'])\n dirpath = sequence.dirpath_ext\n try:\n files = self.folders[dirpath]\n except KeyError:\n files: Dict[str, 'NetCDFFile'] = collections.OrderedDict()\n self.folders[dirpath] = files\n try:\n file_ = files[descr]\n except KeyError:\n file_ = NetCDFFile(\n name=descr,\n flatten=self._flatten,\n isolate=self._isolate,\n timeaxis=self._timeaxis,\n dirpath=dirpath)\n files[descr] = file_\n file_.log(sequence, infoarray)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read(self) -> None:\n for folder in self.folders.values():\n for file_ in folder.values():\n file_.read()", "response": "Call method |NetCDFFile. read| of all handled |NetCDFFile objects."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write(self) -> None:\n if self.folders:\n init = hydpy.pub.timegrids.init\n timeunits = init.firstdate.to_cfunits('hours')\n timepoints = init.to_timepoints('hours')\n for folder in self.folders.values():\n for file_ in folder.values():\n file_.write(timeunits, timepoints)", "response": "Write all |NetCDFFile| objects in |NetCDFFolder| objects in |NetCDFFile| objects in |NetCDFFolder| objects."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef filenames(self) -> Tuple[str, ...]:\n return tuple(sorted(set(itertools.chain(\n *(_.keys() for _ in self.folders.values())))))", "response": "A |tuple| of names of all handled |NetCDFFile| objects."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nlog the given |IoSequence| to a suitable instance of and a |InfoArray| containing all of the information about the resulting object structure.", "response": "def log(self, sequence, infoarray) -> None:\n \"\"\"Pass the given |IoSequence| to a suitable instance of\n a |NetCDFVariableBase| subclass.\n\n When writing data, the second argument should be an |InfoArray|.\n When reading data, this argument is ignored. Simply pass |None|.\n\n (1) We prepare some devices handling some sequences by applying\n function |prepare_io_example_1|. We limit our attention to the\n returned elements, which handle the more diverse sequences:\n\n >>> from hydpy.core.examples import prepare_io_example_1\n >>> nodes, (element1, element2, element3) = prepare_io_example_1()\n\n (2) We define some shortcuts for the sequences used in the\n following examples:\n\n >>> nied1 = element1.model.sequences.inputs.nied\n >>> nied2 = element2.model.sequences.inputs.nied\n >>> nkor2 = element2.model.sequences.fluxes.nkor\n >>> nkor3 = element3.model.sequences.fluxes.nkor\n\n (3) We define a function that logs these example sequences\n to a given |NetCDFFile| object and prints some information\n about the resulting object structure. Note that sequence\n `nkor2` is logged twice, the first time with its original\n time series data, the second time with averaged values:\n\n >>> from hydpy import classname\n >>> def test(ncfile):\n ... ncfile.log(nied1, nied1.series)\n ... ncfile.log(nied2, nied2.series)\n ... ncfile.log(nkor2, nkor2.series)\n ... ncfile.log(nkor2, nkor2.average_series())\n ... ncfile.log(nkor3, nkor3.average_series())\n ... for name, variable in ncfile.variables.items():\n ... print(name, classname(variable), variable.subdevicenames)\n\n (4) We prepare a |NetCDFFile| object with both options\n `flatten` and `isolate` being disabled:\n\n >>> from hydpy.core.netcdftools import NetCDFFile\n >>> ncfile = NetCDFFile(\n ... 'model', flatten=False, isolate=False, timeaxis=1, dirpath='')\n\n (5) We log all test sequences results in two |NetCDFVariableDeep|\n and one |NetCDFVariableAgg| objects. To keep both NetCDF variables\n related to |lland_fluxes.NKor| distinguishable, the name\n `flux_nkor_mean` includes information about the kind of aggregation\n performed:\n\n >>> test(ncfile)\n input_nied NetCDFVariableDeep ('element1', 'element2')\n flux_nkor NetCDFVariableDeep ('element2',)\n flux_nkor_mean NetCDFVariableAgg ('element2', 'element3')\n\n (6) We confirm that the |NetCDFVariableBase| objects received\n the required information:\n\n >>> ncfile.flux_nkor.element2.sequence.descr_device\n 'element2'\n >>> ncfile.flux_nkor.element2.array\n InfoArray([[ 16., 17.],\n [ 18., 19.],\n [ 20., 21.],\n [ 22., 23.]])\n >>> ncfile.flux_nkor_mean.element2.sequence.descr_device\n 'element2'\n >>> ncfile.flux_nkor_mean.element2.array\n InfoArray([ 16.5, 18.5, 20.5, 22.5])\n\n (7) We again prepare a |NetCDFFile| object, but now with both\n options `flatten` and `isolate` being enabled. To log test\n sequences with their original time series data does now trigger\n the initialisation of class |NetCDFVariableFlat|. When passing\n aggregated data, nothing changes:\n\n >>> ncfile = NetCDFFile(\n ... 'model', flatten=True, isolate=True, timeaxis=1, dirpath='')\n >>> test(ncfile)\n input_nied NetCDFVariableFlat ('element1', 'element2')\n flux_nkor NetCDFVariableFlat ('element2_0', 'element2_1')\n flux_nkor_mean NetCDFVariableAgg ('element2', 'element3')\n >>> ncfile.flux_nkor.element2.sequence.descr_device\n 'element2'\n >>> ncfile.flux_nkor.element2.array\n InfoArray([[ 16., 17.],\n [ 18., 19.],\n [ 20., 21.],\n [ 22., 23.]])\n >>> ncfile.flux_nkor_mean.element2.sequence.descr_device\n 'element2'\n >>> ncfile.flux_nkor_mean.element2.array\n InfoArray([ 16.5, 18.5, 20.5, 22.5])\n\n (8) We technically confirm that the `isolate` argument is passed\n to the constructor of subclasses of |NetCDFVariableBase| correctly:\n\n >>> from unittest.mock import patch\n >>> with patch('hydpy.core.netcdftools.NetCDFVariableFlat') as mock:\n ... ncfile = NetCDFFile(\n ... 'model', flatten=True, isolate=False, timeaxis=0,\n ... dirpath='')\n ... ncfile.log(nied1, nied1.series)\n ... mock.assert_called_once_with(\n ... name='input_nied', timeaxis=0, isolate=False)\n \"\"\"\n aggregated = ((infoarray is not None) and\n (infoarray.info['type'] != 'unmodified'))\n descr = sequence.descr_sequence\n if aggregated:\n descr = '_'.join([descr, infoarray.info['type']])\n if descr in self.variables:\n var_ = self.variables[descr]\n else:\n if aggregated:\n cls = NetCDFVariableAgg\n elif self._flatten:\n cls = NetCDFVariableFlat\n else:\n cls = NetCDFVariableDeep\n var_ = cls(name=descr,\n isolate=self._isolate,\n timeaxis=self._timeaxis)\n self.variables[descr] = var_\n var_.log(sequence, infoarray)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nopens an existing NetCDF file temporarily and call method Quick. read of all handled |NetCDFVariableBase| objects.", "response": "def read(self) -> None:\n \"\"\"Open an existing NetCDF file temporarily and call method\n |NetCDFVariableDeep.read| of all handled |NetCDFVariableBase|\n objects.\"\"\"\n try:\n with netcdf4.Dataset(self.filepath, \"r\") as ncfile:\n timegrid = query_timegrid(ncfile)\n for variable in self.variables.values():\n variable.read(ncfile, timegrid)\n except BaseException:\n objecttools.augment_excmessage(\n f'While trying to read data from NetCDF file `{self.filepath}`')"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write(self, timeunit, timepoints) -> None:\n with netcdf4.Dataset(self.filepath, \"w\") as ncfile:\n ncfile.Conventions = 'CF-1.6'\n self._insert_timepoints(ncfile, timepoints, timeunit)\n for variable in self.variables.values():\n variable.write(ncfile)", "response": "Write |NetCDFVariableBase. write| of all handled |NetCDFVariableBase| objects."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the index of the object with a specialized error message.", "response": "def get_index(self, name_subdevice) -> int:\n \"\"\"Item access to the wrapped |dict| object with a specialized\n error message.\"\"\"\n try:\n return self.dict_[name_subdevice]\n except KeyError:\n raise OSError(\n 'No data for sequence `%s` and (sub)device `%s` '\n 'in NetCDF file `%s` available.'\n % (self.name_sequence,\n name_subdevice,\n self.name_ncfile))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlogging the given |IOSequence| object either for reading or writing or a modified time series.", "response": "def log(self, sequence, infoarray) -> None:\n \"\"\"Log the given |IOSequence| object either for reading or writing\n data.\n\n The optional `array` argument allows for passing alternative data\n in an |InfoArray| object replacing the series of the |IOSequence|\n object, which is useful for writing modified (e.g. spatially\n averaged) time series.\n\n Logged time series data is available via attribute access:\n\n >>> from hydpy.core.netcdftools import NetCDFVariableBase\n >>> from hydpy import make_abc_testable\n >>> NCVar = make_abc_testable(NetCDFVariableBase)\n >>> ncvar = NCVar('flux_nkor', isolate=True, timeaxis=1)\n >>> from hydpy.core.examples import prepare_io_example_1\n >>> nodes, elements = prepare_io_example_1()\n >>> nkor = elements.element1.model.sequences.fluxes.nkor\n >>> ncvar.log(nkor, nkor.series)\n >>> 'element1' in dir(ncvar)\n True\n >>> ncvar.element1.sequence is nkor\n True\n >>> 'element2' in dir(ncvar)\n False\n >>> ncvar.element2\n Traceback (most recent call last):\n ...\n AttributeError: The NetCDFVariable object `flux_nkor` does \\\nneither handle time series data under the (sub)device name `element2` \\\nnor does it define a member named `element2`.\n \"\"\"\n descr_device = sequence.descr_device\n self.sequences[descr_device] = sequence\n self.arrays[descr_device] = infoarray"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef insert_subdevices(self, ncfile) -> None:\n prefix = self.prefix\n nmb_subdevices = '%s%s' % (prefix, dimmapping['nmb_subdevices'])\n nmb_characters = '%s%s' % (prefix, dimmapping['nmb_characters'])\n subdevices = '%s%s' % (prefix, varmapping['subdevices'])\n statchars = str2chars(self.subdevicenames)\n create_dimension(ncfile, nmb_subdevices, statchars.shape[0])\n create_dimension(ncfile, nmb_characters, statchars.shape[1])\n create_variable(\n ncfile, subdevices, 'S1', (nmb_subdevices, nmb_characters))\n ncfile[subdevices][:, :] = statchars", "response": "Insert a variable of the names of the subdevices of the logged\n sequences into the given NetCDF file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nquery the names of the subdevices of the logged sequences from the given NetCDF file.", "response": "def query_subdevices(self, ncfile) -> List[str]:\n \"\"\"Query the names of the (sub)devices of the logged sequences\n from the given NetCDF file\n\n (1) We apply function |NetCDFVariableBase.query_subdevices| on\n an empty NetCDF file. The error message shows that the method\n tries to query the (sub)device names both under the assumptions\n that variables have been isolated or not:\n\n >>> from hydpy.core.netcdftools import NetCDFVariableBase\n >>> from hydpy import make_abc_testable, TestIO\n >>> from hydpy.core.netcdftools import netcdf4\n >>> with TestIO():\n ... ncfile = netcdf4.Dataset('model.nc', 'w')\n >>> Var = make_abc_testable(NetCDFVariableBase)\n >>> Var.subdevicenames = 'element1', 'element_2'\n >>> var = Var('flux_prec', isolate=False, timeaxis=1)\n >>> var.query_subdevices(ncfile)\n Traceback (most recent call last):\n ...\n OSError: NetCDF file `model.nc` does neither contain a variable \\\nnamed `flux_prec_station_id` nor `station_id` for defining the \\\ncoordinate locations of variable `flux_prec`.\n\n (2) After inserting the (sub)device name, they can be queried\n and returned:\n\n >>> var.insert_subdevices(ncfile)\n >>> Var('flux_prec', isolate=False, timeaxis=1).query_subdevices(ncfile)\n ['element1', 'element_2']\n >>> Var('flux_prec', isolate=True, timeaxis=1).query_subdevices(ncfile)\n ['element1', 'element_2']\n\n >>> ncfile.close()\n \"\"\"\n tests = ['%s%s' % (prefix, varmapping['subdevices'])\n for prefix in ('%s_' % self.name, '')]\n for subdevices in tests:\n try:\n chars = ncfile[subdevices][:]\n break\n except (IndexError, KeyError):\n pass\n else:\n raise IOError(\n 'NetCDF file `%s` does neither contain a variable '\n 'named `%s` nor `%s` for defining the coordinate '\n 'locations of variable `%s`.'\n % (get_filepath(ncfile), tests[0], tests[1], self.name))\n return chars2str(chars)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nquerying |NetCDFVariableBase. query_subdevices| and return |Subdevice2Index| object that maps the sub device names to their position within the given NetCDF file.", "response": "def query_subdevice2index(self, ncfile) -> Subdevice2Index:\n \"\"\"Return a |Subdevice2Index| that maps the (sub)device names to\n their position within the given NetCDF file.\n\n Method |NetCDFVariableBase.query_subdevice2index| is based on\n |NetCDFVariableBase.query_subdevices|. The returned\n |Subdevice2Index| object remembers the NetCDF file the\n (sub)device names stem from, allowing for clear error messages:\n\n >>> from hydpy.core.netcdftools import NetCDFVariableBase, str2chars\n >>> from hydpy import make_abc_testable, TestIO\n >>> from hydpy.core.netcdftools import netcdf4\n >>> with TestIO():\n ... ncfile = netcdf4.Dataset('model.nc', 'w')\n >>> Var = make_abc_testable(NetCDFVariableBase)\n >>> Var.subdevicenames = [\n ... 'element3', 'element1', 'element1_1', 'element2']\n >>> var = Var('flux_prec', isolate=True, timeaxis=1)\n >>> var.insert_subdevices(ncfile)\n >>> subdevice2index = var.query_subdevice2index(ncfile)\n >>> subdevice2index.get_index('element1_1')\n 2\n >>> subdevice2index.get_index('element3')\n 0\n >>> subdevice2index.get_index('element5')\n Traceback (most recent call last):\n ...\n OSError: No data for sequence `flux_prec` and (sub)device \\\n`element5` in NetCDF file `model.nc` available.\n\n Additionally, |NetCDFVariableBase.query_subdevice2index|\n checks for duplicates:\n\n >>> ncfile['station_id'][:] = str2chars(\n ... ['element3', 'element1', 'element1_1', 'element1'])\n >>> var.query_subdevice2index(ncfile)\n Traceback (most recent call last):\n ...\n OSError: The NetCDF file `model.nc` contains duplicate (sub)device \\\nnames for variable `flux_prec` (the first found duplicate is `element1`).\n\n >>> ncfile.close()\n \"\"\"\n subdevices = self.query_subdevices(ncfile)\n self._test_duplicate_exists(ncfile, subdevices)\n subdev2index = {subdev: idx for (idx, subdev) in enumerate(subdevices)}\n return Subdevice2Index(subdev2index, self.name, get_filepath(ncfile))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sort_timeplaceentries(self, timeentry, placeentry) -> Tuple[Any, Any]:\n if self._timeaxis:\n return placeentry, timeentry\n return timeentry, placeentry", "response": "Return a |tuple| containing the given timeentry and placeentry sorted in agreement with the currently selected timeaxis."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a |tuple| for indexing a complete time series of a certain location available in |NetCDFVariableBase. array|.", "response": "def get_timeplaceslice(self, placeindex) -> \\\n Union[Tuple[slice, int], Tuple[int, slice]]:\n \"\"\"Return a |tuple| for indexing a complete time series of a certain\n location available in |NetCDFVariableBase.array|.\n\n >>> from hydpy.core.netcdftools import NetCDFVariableBase\n >>> from hydpy import make_abc_testable\n >>> NCVar = make_abc_testable(NetCDFVariableBase)\n >>> ncvar = NCVar('flux_nkor', isolate=True, timeaxis=1)\n >>> ncvar.get_timeplaceslice(2)\n (2, slice(None, None, None))\n >>> ncvar = NetCDFVariableDeep('test', isolate=False, timeaxis=0)\n >>> ncvar.get_timeplaceslice(2)\n (slice(None, None, None), 2)\n \"\"\"\n return self.sort_timeplaceentries(slice(None), int(placeindex))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef subdevicenames(self) -> Tuple[str, ...]:\n self: NetCDFVariableBase\n return tuple(self.sequences.keys())", "response": "A |tuple| containing the device names."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write(self, ncfile) -> None:\n self: NetCDFVariableBase\n self.insert_subdevices(ncfile)\n dimensions = self.dimensions\n array = self.array\n for dimension, length in zip(dimensions[2:], array.shape[2:]):\n create_dimension(ncfile, dimension, length)\n create_variable(ncfile, self.name, 'f8', dimensions)\n ncfile[self.name][:] = array", "response": "Write the data to the given NetCDF file."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_slices(self, idx, shape) -> Tuple[IntOrSlice, ...]:\n slices = list(self.get_timeplaceslice(idx))\n for length in shape:\n slices.append(slice(0, length))\n return tuple(slices)", "response": "Return a |tuple| of one int and some |slice| objects to\n accesses all values of a certain device within the given |NetCDFVariableDeep. array|."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nrequires shape of |NetCDFVariableDeep. array|.", "response": "def shape(self) -> Tuple[int, ...]:\n \"\"\"Required shape of |NetCDFVariableDeep.array|.\n\n For the default configuration, the first axis corresponds to the\n number of devices, and the second one to the number of timesteps.\n We show this for the 0-dimensional input sequence |lland_inputs.Nied|:\n\n >>> from hydpy.core.examples import prepare_io_example_1\n >>> nodes, elements = prepare_io_example_1()\n >>> from hydpy.core.netcdftools import NetCDFVariableDeep\n >>> ncvar = NetCDFVariableDeep('input_nied', isolate=False, timeaxis=1)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.inputs.nied, None)\n >>> ncvar.shape\n (3, 4)\n\n For higher dimensional sequences, each new entry corresponds\n to the maximum number of fields the respective sequences require.\n In the next example, we select the 1-dimensional sequence\n |lland_fluxes.NKor|. The maximum number 3 (last value of the\n returned |tuple|) is due to the third element defining three\n hydrological response units:\n\n >>> ncvar = NetCDFVariableDeep('flux_nkor', isolate=False, timeaxis=1)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.fluxes.nkor, None)\n >>> ncvar.shape\n (3, 4, 3)\n\n When using the first axis for time (`timeaxis=0`) the order of the\n first two |tuple| entries turns:\n\n >>> ncvar = NetCDFVariableDeep('flux_nkor', isolate=False, timeaxis=0)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.fluxes.nkor, None)\n >>> ncvar.shape\n (4, 3, 3)\n \"\"\"\n nmb_place = len(self.sequences)\n nmb_time = len(hydpy.pub.timegrids.init)\n nmb_others = collections.deque()\n for sequence in self.sequences.values():\n nmb_others.append(sequence.shape)\n nmb_others_max = tuple(numpy.max(nmb_others, axis=0))\n return self.sort_timeplaceentries(nmb_time, nmb_place) + nmb_others_max"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef dimensions(self) -> Tuple[str, ...]:\n nmb_timepoints = dimmapping['nmb_timepoints']\n nmb_subdevices = '%s%s' % (self.prefix, dimmapping['nmb_subdevices'])\n dimensions = list(self.sort_timeplaceentries(\n nmb_timepoints, nmb_subdevices))\n for idx in range(list(self.sequences.values())[0].NDIM):\n dimensions.append('%saxis%d' % (self.prefix, idx + 3))\n return tuple(dimensions)", "response": "The dimension names of the NetCDF variable."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read(self, ncfile, timegrid_data) -> None:\n array = query_array(ncfile, self.name)\n subdev2index = self.query_subdevice2index(ncfile)\n for subdevice, sequence in self.sequences.items():\n idx = subdev2index.get_index(subdevice)\n values = array[self.get_slices(idx, sequence.shape)]\n sequence.series = sequence.adjust_series(\n timegrid_data, values)", "response": "Read the data from the given NetCDF file."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrequire shape of |NetCDFVariableAgg. array|.", "response": "def shape(self) -> Tuple[int, int]:\n \"\"\"Required shape of |NetCDFVariableAgg.array|.\n\n For the default configuration, the first axis corresponds to the\n number of devices, and the second one to the number of timesteps.\n We show this for the 1-dimensional input sequence |lland_fluxes.NKor|:\n\n >>> from hydpy.core.examples import prepare_io_example_1\n >>> nodes, elements = prepare_io_example_1()\n >>> from hydpy.core.netcdftools import NetCDFVariableAgg\n >>> ncvar = NetCDFVariableAgg('flux_nkor', isolate=False, timeaxis=1)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.fluxes.nkor, None)\n >>> ncvar.shape\n (3, 4)\n\n When using the first axis as the \"timeaxis\", the order of |tuple|\n entries turns:\n\n >>> ncvar = NetCDFVariableAgg('flux_nkor', isolate=False, timeaxis=0)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.fluxes.nkor, None)\n >>> ncvar.shape\n (4, 3)\n \"\"\"\n return self.sort_timeplaceentries(\n len(hydpy.pub.timegrids.init), len(self.sequences))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nrequires shape of |NetCDFVariableFlat. array|.", "response": "def shape(self) -> Tuple[int, int]:\n \"\"\"Required shape of |NetCDFVariableFlat.array|.\n\n For 0-dimensional sequences like |lland_inputs.Nied| and for the\n default configuration (`timeaxis=1`), the first axis corresponds\n to the number of devices, and the second one two the number of\n timesteps:\n\n >>> from hydpy.core.examples import prepare_io_example_1\n >>> nodes, elements = prepare_io_example_1()\n >>> from hydpy.core.netcdftools import NetCDFVariableFlat\n >>> ncvar = NetCDFVariableFlat('input_nied', isolate=False, timeaxis=1)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.inputs.nied, None)\n >>> ncvar.shape\n (3, 4)\n\n For higher dimensional sequences, the first axis corresponds\n to \"subdevices\", e.g. hydrological response units within\n different elements. The 1-dimensional sequence |lland_fluxes.NKor|\n is logged for three elements with one, two, and three response\n units respectively, making up a sum of six subdevices:\n\n >>> ncvar = NetCDFVariableFlat('flux_nkor', isolate=False, timeaxis=1)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.fluxes.nkor, None)\n >>> ncvar.shape\n (6, 4)\n\n When using the first axis as the \"timeaxis\", the order of |tuple|\n entries turns:\n\n >>> ncvar = NetCDFVariableFlat('flux_nkor', isolate=False, timeaxis=0)\n >>> for element in elements:\n ... ncvar.log(element.model.sequences.fluxes.nkor, None)\n >>> ncvar.shape\n (4, 6)\n \"\"\"\n return self.sort_timeplaceentries(\n len(hydpy.pub.timegrids.init),\n sum(len(seq) for seq in self.sequences.values()))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef subdevicenames(self) -> Tuple[str, ...]:\n stats: List[str] = collections.deque()\n for devicename, seq in self.sequences.items():\n if seq.NDIM:\n temp = devicename + '_'\n for prod in self._product(seq.shape):\n stats.append(temp + '_'.join(str(idx) for idx in prod))\n else:\n stats.append(devicename)\n return tuple(stats)", "response": "A |tuple| containing the subdevice names of the |NetCDFVariableFlat| object."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef read(self, ncfile, timegrid_data) -> None:\n array = query_array(ncfile, self.name)\n idxs: Tuple[Any] = (slice(None),)\n subdev2index = self.query_subdevice2index(ncfile)\n for devicename, seq in self.sequences.items():\n if seq.NDIM:\n if self._timeaxis:\n subshape = (array.shape[1],) + seq.shape\n else:\n subshape = (array.shape[0],) + seq.shape\n subarray = numpy.empty(subshape)\n temp = devicename + '_'\n for prod in self._product(seq.shape):\n station = temp + '_'.join(str(idx) for idx in prod)\n idx0 = subdev2index.get_index(station)\n subarray[idxs+prod] = array[self.get_timeplaceslice(idx0)]\n else:\n idx = subdev2index.get_index(devicename)\n subarray = array[self.get_timeplaceslice(idx)]\n seq.series = seq.adjust_series(timegrid_data, subarray)", "response": "Read the data from the given NetCDF file."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef write(self, ncfile) -> None:\n self.insert_subdevices(ncfile)\n create_variable(ncfile, self.name, 'f8', self.dimensions)\n ncfile[self.name][:] = self.array", "response": "Write the data to the given NetCDF file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef update(self):\n maxdt = self.subpars.pars.control.maxdt\n seconds = self.simulationstep.seconds\n self.value = numpy.ceil(seconds/maxdt)", "response": "Update the internal state of the internal state of the current entry."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nupdates the internal state of the object with the current values from the auxilary term.", "response": "def update(self):\n \"\"\"Calulate the auxilary term.\n\n >>> from hydpy.models.llake import *\n >>> parameterstep('1d')\n >>> simulationstep('12h')\n >>> n(3)\n >>> v(0., 1e5, 1e6)\n >>> q(_1=[0., 1., 2.], _7=[0., 2., 5.])\n >>> maxdt('12h')\n >>> derived.seconds.update()\n >>> derived.nmbsubsteps.update()\n >>> derived.vq.update()\n >>> derived.vq\n vq(toy_1_1_0_0_0=[0.0, 243200.0, 2086400.0],\n toy_7_1_0_0_0=[0.0, 286400.0, 2216000.0])\n \"\"\"\n con = self.subpars.pars.control\n der = self.subpars\n for (toy, qs) in con.q:\n setattr(self, str(toy), 2.*con.v+der.seconds/der.nmbsubsteps*qs)\n self.refresh()"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef prepare_io_example_1() -> Tuple[devicetools.Nodes, devicetools.Elements]:\n # noinspection PyUnresolvedReferences\n \"\"\"Prepare an IO example configuration.\n\n >>> from hydpy.core.examples import prepare_io_example_1\n >>> nodes, elements = prepare_io_example_1()\n\n (1) Prepares a short initialisation period of five days:\n\n >>> from hydpy import pub\n >>> pub.timegrids\n Timegrids(Timegrid('2000-01-01 00:00:00',\n '2000-01-05 00:00:00',\n '1d'))\n\n (2) Prepares a plain IO testing directory structure:\n\n >>> pub.sequencemanager.inputdirpath\n 'inputpath'\n >>> pub.sequencemanager.fluxdirpath\n 'outputpath'\n >>> pub.sequencemanager.statedirpath\n 'outputpath'\n >>> pub.sequencemanager.nodedirpath\n 'nodepath'\n >>> import os\n >>> from hydpy import TestIO\n >>> with TestIO():\n ... print(sorted(filename for filename in os.listdir('.')\n ... if not filename.startswith('_')))\n ['inputpath', 'nodepath', 'outputpath']\n\n (3) Returns three |Element| objects handling either application model\n |lland_v1| or |lland_v2|, and two |Node| objects handling variables\n `Q` and `T`:\n\n >>> for element in elements:\n ... print(element.name, element.model)\n element1 lland_v1\n element2 lland_v1\n element3 lland_v2\n >>> for node in nodes:\n ... print(node.name, node.variable)\n node1 Q\n node2 T\n\n (4) Prepares the time series data of the input sequence\n |lland_inputs.Nied|, flux sequence |lland_fluxes.NKor|, and state\n sequence |lland_states.BoWa| for each model instance, and |Sim| for\n each node instance (all values are different), e.g.:\n\n >>> nied1 = elements.element1.model.sequences.inputs.nied\n >>> nied1.series\n InfoArray([ 0., 1., 2., 3.])\n >>> nkor1 = elements.element1.model.sequences.fluxes.nkor\n >>> nkor1.series\n InfoArray([[ 12.],\n [ 13.],\n [ 14.],\n [ 15.]])\n >>> bowa3 = elements.element3.model.sequences.states.bowa\n >>> bowa3.series\n InfoArray([[ 48., 49., 50.],\n [ 51., 52., 53.],\n [ 54., 55., 56.],\n [ 57., 58., 59.]])\n >>> sim2 = nodes.node2.sequences.sim\n >>> sim2.series\n InfoArray([ 64., 65., 66., 67.])\n\n (5) All sequences carry |numpy.ndarray| objects with (deep) copies\n of the time series data for testing:\n\n >>> import numpy\n >>> (numpy.all(nied1.series == nied1.testarray) and\n ... numpy.all(nkor1.series == nkor1.testarray) and\n ... numpy.all(bowa3.series == bowa3.testarray) and\n ... numpy.all(sim2.series == sim2.testarray))\n InfoArray(True, dtype=bool)\n >>> bowa3.series[1, 2] = -999.0\n >>> numpy.all(bowa3.series == bowa3.testarray)\n InfoArray(False, dtype=bool)\n \"\"\"\n from hydpy import TestIO\n TestIO.clear()\n from hydpy.core.filetools import SequenceManager\n hydpy.pub.sequencemanager = SequenceManager()\n with TestIO():\n hydpy.pub.sequencemanager.inputdirpath = 'inputpath'\n hydpy.pub.sequencemanager.fluxdirpath = 'outputpath'\n hydpy.pub.sequencemanager.statedirpath = 'outputpath'\n hydpy.pub.sequencemanager.nodedirpath = 'nodepath'\n\n hydpy.pub.timegrids = '2000-01-01', '2000-01-05', '1d'\n\n from hydpy import Node, Nodes, Element, Elements, prepare_model\n node1 = Node('node1')\n node2 = Node('node2', variable='T')\n nodes = Nodes(node1, node2)\n element1 = Element('element1', outlets=node1)\n element2 = Element('element2', outlets=node1)\n element3 = Element('element3', outlets=node1)\n elements = Elements(element1, element2, element3)\n\n from hydpy.models import lland_v1, lland_v2\n element1.model = prepare_model(lland_v1)\n element2.model = prepare_model(lland_v1)\n element3.model = prepare_model(lland_v2)\n\n from hydpy.models.lland import ACKER\n for idx, element in enumerate(elements):\n parameters = element.model.parameters\n parameters.control.nhru(idx+1)\n parameters.control.lnk(ACKER)\n parameters.derived.absfhru(10.0)\n\n with hydpy.pub.options.printprogress(False):\n nodes.prepare_simseries()\n elements.prepare_inputseries()\n elements.prepare_fluxseries()\n elements.prepare_stateseries()\n\n def init_values(seq, value1_):\n value2_ = value1_ + len(seq.series.flatten())\n values_ = numpy.arange(value1_, value2_, dtype=float)\n seq.testarray = values_.reshape(seq.seriesshape)\n seq.series = seq.testarray.copy()\n return value2_\n\n import numpy\n value1 = 0\n for subname, seqname in zip(['inputs', 'fluxes', 'states'],\n ['nied', 'nkor', 'bowa']):\n for element in elements:\n subseqs = getattr(element.model.sequences, subname)\n value1 = init_values(getattr(subseqs, seqname), value1)\n for node in nodes:\n value1 = init_values(node.sequences.sim, value1)\n\n return nodes, elements", "response": "Prepare an IO example configuration."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef prepare_full_example_1() -> None:\n testtools.TestIO.clear()\n shutil.copytree(\n os.path.join(data.__path__[0], 'LahnH'),\n os.path.join(iotesting.__path__[0], 'LahnH'))\n seqpath = os.path.join(iotesting.__path__[0], 'LahnH', 'series')\n for folder in ('output', 'node', 'temp'):\n os.makedirs(os.path.join(seqpath, folder))", "response": "Prepare the complete LahnH project for testing."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef prepare_full_example_2(lastdate='1996-01-05') -> (\n hydpytools.HydPy, hydpy.pub, testtools.TestIO):\n \"\"\"Prepare the complete `LahnH` project for testing.\n\n |prepare_full_example_2| calls |prepare_full_example_1|, but also\n returns a readily prepared |HydPy| instance, as well as module\n |pub| and class |TestIO|, for convenience:\n\n >>> from hydpy.core.examples import prepare_full_example_2\n >>> hp, pub, TestIO = prepare_full_example_2()\n >>> hp.nodes\n Nodes(\"dill\", \"lahn_1\", \"lahn_2\", \"lahn_3\")\n >>> hp.elements\n Elements(\"land_dill\", \"land_lahn_1\", \"land_lahn_2\", \"land_lahn_3\",\n \"stream_dill_lahn_2\", \"stream_lahn_1_lahn_2\",\n \"stream_lahn_2_lahn_3\")\n >>> pub.timegrids\n Timegrids(Timegrid('1996-01-01 00:00:00',\n '1996-01-05 00:00:00',\n '1d'))\n >>> from hydpy import classname\n >>> classname(TestIO)\n 'TestIO'\n\n The last date of the initialisation period is configurable:\n\n >>> hp, pub, TestIO = prepare_full_example_2('1996-02-01')\n >>> pub.timegrids\n Timegrids(Timegrid('1996-01-01 00:00:00',\n '1996-02-01 00:00:00',\n '1d'))\n \"\"\"\n prepare_full_example_1()\n with testtools.TestIO():\n hp = hydpytools.HydPy('LahnH')\n hydpy.pub.timegrids = '1996-01-01', lastdate, '1d'\n hp.prepare_everything()\n return hp, hydpy.pub, testtools.TestIO", "response": "Prepare the complete LahnH project for testing."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_postalcodes_around_radius(self, pc, radius):\n postalcodes = self.get(pc)\n if postalcodes is None:\n raise PostalCodeNotFoundException(\"Could not find postal code you're searching for.\")\n else:\n pc = postalcodes[0]\n \n radius = float(radius)\n \n '''\n Bounding box calculations updated from pyzipcode\n ''' \n earth_radius = 6371\n dlat = radius / earth_radius\n dlon = asin(sin(dlat) / cos(radians(pc.latitude)))\n lat_delta = degrees(dlat)\n lon_delta = degrees(dlon)\n \n if lat_delta < 0:\n lat_range = (pc.latitude + lat_delta, pc.latitude - lat_delta)\n else:\n lat_range = (pc.latitude - lat_delta, pc.latitude + lat_delta)\n \n long_range = (pc.longitude - lat_delta, pc.longitude + lon_delta) \n \n return format_result(self.conn_manager.query(PC_RANGE_QUERY % (\n long_range[0], long_range[1],\n lat_range[0], lat_range[1]\n )))", "response": "Returns a list of postal codes around a radius."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_all_player_ids(ids=\"shots\"):\n url = \"http://stats.nba.com/stats/commonallplayers?IsOnlyCurrentSeason=0&LeagueID=00&Season=2015-16\"\n\n # get the web page\n response = requests.get(url, headers=HEADERS)\n response.raise_for_status()\n # access 'resultSets', which is a list containing the dict with all the data\n # The 'header' key accesses the headers\n headers = response.json()['resultSets'][0]['headers']\n # The 'rowSet' key contains the player data along with their IDs\n players = response.json()['resultSets'][0]['rowSet']\n # Create dataframe with proper numeric types\n df = pd.DataFrame(players, columns=headers)\n\n # Dealing with different means of converision for pandas 0.17.0 or 0.17.1\n # and 0.15.0 or loweer\n if '0.17' in pd.__version__:\n # alternative to convert_objects() to numeric to get rid of warning\n # as convert_objects() is deprecated in pandas 0.17.0+\n df = df.apply(pd.to_numeric, args=('ignore',))\n else:\n df = df.convert_objects(convert_numeric=True)\n\n if ids == \"shots\":\n df = df.query(\"(FROM_YEAR >= 2001) or (TO_YEAR >= 2001)\")\n df = df.reset_index(drop=True)\n # just keep the player ids and names\n df = df.iloc[:, 0:2]\n return df\n if ids == \"all_players\":\n df = df.iloc[:, 0:2]\n return df\n if ids == \"all_data\":\n return df\n else:\n er = \"Invalid 'ids' value. It must be 'shots', 'all_shots', or 'all_data'.\"\n raise ValueError(er)", "response": "Returns a pandas DataFrame containing the player IDs used in the stats. nba. com API."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_player_id(player):\n players_df = get_all_player_ids(\"all_data\")\n player = players_df[players_df.DISPLAY_LAST_COMMA_FIRST == player]\n # if there are no plyaers by the given name, raise an a error\n if len(player) == 0:\n er = \"Invalid player name passed or there is no player with that name.\"\n raise ValueError(er)\n player_id = player.PERSON_ID.values\n return player_id", "response": "Returns the player ID associated with the given player name."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a pandas DataFrame with all Team IDs", "response": "def get_all_team_ids():\n \"\"\"Returns a pandas DataFrame with all Team IDs\"\"\"\n df = get_all_player_ids(\"all_data\")\n df = pd.DataFrame({\"TEAM_NAME\": df.TEAM_NAME.unique(),\n \"TEAM_ID\": df.TEAM_ID.unique()})\n return df"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the team ID associated with the team name that is passed in.", "response": "def get_team_id(team_name):\n \"\"\" Returns the team ID associated with the team name that is passed in.\n\n Parameters\n ----------\n team_name : str\n The team name whose ID we want. NOTE: Only pass in the team name\n (e.g. \"Lakers\"), not the city, or city and team name, or the team\n abbreviation.\n\n Returns\n -------\n team_id : int\n The team ID associated with the team name.\n\n \"\"\"\n df = get_all_team_ids()\n df = df[df.TEAM_NAME == team_name]\n if len(df) == 0:\n er = \"Invalid team name or there is no team with that name.\"\n raise ValueError(er)\n team_id = df.TEAM_ID.iloc[0]\n return team_id"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_player_img(player_id):\n url = \"http://stats.nba.com/media/players/230x185/\"+str(player_id)+\".png\"\n img_file = str(player_id) + \".png\"\n pic = urlretrieve(url, img_file)\n player_img = plt.imread(pic[0])\n return player_img", "response": "Returns the image of the player from stats. nba. com."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning team game logs as a pandas DataFrame", "response": "def get_game_logs(self):\n \"\"\"Returns team game logs as a pandas DataFrame\"\"\"\n logs = self.response.json()['resultSets'][0]['rowSet']\n headers = self.response.json()['resultSets'][0]['headers']\n df = pd.DataFrame(logs, columns=headers)\n df.GAME_DATE = pd.to_datetime(df.GAME_DATE)\n return df"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_game_id(self, date):\n df = self.get_game_logs()\n game_id = df[df.GAME_DATE == date].Game_ID.values[0]\n return game_id", "response": "Returns the Game ID associated with the date that is passed in."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update_params(self, parameters):\n self.url_paramaters.update(parameters)\n self.response = requests.get(self.base_url, params=self.url_paramaters,\n headers=HEADERS)\n # raise error if status code is not 200\n self.response.raise_for_status()\n return self", "response": "Pass in a dictionary to update url parameters for NBA stats API\n API parameters."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the shot chart data as a pandas DataFrame.", "response": "def get_shots(self):\n \"\"\"Returns the shot chart data as a pandas DataFrame.\"\"\"\n shots = self.response.json()['resultSets'][0]['rowSet']\n headers = self.response.json()['resultSets'][0]['headers']\n return pd.DataFrame(shots, columns=headers)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nconnecting to the NATS server.", "response": "def connect(self):\n \"\"\"\n Connect will attempt to connect to the NATS server. The url can\n contain username/password semantics.\n \"\"\"\n self._build_socket()\n self._connect_socket()\n self._build_file_socket()\n self._send_connect_msg()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef subscribe(self, subject, callback, queue=''):\n s = Subscription(\n sid=self._next_sid,\n subject=subject,\n queue=queue,\n callback=callback,\n connetion=self\n )\n\n self._subscriptions[s.sid] = s\n self._send('SUB %s %s %d' % (s.subject, s.queue, s.sid))\n self._next_sid += 1\n\n return s", "response": "Subscribe to a subject."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef publish(self, subject, msg, reply=None):\n if msg is None:\n msg = ''\n\n if reply is None:\n command = 'PUB %s %d' % (subject, len(msg))\n else:\n command = 'PUB %s %s %d' % (subject, reply, len(msg))\n\n self._send(command)\n self._send(msg)", "response": "Publishes the data argument to the subject."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\npublishing a message with an implicit inbox listener as the reply.", "response": "def request(self, subject, callback, msg=None):\n \"\"\"\n ublish a message with an implicit inbox listener as the reply.\n Message is optional.\n\n Args:\n subject (string): a string with the subject\n callback (function): callback to be called\n msg (string=None): payload string\n \"\"\"\n inbox = self._build_inbox()\n s = self.subscribe(inbox, callback)\n self.unsubscribe(s, 1)\n self.publish(subject, msg, inbox)\n\n return s"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwait for a given number of seconds from any subject.", "response": "def wait(self, duration=None, count=0):\n \"\"\"\n Publish publishes the data argument to the given subject.\n\n Args:\n duration (float): will wait for the given number of seconds\n count (count): stop of wait after n messages from any subject\n \"\"\"\n start = time.time()\n total = 0\n while True:\n type, result = self._recv(MSG, PING, OK)\n if type is MSG:\n total += 1\n if self._handle_msg(result) is False:\n break\n\n if count and total >= count:\n break\n\n elif type is PING:\n self._handle_ping()\n\n if duration and time.time() - start > duration:\n break"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef draw_court(ax=None, color='gray', lw=1, outer_lines=False):\n if ax is None:\n ax = plt.gca()\n\n # Create the various parts of an NBA basketball court\n\n # Create the basketball hoop\n hoop = Circle((0, 0), radius=7.5, linewidth=lw, color=color, fill=False)\n\n # Create backboard\n backboard = Rectangle((-30, -12.5), 60, 0, linewidth=lw, color=color)\n\n # The paint\n # Create the outer box 0f the paint, width=16ft, height=19ft\n outer_box = Rectangle((-80, -47.5), 160, 190, linewidth=lw, color=color,\n fill=False)\n # Create the inner box of the paint, widt=12ft, height=19ft\n inner_box = Rectangle((-60, -47.5), 120, 190, linewidth=lw, color=color,\n fill=False)\n\n # Create free throw top arc\n top_free_throw = Arc((0, 142.5), 120, 120, theta1=0, theta2=180,\n linewidth=lw, color=color, fill=False)\n # Create free throw bottom arc\n bottom_free_throw = Arc((0, 142.5), 120, 120, theta1=180, theta2=0,\n linewidth=lw, color=color, linestyle='dashed')\n # Restricted Zone, it is an arc with 4ft radius from center of the hoop\n restricted = Arc((0, 0), 80, 80, theta1=0, theta2=180, linewidth=lw,\n color=color)\n\n # Three point line\n # Create the right side 3pt lines, it's 14ft long before it arcs\n corner_three_a = Rectangle((-220, -47.5), 0, 140, linewidth=lw,\n color=color)\n # Create the right side 3pt lines, it's 14ft long before it arcs\n corner_three_b = Rectangle((220, -47.5), 0, 140, linewidth=lw, color=color)\n # 3pt arc - center of arc will be the hoop, arc is 23'9\" away from hoop\n three_arc = Arc((0, 0), 475, 475, theta1=22, theta2=158, linewidth=lw,\n color=color)\n\n # Center Court\n center_outer_arc = Arc((0, 422.5), 120, 120, theta1=180, theta2=0,\n linewidth=lw, color=color)\n center_inner_arc = Arc((0, 422.5), 40, 40, theta1=180, theta2=0,\n linewidth=lw, color=color)\n\n # List of the court elements to be plotted onto the axes\n court_elements = [hoop, backboard, outer_box, inner_box, top_free_throw,\n bottom_free_throw, restricted, corner_three_a,\n corner_three_b, three_arc, center_outer_arc,\n center_inner_arc]\n\n if outer_lines:\n # Draw the half court line, baseline and side out bound lines\n outer_lines = Rectangle((-250, -47.5), 500, 470, linewidth=lw,\n color=color, fill=False)\n court_elements.append(outer_lines)\n\n # Add the court elements onto the axes\n for element in court_elements:\n ax.add_patch(element)\n\n return ax", "response": "Returns an axes with a basketball court drawn onto to it."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a scatter plot of the player shots.", "response": "def shot_chart(x, y, kind=\"scatter\", title=\"\", color=\"b\", cmap=None,\n xlim=(-250, 250), ylim=(422.5, -47.5),\n court_color=\"gray\", court_lw=1, outer_lines=False,\n flip_court=False, kde_shade=True, gridsize=None, ax=None,\n despine=False, **kwargs):\n \"\"\"\n Returns an Axes object with player shots plotted.\n\n Parameters\n ----------\n\n x, y : strings or vector\n The x and y coordinates of the shots taken. They can be passed in as\n vectors (such as a pandas Series) or as columns from the pandas\n DataFrame passed into ``data``.\n data : DataFrame, optional\n DataFrame containing shots where ``x`` and ``y`` represent the\n shot location coordinates.\n kind : { \"scatter\", \"kde\", \"hex\" }, optional\n The kind of shot chart to create.\n title : str, optional\n The title for the plot.\n color : matplotlib color, optional\n Color used to plot the shots\n cmap : matplotlib Colormap object or name, optional\n Colormap for the range of data values. If one isn't provided, the\n colormap is derived from the valuue passed to ``color``. Used for KDE\n and Hexbin plots.\n {x, y}lim : two-tuples, optional\n The axis limits of the plot.\n court_color : matplotlib color, optional\n The color of the court lines.\n court_lw : float, optional\n The linewidth the of the court lines.\n outer_lines : boolean, optional\n If ``True`` the out of bound lines are drawn in as a matplotlib\n Rectangle.\n flip_court : boolean, optional\n If ``True`` orients the hoop towards the bottom of the plot. Default\n is ``False``, which orients the court where the hoop is towards the top\n of the plot.\n kde_shade : boolean, optional\n Default is ``True``, which shades in the KDE contours.\n gridsize : int, optional\n Number of hexagons in the x-direction. The default is calculated using\n the Freedman-Diaconis method.\n ax : Axes, optional\n The Axes object to plot the court onto.\n despine : boolean, optional\n If ``True``, removes the spines.\n kwargs : key, value pairs\n Keyword arguments for matplotlib Collection properties or seaborn plots.\n\n Returns\n -------\n ax : Axes\n The Axes object with the shot chart plotted on it.\n\n \"\"\"\n\n if ax is None:\n ax = plt.gca()\n\n if cmap is None:\n cmap = sns.light_palette(color, as_cmap=True)\n\n if not flip_court:\n ax.set_xlim(xlim)\n ax.set_ylim(ylim)\n else:\n ax.set_xlim(xlim[::-1])\n ax.set_ylim(ylim[::-1])\n\n ax.tick_params(labelbottom=\"off\", labelleft=\"off\")\n ax.set_title(title, fontsize=18)\n\n draw_court(ax, color=court_color, lw=court_lw, outer_lines=outer_lines)\n\n if kind == \"scatter\":\n ax.scatter(x, y, c=color, **kwargs)\n\n elif kind == \"kde\":\n sns.kdeplot(x, y, shade=kde_shade, cmap=cmap, ax=ax, **kwargs)\n ax.set_xlabel('')\n ax.set_ylabel('')\n\n elif kind == \"hex\":\n if gridsize is None:\n # Get the number of bins for hexbin using Freedman-Diaconis rule\n # This is idea was taken from seaborn, which got the calculation\n # from http://stats.stackexchange.com/questions/798/\n from seaborn.distributions import _freedman_diaconis_bins\n x_bin = _freedman_diaconis_bins(x)\n y_bin = _freedman_diaconis_bins(y)\n gridsize = int(np.mean([x_bin, y_bin]))\n\n ax.hexbin(x, y, gridsize=gridsize, cmap=cmap, **kwargs)\n\n else:\n raise ValueError(\"kind must be 'scatter', 'kde', or 'hex'.\")\n\n # Set the spines to match the rest of court lines, makes outer_lines\n # somewhate unnecessary\n for spine in ax.spines:\n ax.spines[spine].set_lw(court_lw)\n ax.spines[spine].set_color(court_color)\n\n if despine:\n ax.spines[\"top\"].set_visible(False)\n ax.spines[\"bottom\"].set_visible(False)\n ax.spines[\"right\"].set_visible(False)\n ax.spines[\"left\"].set_visible(False)\n\n return ax"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a JointGrid object containing the shot chart for the given location.", "response": "def shot_chart_jointgrid(x, y, data=None, joint_type=\"scatter\", title=\"\",\n joint_color=\"b\", cmap=None, xlim=(-250, 250),\n ylim=(422.5, -47.5), court_color=\"gray\", court_lw=1,\n outer_lines=False, flip_court=False,\n joint_kde_shade=True, gridsize=None,\n marginals_color=\"b\", marginals_type=\"both\",\n marginals_kde_shade=True, size=(12, 11), space=0,\n despine=False, joint_kws=None, marginal_kws=None,\n **kwargs):\n\n \"\"\"\n Returns a JointGrid object containing the shot chart.\n\n This function allows for more flexibility in customizing your shot chart\n than the ``shot_chart_jointplot`` function.\n\n Parameters\n ----------\n\n x, y : strings or vector\n The x and y coordinates of the shots taken. They can be passed in as\n vectors (such as a pandas Series) or as columns from the pandas\n DataFrame passed into ``data``.\n data : DataFrame, optional\n DataFrame containing shots where ``x`` and ``y`` represent the shot\n location coordinates.\n joint_type : { \"scatter\", \"kde\", \"hex\" }, optional\n The type of shot chart for the joint plot.\n title : str, optional\n The title for the plot.\n joint_color : matplotlib color, optional\n Color used to plot the shots on the joint plot.\n cmap : matplotlib Colormap object or name, optional\n Colormap for the range of data values. If one isn't provided, the\n colormap is derived from the value passed to ``color``. Used for KDE\n and Hexbin joint plots.\n {x, y}lim : two-tuples, optional\n The axis limits of the plot. The defaults represent the out of bounds\n lines and half court line.\n court_color : matplotlib color, optional\n The color of the court lines.\n court_lw : float, optional\n The linewidth the of the court lines.\n outer_lines : boolean, optional\n If ``True`` the out of bound lines are drawn in as a matplotlib\n Rectangle.\n flip_court : boolean, optional\n If ``True`` orients the hoop towards the bottom of the plot. Default is\n ``False``, which orients the court where the hoop is towards the top of\n the plot.\n joint_kde_shade : boolean, optional\n Default is ``True``, which shades in the KDE contours on the joint plot.\n gridsize : int, optional\n Number of hexagons in the x-direction. The default is calculated using\n the Freedman-Diaconis method.\n marginals_color : matplotlib color, optional\n Color used to plot the shots on the marginal plots.\n marginals_type : { \"both\", \"hist\", \"kde\"}, optional\n The type of plot for the marginal plots.\n marginals_kde_shade : boolean, optional\n Default is ``True``, which shades in the KDE contours on the marginal\n plots.\n size : tuple, optional\n The width and height of the plot in inches.\n space : numeric, optional\n The space between the joint and marginal plots.\n despine : boolean, optional\n If ``True``, removes the spines.\n {joint, marginal}_kws : dicts\n Additional kewyord arguments for joint and marginal plot components.\n kwargs : key, value pairs\n Keyword arguments for matplotlib Collection properties or seaborn plots.\n\n Returns\n -------\n grid : JointGrid\n The JointGrid object with the shot chart plotted on it.\n\n \"\"\"\n\n # The joint_kws and marginal_kws idea was taken from seaborn\n # Create the default empty kwargs for joint and marginal plots\n if joint_kws is None:\n joint_kws = {}\n joint_kws.update(kwargs)\n\n if marginal_kws is None:\n marginal_kws = {}\n\n # If a colormap is not provided, then it is based off of the joint_color\n if cmap is None:\n cmap = sns.light_palette(joint_color, as_cmap=True)\n\n # Flip the court so that the hoop is by the bottom of the plot\n if flip_court:\n xlim = xlim[::-1]\n ylim = ylim[::-1]\n\n # Create the JointGrid to draw the shot chart plots onto\n grid = sns.JointGrid(x=x, y=y, data=data, xlim=xlim, ylim=ylim,\n space=space)\n\n # Joint Plot\n # Create the main plot of the joint shot chart\n if joint_type == \"scatter\":\n grid = grid.plot_joint(plt.scatter, color=joint_color, **joint_kws)\n\n elif joint_type == \"kde\":\n grid = grid.plot_joint(sns.kdeplot, cmap=cmap,\n shade=joint_kde_shade, **joint_kws)\n\n elif joint_type == \"hex\":\n if gridsize is None:\n # Get the number of bins for hexbin using Freedman-Diaconis rule\n # This is idea was taken from seaborn, which got the calculation\n # from http://stats.stackexchange.com/questions/798/\n from seaborn.distributions import _freedman_diaconis_bins\n x_bin = _freedman_diaconis_bins(x)\n y_bin = _freedman_diaconis_bins(y)\n gridsize = int(np.mean([x_bin, y_bin]))\n\n grid = grid.plot_joint(plt.hexbin, gridsize=gridsize, cmap=cmap,\n **joint_kws)\n\n else:\n raise ValueError(\"joint_type must be 'scatter', 'kde', or 'hex'.\")\n\n # Marginal plots\n # Create the plots on the axis of the main plot of the joint shot chart.\n if marginals_type == \"both\":\n grid = grid.plot_marginals(sns.distplot, color=marginals_color,\n **marginal_kws)\n\n elif marginals_type == \"hist\":\n grid = grid.plot_marginals(sns.distplot, color=marginals_color,\n kde=False, **marginal_kws)\n\n elif marginals_type == \"kde\":\n grid = grid.plot_marginals(sns.kdeplot, color=marginals_color,\n shade=marginals_kde_shade, **marginal_kws)\n\n else:\n raise ValueError(\"marginals_type must be 'both', 'hist', or 'kde'.\")\n\n # Set the size of the joint shot chart\n grid.fig.set_size_inches(size)\n\n # Extract the the first axes, which is the main plot of the\n # joint shot chart, and draw the court onto it\n ax = grid.fig.get_axes()[0]\n draw_court(ax, color=court_color, lw=court_lw, outer_lines=outer_lines)\n\n # Get rid of the axis labels\n grid.set_axis_labels(xlabel=\"\", ylabel=\"\")\n # Get rid of all tick labels\n ax.tick_params(labelbottom=\"off\", labelleft=\"off\")\n # Set the title above the top marginal plot\n ax.set_title(title, y=1.2, fontsize=18)\n\n # Set the spines to match the rest of court lines, makes outer_lines\n # somewhate unnecessary\n for spine in ax.spines:\n ax.spines[spine].set_lw(court_lw)\n ax.spines[spine].set_color(court_color)\n # set the marginal spines to be the same as the rest of the spines\n grid.ax_marg_x.spines[spine].set_lw(court_lw)\n grid.ax_marg_x.spines[spine].set_color(court_color)\n grid.ax_marg_y.spines[spine].set_lw(court_lw)\n grid.ax_marg_y.spines[spine].set_color(court_color)\n\n if despine:\n ax.spines[\"top\"].set_visible(False)\n ax.spines[\"bottom\"].set_visible(False)\n ax.spines[\"right\"].set_visible(False)\n ax.spines[\"left\"].set_visible(False)\n\n return grid"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef shot_chart_jointplot(x, y, data=None, kind=\"scatter\", title=\"\", color=\"b\",\n cmap=None, xlim=(-250, 250), ylim=(422.5, -47.5),\n court_color=\"gray\", court_lw=1, outer_lines=False,\n flip_court=False, size=(12, 11), space=0,\n despine=False, joint_kws=None, marginal_kws=None,\n **kwargs):\n \"\"\"\n Returns a seaborn JointGrid using sns.jointplot\n\n Parameters\n ----------\n\n x, y : strings or vector\n The x and y coordinates of the shots taken. They can be passed in as\n vectors (such as a pandas Series) or as column names from the pandas\n DataFrame passed into ``data``.\n data : DataFrame, optional\n DataFrame containing shots where ``x`` and ``y`` represent the\n shot location coordinates.\n kind : { \"scatter\", \"kde\", \"hex\" }, optional\n The kind of shot chart to create.\n title : str, optional\n The title for the plot.\n color : matplotlib color, optional\n Color used to plot the shots\n cmap : matplotlib Colormap object or name, optional\n Colormap for the range of data values. If one isn't provided, the\n colormap is derived from the valuue passed to ``color``. Used for KDE\n and Hexbin joint plots.\n {x, y}lim : two-tuples, optional\n The axis limits of the plot. The defaults represent the out of bounds\n lines and half court line.\n court_color : matplotlib color, optional\n The color of the court lines.\n court_lw : float, optional\n The linewidth the of the court lines.\n outer_lines : boolean, optional\n If ``True`` the out of bound lines are drawn in as a matplotlib\n Rectangle.\n flip_court : boolean, optional\n If ``True`` orients the hoop towards the bottom of the plot. Default\n is ``False``, which orients the court where the hoop is towards the top\n of the plot.\n gridsize : int, optional\n Number of hexagons in the x-direction. The default is calculated using\n the Freedman-Diaconis method.\n size : tuple, optional\n The width and height of the plot in inches.\n space : numeric, optional\n The space between the joint and marginal plots.\n {joint, marginal}_kws : dicts\n Additional kewyord arguments for joint and marginal plot components.\n kwargs : key, value pairs\n Keyword arguments for matplotlib Collection properties or seaborn plots.\n\n Returns\n -------\n grid : JointGrid\n The JointGrid object with the shot chart plotted on it.\n\n \"\"\"\n\n # If a colormap is not provided, then it is based off of the color\n if cmap is None:\n cmap = sns.light_palette(color, as_cmap=True)\n\n if kind not in [\"scatter\", \"kde\", \"hex\"]:\n raise ValueError(\"kind must be 'scatter', 'kde', or 'hex'.\")\n\n grid = sns.jointplot(x=x, y=y, data=data, stat_func=None, kind=kind,\n space=0, color=color, cmap=cmap, joint_kws=joint_kws,\n marginal_kws=marginal_kws, **kwargs)\n\n grid.fig.set_size_inches(size)\n\n # A joint plot has 3 Axes, the first one called ax_joint\n # is the one we want to draw our court onto and adjust some other settings\n ax = grid.ax_joint\n\n if not flip_court:\n ax.set_xlim(xlim)\n ax.set_ylim(ylim)\n else:\n ax.set_xlim(xlim[::-1])\n ax.set_ylim(ylim[::-1])\n\n draw_court(ax, color=court_color, lw=court_lw, outer_lines=outer_lines)\n\n # Get rid of axis labels and tick marks\n ax.set_xlabel('')\n ax.set_ylabel('')\n ax.tick_params(labelbottom='off', labelleft='off')\n\n # Add a title\n ax.set_title(title, y=1.2, fontsize=18)\n\n # Set the spines to match the rest of court lines, makes outer_lines\n # somewhate unnecessary\n for spine in ax.spines:\n ax.spines[spine].set_lw(court_lw)\n ax.spines[spine].set_color(court_color)\n # set the margin joint spines to be same as the rest of the plot\n grid.ax_marg_x.spines[spine].set_lw(court_lw)\n grid.ax_marg_x.spines[spine].set_color(court_color)\n grid.ax_marg_y.spines[spine].set_lw(court_lw)\n grid.ax_marg_y.spines[spine].set_color(court_color)\n\n if despine:\n ax.spines[\"top\"].set_visible(False)\n ax.spines[\"bottom\"].set_visible(False)\n ax.spines[\"right\"].set_visible(False)\n ax.spines[\"left\"].set_visible(False)\n\n return grid", "response": "Creates a seaborn JointGrid from a set of shots taken from a pandas DataFrame or a pandas Series."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nplotting a heatmap for the court and court number of times shot.", "response": "def heatmap(x, y, z, title=\"\", cmap=plt.cm.YlOrRd, bins=20,\n xlim=(-250, 250), ylim=(422.5, -47.5),\n facecolor='lightgray', facecolor_alpha=0.4,\n court_color=\"black\", court_lw=0.5, outer_lines=False,\n flip_court=False, ax=None, **kwargs):\n\n \"\"\"\n Returns an AxesImage object that contains a heatmap.\n\n TODO: Redo some code and explain parameters\n \"\"\"\n\n # Bin the FGA (x, y) and Calculcate the mean number of times shot was\n # made (z) within each bin\n # mean is the calculated FG percentage for each bin\n mean, xedges, yedges, binnumber = binned_statistic_2d(x=x, y=y,\n values=z,\n statistic='mean',\n bins=bins)\n\n if ax is None:\n ax = plt.gca()\n\n if not flip_court:\n ax.set_xlim(xlim)\n ax.set_ylim(ylim)\n else:\n ax.set_xlim(xlim[::-1])\n ax.set_ylim(ylim[::-1])\n\n ax.tick_params(labelbottom=\"off\", labelleft=\"off\")\n ax.set_title(title, fontsize=18)\n\n ax.patch.set_facecolor(facecolor)\n ax.patch.set_alpha(facecolor_alpha)\n\n draw_court(ax, color=court_color, lw=court_lw, outer_lines=outer_lines)\n\n heatmap = ax.imshow(mean.T, origin='lower', extent=[xedges[0], xedges[-1],\n yedges[0], yedges[-1]], interpolation='nearest',\n cmap=cmap)\n\n return heatmap"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a Bokeh figure with the basketball court lines drawn onto it", "response": "def bokeh_draw_court(figure, line_color='gray', line_width=1):\n \"\"\"Returns a figure with the basketball court lines drawn onto it\n\n This function draws a court based on the x and y-axis values that the NBA\n stats API provides for the shot chart data. For example the center of the\n hoop is located at the (0,0) coordinate. Twenty-two feet from the left of\n the center of the hoop in is represented by the (-220,0) coordinates.\n So one foot equals +/-10 units on the x and y-axis.\n\n Parameters\n ----------\n figure : Bokeh figure object\n The Axes object to plot the court onto.\n line_color : str, optional\n The color of the court lines. Can be a a Hex value.\n line_width : float, optional\n The linewidth the of the court lines in pixels.\n\n Returns\n -------\n figure : Figure\n The Figure object with the court on it.\n\n \"\"\"\n\n # hoop\n figure.circle(x=0, y=0, radius=7.5, fill_alpha=0,\n line_color=line_color, line_width=line_width)\n\n # backboard\n figure.line(x=range(-30, 31), y=-12.5, line_color=line_color)\n\n # The paint\n # outerbox\n figure.rect(x=0, y=47.5, width=160, height=190, fill_alpha=0,\n line_color=line_color, line_width=line_width)\n # innerbox\n # left inner box line\n figure.line(x=-60, y=np.arange(-47.5, 143.5), line_color=line_color,\n line_width=line_width)\n # right inner box line\n figure.line(x=60, y=np.arange(-47.5, 143.5), line_color=line_color,\n line_width=line_width)\n\n # Restricted Zone\n figure.arc(x=0, y=0, radius=40, start_angle=pi, end_angle=0,\n line_color=line_color, line_width=line_width)\n\n # top free throw arc\n figure.arc(x=0, y=142.5, radius=60, start_angle=pi, end_angle=0,\n line_color=line_color)\n\n # bottome free throw arc\n figure.arc(x=0, y=142.5, radius=60, start_angle=0, end_angle=pi,\n line_color=line_color, line_dash=\"dashed\")\n\n # Three point line\n # corner three point lines\n figure.line(x=-220, y=np.arange(-47.5, 92.5), line_color=line_color,\n line_width=line_width)\n figure.line(x=220, y=np.arange(-47.5, 92.5), line_color=line_color,\n line_width=line_width)\n # # three point arc\n figure.arc(x=0, y=0, radius=237.5, start_angle=3.528, end_angle=-0.3863,\n line_color=line_color, line_width=line_width)\n\n # add center court\n # outer center arc\n figure.arc(x=0, y=422.5, radius=60, start_angle=0, end_angle=pi,\n line_color=line_color, line_width=line_width)\n # inner center arct\n figure.arc(x=0, y=422.5, radius=20, start_angle=0, end_angle=pi,\n line_color=line_color, line_width=line_width)\n\n # outer lines, consistting of half court lines and out of bounds lines\n figure.rect(x=0, y=187.5, width=500, height=470, fill_alpha=0,\n line_color=line_color, line_width=line_width)\n\n return figure"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a figure that shows the shot chart on it.", "response": "def bokeh_shot_chart(data, x=\"LOC_X\", y=\"LOC_Y\", fill_color=\"#1f77b4\",\n scatter_size=10, fill_alpha=0.4, line_alpha=0.4,\n court_line_color='gray', court_line_width=1,\n hover_tool=False, tooltips=None, **kwargs):\n\n # TODO: Settings for hover tooltip\n \"\"\"\n Returns a figure with both FGA and basketball court lines drawn onto it.\n\n This function expects data to be a ColumnDataSource with the x and y values\n named \"LOC_X\" and \"LOC_Y\". Otherwise specify x and y.\n\n Parameters\n ----------\n\n data : DataFrame\n The DataFrame that contains the shot chart data.\n x, y : str, optional\n The x and y coordinates of the shots taken.\n fill_color : str, optional\n The fill color of the shots. Can be a a Hex value.\n scatter_size : int, optional\n The size of the dots for the scatter plot.\n fill_alpha : float, optional\n Alpha value for the shots. Must be a floating point value between 0\n (transparent) to 1 (opaque).\n line_alpha : float, optiona\n Alpha value for the outer lines of the plotted shots. Must be a\n floating point value between 0 (transparent) to 1 (opaque).\n court_line_color : str, optional\n The color of the court lines. Can be a a Hex value.\n court_line_width : float, optional\n The linewidth the of the court lines in pixels.\n hover_tool : boolean, optional\n If ``True``, creates hover tooltip for the plot.\n tooltips : List of tuples, optional\n Provides the information for the the hover tooltip.\n\n Returns\n -------\n fig : Figure\n The Figure object with the shot chart plotted on it.\n\n \"\"\"\n source = ColumnDataSource(data)\n\n fig = figure(width=700, height=658, x_range=[-250, 250],\n y_range=[422.5, -47.5], min_border=0, x_axis_type=None,\n y_axis_type=None, outline_line_color=\"black\", **kwargs)\n\n fig.scatter(x, y, source=source, size=scatter_size, color=fill_color,\n alpha=fill_alpha, line_alpha=line_alpha)\n\n bokeh_draw_court(fig, line_color=court_line_color,\n line_width=court_line_width)\n\n if hover_tool:\n hover = HoverTool(renderers=[fig.renderers[0]], tooltips=tooltips)\n fig.add_tools(hover)\n\n return fig"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates Cluster Centers and Sse centers for each cluster.", "response": "def _update_centers(X, membs, n_clusters, distance):\n \"\"\" Update Cluster Centers:\n calculate the mean of feature vectors for each cluster.\n\n distance can be a string or callable.\n \"\"\"\n centers = np.empty(shape=(n_clusters, X.shape[1]), dtype=float)\n sse = np.empty(shape=n_clusters, dtype=float)\n for clust_id in range(n_clusters):\n memb_ids = np.where(membs == clust_id)[0]\n X_clust = X[memb_ids,:]\n\n dist = np.empty(shape=memb_ids.shape[0], dtype=float)\n for i,x in enumerate(X_clust):\n dist[i] = np.sum(scipy.spatial.distance.cdist(X_clust, np.array([x]), distance))\n\n inx_min = np.argmin(dist)\n centers[clust_id,:] = X_clust[inx_min,:]\n sse[clust_id] = dist[inx_min]\n return(centers, sse)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrun a single trial of k - medoids clustering on dataset X and given number of clusters n_clusters and distance distance.", "response": "def _kmedoids_run(X, n_clusters, distance, max_iter, tol, rng):\n \"\"\" Run a single trial of k-medoids clustering\n on dataset X, and given number of clusters\n \"\"\"\n membs = np.empty(shape=X.shape[0], dtype=int)\n centers = kmeans._kmeans_init(X, n_clusters, method='', rng=rng)\n\n sse_last = 9999.9\n n_iter = 0\n for it in range(1,max_iter):\n membs = kmeans._assign_clusters(X, centers)\n centers,sse_arr = _update_centers(X, membs, n_clusters, distance)\n sse_total = np.sum(sse_arr)\n if np.abs(sse_total - sse_last) < tol:\n n_iter = it\n break\n sse_last = sse_total\n\n return(centers, membs, sse_total, sse_arr, n_iter)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fit(self, X):\n self.centers_, self.labels_, self.sse_arr_, self.n_iter_ = \\\n _kmedoids(X, self.n_clusters, self.distance, self.max_iter, self.n_trials, self.tol, self.rng)", "response": "Apply KMeans Clustering\n self. n_clusters clustering clustering"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _kernelized_dist2centers(K, n_clusters, wmemb, kernel_dist):\n n_samples = K.shape[0]\n \n for j in range(n_clusters):\n memb_j = np.where(wmemb == j)[0]\n size_j = memb_j.shape[0]\n\n K_sub_j = K[memb_j][:, memb_j]\n \n kernel_dist[:,j] = 1 + np.sum(K_sub_j) /(size_j*size_j)\n kernel_dist[:,j] -= 2 * np.sum(K[:, memb_j], axis=1) / size_j\n\n return", "response": "Compute the distance in transformed feature space to \n cluster centers."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncomputes the log probability of a multivariate density of a set of means and covars.", "response": "def _log_multivariate_density(X, means, covars):\n \"\"\"\n Class conditional density:\n P(x | mu, Sigma) = 1/((2pi)^d/2 * |Sigma|^1/2) * exp(-1/2 * (x-mu)^T * Sigma^-1 * (x-mu))\n\n log of class conditional density:\n log P(x | mu, Sigma) = -1/2*(d*log(2pi) + log(|Sigma|) + (x-mu)^T * Sigma^-1 * (x-mu))\n \"\"\"\n n_samples, n_dim = X.shape\n n_components = means.shape[0]\n\n assert(means.shape[0] == covars.shape[0])\n\n log_proba = np.empty(shape=(n_samples, n_components), dtype=float)\n for i, (mu, cov) in enumerate(zip(means, covars)):\n try:\n cov_chol = scipy.linalg.cholesky(cov, lower=True)\n except scipy.linalg.LinAlgError:\n try:\n cov_chol = scipy.linalg.cholesky(cov + Lambda*np.eye(n_dim), lower=True)\n except:\n raise ValueError(\"Triangular Matrix Decomposition not performed!\\n\")\n\n cov_log_det = 2 * np.sum(np.log(np.diagonal(cov_chol)))\n\n try:\n cov_solve = scipy.linalg.solve_triangular(cov_chol, (X - mu).T, lower=True).T\n except:\n raise ValueError(\"Solve_triangular not perormed!\\n\")\n\n log_proba[:, i] = -0.5 * (np.sum(cov_solve ** 2, axis=1) + \\\n n_dim * np.log(2 * np.pi) + cov_log_det)\n\n return(log_proba)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncompute the log likelihood of a mixture parameter given data.", "response": "def _log_likelihood_per_sample(X, means, covars):\n \"\"\"\n Theta = (theta_1, theta_2, ... theta_M)\n Likelihood of mixture parameters given data: L(Theta | X) = product_i P(x_i | Theta)\n log likelihood: log L(Theta | X) = sum_i log(P(x_i | Theta))\n\n and note that p(x_i | Theta) = sum_j prior_j * p(x_i | theta_j)\n\n\n Probability of sample x being generated from component i:\n P(w_i | x) = P(x|w_i) * P(w_i) / P(X)\n where P(X) = sum_i P(x|w_i) * P(w_i)\n\n Here post_proba = P/(w_i | x)\n and log_likelihood = log(P(x|w_i))\n \"\"\"\n \n logden = _log_multivariate_density(X, means, covars) \n\n logden_max = logden.max(axis=1)\n log_likelihood = np.log(np.sum(np.exp(logden.T - logden_max) + Epsilon, axis=0))\n log_likelihood += logden_max\n\n post_proba = np.exp(logden - log_likelihood[:, np.newaxis])\n\n return (log_likelihood, post_proba)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _validate_params(priors, means, covars):\n\n for i,(p,m,cv) in enumerate(zip(priors, means, covars)):\n if np.any(np.isinf(p)) or np.any(np.isnan(p)):\n raise ValueError(\"Component %d of priors is not valid \" % i)\n\n if np.any(np.isinf(m)) or np.any(np.isnan(m)):\n raise ValueError(\"Component %d of means is not valid \" % i)\n\n if np.any(np.isinf(cv)) or np.any(np.isnan(cv)):\n raise ValueError(\"Component %d of covars is not valid \" % i)\n\n if (not np.allclose(cv, cv.T) or np.any(scipy.linalg.eigvalsh(cv) <= 0)):\n raise ValueError(\"Component %d of covars must be positive-definite\" % i)", "response": "Validate parameters of the M. L. paramateres\n ."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nupdate class parameters as below: priors: P(w_i) = sum_x P(w_i | x) ==> Then normalize to get in [0,1] Class means: center_w_i = sum_x P(w_i|x)*x / sum_i sum_x P(w_i|x)", "response": "def _maximization_step(X, posteriors):\n \"\"\" \n Update class parameters as below:\n priors: P(w_i) = sum_x P(w_i | x) ==> Then normalize to get in [0,1]\n Class means: center_w_i = sum_x P(w_i|x)*x / sum_i sum_x P(w_i|x)\n \"\"\"\n\n ### Prior probabilities or class weights\n sum_post_proba = np.sum(posteriors, axis=0)\n prior_proba = sum_post_proba / (sum_post_proba.sum() + Epsilon)\n \n ### means\n means = np.dot(posteriors.T, X) / (sum_post_proba[:, np.newaxis] + Epsilon)\n\n ### covariance matrices\n n_components = posteriors.shape[1]\n n_features = X.shape[1]\n covars = np.empty(shape=(n_components, n_features, n_features), dtype=float)\n \n for i in range(n_components):\n post_i = posteriors[:, i]\n mean_i = means[i]\n diff_i = X - mean_i\n\n with np.errstate(under='ignore'):\n covar_i = np.dot(post_i * diff_i.T, diff_i) / (post_i.sum() + Epsilon)\n covars[i] = covar_i + Lambda * np.eye(n_features)\n\n\n _validate_params(prior_proba, means, covars)\n return(prior_proba, means, covars)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef fit(self, X):\n params_dict = _fit_gmm_params(X=X, n_mixtures=self.n_clusters, \\\n n_init=self.n_trials, init_method=self.init_method, \\\n n_iter=self.max_iter, tol=self.tol)\n self.priors_ = params_dict['priors']\n self.means_ = params_dict['means']\n self.covars_ = params_dict['covars']\n\n self.converged = True\n self.labels_ = self.predict(X)", "response": "Fit mixture - density parameters with EM algorithm"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _kmeans_init(X, n_clusters, method='balanced', rng=None):\n n_samples = X.shape[0]\n if rng is None:\n cent_idx = np.random.choice(n_samples, replace=False, size=n_clusters)\n else:\n #print('Generate random centers using RNG')\n cent_idx = rng.choice(n_samples, replace=False, size=n_clusters)\n \n centers = X[cent_idx,:]\n mean_X = np.mean(X, axis=0)\n \n if method == 'balanced':\n centers[n_clusters-1] = n_clusters*mean_X - np.sum(centers[:(n_clusters-1)], axis=0)\n \n return (centers)", "response": "Initialize k = n_clusters centroids randomly\n "} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _assign_clusters(X, centers):\n dist2cents = scipy.spatial.distance.cdist(X, centers, metric='seuclidean')\n membs = np.argmin(dist2cents, axis=1)\n\n return(membs)", "response": "Assign each point to the closet cluster center"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _cal_dist2center(X, center):\n dmemb2cen = scipy.spatial.distance.cdist(X, center.reshape(1,X.shape[1]), metric='seuclidean')\n return(np.sum(dmemb2cen))", "response": "Calculate the SSE to the cluster center"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _update_centers(X, membs, n_clusters):\n centers = np.empty(shape=(n_clusters, X.shape[1]), dtype=float)\n sse = np.empty(shape=n_clusters, dtype=float)\n for clust_id in range(n_clusters):\n memb_ids = np.where(membs == clust_id)[0]\n\n if memb_ids.shape[0] == 0:\n memb_ids = np.random.choice(X.shape[0], size=1)\n #print(\"Empty cluster replaced with \", memb_ids)\n centers[clust_id,:] = np.mean(X[memb_ids,:], axis=0)\n \n sse[clust_id] = _cal_dist2center(X[memb_ids,:], centers[clust_id,:]) \n return(centers, sse)", "response": "Update Cluster Centers:\n calculate the mean of feature vectors for each cluster"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun a single trial of k - means clustering on dataset X and given number of clusters", "response": "def _kmeans_run(X, n_clusters, max_iter, tol):\n \"\"\" Run a single trial of k-means clustering\n on dataset X, and given number of clusters\n \"\"\"\n membs = np.empty(shape=X.shape[0], dtype=int)\n centers = _kmeans_init(X, n_clusters)\n\n sse_last = 9999.9\n n_iter = 0\n for it in range(1,max_iter):\n membs = _assign_clusters(X, centers)\n centers,sse_arr = _update_centers(X, membs, n_clusters)\n sse_total = np.sum(sse_arr)\n if np.abs(sse_total - sse_last) < tol:\n n_iter = it\n break\n sse_last = sse_total\n\n return(centers, membs, sse_total, sse_arr, n_iter)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrunning multiple trials of k - means clustering and k - means cluster labels and outputt he best centers and cluster labels", "response": "def _kmeans(X, n_clusters, max_iter, n_trials, tol):\n \"\"\" Run multiple trials of k-means clustering,\n and outputt he best centers, and cluster labels\n \"\"\"\n n_samples, n_features = X.shape[0], X.shape[1]\n\n centers_best = np.empty(shape=(n_clusters,n_features), dtype=float)\n labels_best = np.empty(shape=n_samples, dtype=int)\n for i in range(n_trials):\n centers, labels, sse_tot, sse_arr, n_iter = _kmeans_run(X, n_clusters, max_iter, tol)\n if i==0:\n sse_tot_best = sse_tot\n sse_arr_best = sse_arr\n n_iter_best = n_iter\n centers_best = centers.copy()\n labels_best = labels.copy()\n if sse_tot < sse_tot_best:\n sse_tot_best = sse_tot\n sse_arr_best = sse_arr\n n_iter_best = n_iter\n centers_best = centers.copy()\n labels_best = labels.copy()\n\n return(centers_best, labels_best, sse_arr_best, n_iter_best)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fit(self, X):\n self.centers_, self.labels_, self.sse_arr_, self.n_iter_ = \\\n _kmeans(X, self.n_clusters, self.max_iter, self.n_trials, self.tol)", "response": "Apply KMeans Clustering\n X: dataset with feature vectors"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _cut_tree(tree, n_clusters, membs):\n ## starting from root,\n ## a node is added to the cut_set or \n ## its children are added to node_set\n assert(n_clusters >= 2)\n assert(n_clusters <= len(tree.leaves()))\n\n cut_centers = dict() #np.empty(shape=(n_clusters, ndim), dtype=float)\n \n for i in range(n_clusters-1):\n if i==0:\n search_set = set(tree.children(0))\n node_set,cut_set = set(), set()\n else:\n search_set = node_set.union(cut_set)\n node_set,cut_set = set(), set()\n\n if i+2 == n_clusters:\n cut_set = search_set\n else:\n for _ in range(len(search_set)):\n n = search_set.pop()\n \n if n.data['ilev'] is None or n.data['ilev']>i+2:\n cut_set.add(n)\n else:\n nid = n.identifier\n if n.data['ilev']-2==i:\n node_set = node_set.union(set(tree.children(nid)))\n \n conv_membs = membs.copy()\n for node in cut_set:\n nid = node.identifier\n label = node.data['label']\n cut_centers[label] = node.data['center']\n sub_leaves = tree.leaves(nid)\n for leaf in sub_leaves:\n indx = np.where(conv_membs == leaf)[0]\n conv_membs[indx] = nid\n\n return(conv_membs, cut_centers)", "response": "Cut the tree to get desired number of clusters as n_desired < n_clusters"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _add_tree_node(tree, label, ilev, X=None, size=None, center=None, sse=None, parent=None):\n if size is None:\n size = X.shape[0]\n if (center is None):\n center = np.mean(X, axis=0)\n if (sse is None):\n sse = _kmeans._cal_dist2center(X, center)\n\n center = list(center)\n datadict = {\n 'size' : size,\n 'center': center, \n 'label' : label, \n 'sse' : sse,\n 'ilev' : None \n }\n if (parent is None):\n tree.create_node(label, label, data=datadict)\n else:\n tree.create_node(label, label, parent=parent, data=datadict)\n tree.get_node(parent).data['ilev'] = ilev\n\n return(tree)", "response": "Add a node to the tree"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nbisect Kmeans clustering to reach n_clusters number of clusters", "response": "def _bisect_kmeans(X, n_clusters, n_trials, max_iter, tol):\n \"\"\" Apply Bisecting Kmeans clustering\n to reach n_clusters number of clusters\n \"\"\"\n membs = np.empty(shape=X.shape[0], dtype=int)\n centers = dict() #np.empty(shape=(n_clusters,X.shape[1]), dtype=float)\n sse_arr = dict() #-1.0*np.ones(shape=n_clusters, dtype=float)\n\n ## data structure to store cluster hierarchies\n tree = treelib.Tree()\n tree = _add_tree_node(tree, 0, ilev=0, X=X) \n\n km = _kmeans.KMeans(n_clusters=2, n_trials=n_trials, max_iter=max_iter, tol=tol)\n for i in range(1,n_clusters):\n sel_clust_id,sel_memb_ids = _select_cluster_2_split(membs, tree)\n X_sub = X[sel_memb_ids,:]\n km.fit(X_sub)\n\n #print(\"Bisecting Step %d :\"%i, sel_clust_id, km.sse_arr_, km.centers_)\n ## Updating the clusters & properties\n #sse_arr[[sel_clust_id,i]] = km.sse_arr_\n #centers[[sel_clust_id,i]] = km.centers_\n tree = _add_tree_node(tree, 2*i-1, i, \\\n size=np.sum(km.labels_ == 0), center=km.centers_[0], \\\n sse=km.sse_arr_[0], parent= sel_clust_id)\n tree = _add_tree_node(tree, 2*i, i, \\\n size=np.sum(km.labels_ == 1), center=km.centers_[1], \\\n sse=km.sse_arr_[1], parent= sel_clust_id)\n\n pred_labels = km.labels_\n pred_labels[np.where(pred_labels == 1)[0]] = 2*i\n pred_labels[np.where(pred_labels == 0)[0]] = 2*i - 1\n #if sel_clust_id == 1:\n # pred_labels[np.where(pred_labels == 0)[0]] = sel_clust_id\n # pred_labels[np.where(pred_labels == 1)[0]] = i\n #else:\n # pred_labels[np.where(pred_labels == 1)[0]] = i\n # pred_labels[np.where(pred_labels == 0)[0]] = sel_clust_id\n\n membs[sel_memb_ids] = pred_labels\n\n\n for n in tree.leaves():\n label = n.data['label']\n centers[label] = n.data['center']\n sse_arr[label] = n.data['sse']\n\n return(centers, membs, sse_arr, tree)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a LaTeX ready table of model comparisons.", "response": "def comparison_table(self, caption=None, label=\"tab:model_comp\", hlines=True,\n aic=True, bic=True, dic=True, sort=\"bic\", descending=True): # pragma: no cover\n \"\"\"\n Return a LaTeX ready table of model comparisons.\n\n Parameters\n ----------\n caption : str, optional\n The table caption to insert.\n label : str, optional\n The table label to insert.\n hlines : bool, optional\n Whether to insert hlines in the table or not.\n aic : bool, optional\n Whether to include a column for AICc or not.\n bic : bool, optional\n Whether to include a column for BIC or not.\n dic : bool, optional\n Whether to include a column for DIC or not.\n sort : str, optional\n How to sort the models. Should be one of \"bic\", \"aic\" or \"dic\".\n descending : bool, optional\n The sort order.\n\n Returns\n -------\n str\n A LaTeX table to be copied into your document.\n \"\"\"\n\n if sort == \"bic\":\n assert bic, \"You cannot sort by BIC if you turn it off\"\n if sort == \"aic\":\n assert aic, \"You cannot sort by AIC if you turn it off\"\n if sort == \"dic\":\n assert dic, \"You cannot sort by DIC if you turn it off\"\n\n if caption is None:\n caption = \"\"\n if label is None:\n label = \"\"\n\n base_string = get_latex_table_frame(caption, label)\n end_text = \" \\\\\\\\ \\n\"\n num_cols = 1 + (1 if aic else 0) + (1 if bic else 0)\n column_text = \"c\" * (num_cols + 1)\n center_text = \"\"\n hline_text = \"\\\\hline\\n\"\n if hlines:\n center_text += hline_text\n center_text += \"\\tModel\" + (\" & AIC\" if aic else \"\") + (\" & BIC \" if bic else \"\") \\\n + (\" & DIC \" if dic else \"\") + end_text\n if hlines:\n center_text += \"\\t\" + hline_text\n if aic:\n aics = self.aic()\n else:\n aics = np.zeros(len(self.parent.chains))\n if bic:\n bics = self.bic()\n else:\n bics = np.zeros(len(self.parent.chains))\n if dic:\n dics = self.dic()\n else:\n dics = np.zeros(len(self.parent.chains))\n\n if sort == \"bic\":\n to_sort = bics\n elif sort == \"aic\":\n to_sort = aics\n elif sort == \"dic\":\n to_sort = dics\n else:\n raise ValueError(\"sort %s not recognised, must be dic, aic or dic\" % sort)\n\n good = [i for i, t in enumerate(to_sort) if t is not None]\n names = [self.parent.chains[g].name for g in good]\n aics = [aics[g] for g in good]\n bics = [bics[g] for g in good]\n to_sort = bics if sort == \"bic\" else aics\n\n indexes = np.argsort(to_sort)\n\n if descending:\n indexes = indexes[::-1]\n\n for i in indexes:\n line = \"\\t\" + names[i]\n if aic:\n line += \" & %5.1f \" % aics[i]\n if bic:\n line += \" & %5.1f \" % bics[i]\n if dic:\n line += \" & %5.1f \" % dics[i]\n line += end_text\n center_text += line\n if hlines:\n center_text += \"\\t\" + hline_text\n\n return base_string % (column_text, center_text)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nestimating un - normalised probability density at target points.", "response": "def evaluate(self, data):\n \"\"\" Estimate un-normalised probability density at target points\n \n Parameters\n ----------\n data : np.ndarray\n A `(num_targets, num_dim)` array of points to investigate. \n \n Returns\n -------\n np.ndarray\n A `(num_targets)` length array of estimates\n\n Returns array of probability densities\n \"\"\"\n if len(data.shape) == 1 and self.num_dim == 1:\n data = np.atleast_2d(data).T\n\n _d = np.dot(data - self.mean, self.A)\n\n # Get all points within range of kernels\n neighbors = self.tree.query_ball_point(_d, self.sigma * self.truncation)\n out = []\n for i, n in enumerate(neighbors):\n if len(n) >= self.nmin:\n diff = self.d[n, :] - _d[i]\n distsq = np.sum(diff * diff, axis=1)\n else:\n # If too few points get nmin closest\n dist, n = self.tree.query(_d[i], k=self.nmin)\n distsq = dist * dist\n out.append(np.sum(self.weights[n] * np.exp(self.sigma_fact * distsq)))\n return np.array(out)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot(self, figsize=\"GROW\", parameters=None, chains=None, extents=None, filename=None,\n display=False, truth=None, legend=None, blind=None, watermark=None): # pragma: no cover\n \"\"\" Plot the chain!\n\n Parameters\n ----------\n figsize : str|tuple(float)|float, optional\n The figure size to generate. Accepts a regular two tuple of size in inches,\n or one of several key words. The default value of ``COLUMN`` creates a figure\n of appropriate size of insertion into an A4 LaTeX document in two-column mode.\n ``PAGE`` creates a full page width figure. ``GROW`` creates an image that\n scales with parameters (1.5 inches per parameter). String arguments are not\n case sensitive. If you pass a float, it will scale the default ``GROW`` by\n that amount, so ``2.0`` would result in a plot 3 inches per parameter.\n parameters : list[str]|int, optional\n If set, only creates a plot for those specific parameters (if list). If an\n integer is given, only plots the fist so many parameters.\n chains : int|str, list[str|int], optional\n Used to specify which chain to show if more than one chain is loaded in.\n Can be an integer, specifying the\n chain index, or a str, specifying the chain name.\n extents : list[tuple[float]] or dict[str], optional\n Extents are given as two-tuples. You can pass in a list the same size as\n parameters (or default parameters if you don't specify parameters),\n or as a dictionary.\n filename : str, optional\n If set, saves the figure to this location\n display : bool, optional\n If True, shows the figure using ``plt.show()``.\n truth : list[float] or dict[str], optional\n A list of truth values corresponding to parameters, or a dictionary of\n truth values indexed by key\n legend : bool, optional\n If true, creates a legend in your plot using the chain names.\n blind : bool|string|list[string], optional\n Whether to blind axes values. Can be set to `True` to blind all parameters,\n or can pass in a string (or list of strings) which specify the parameters to blind.\n watermark : str, optional\n A watermark to add to the figure\n\n Returns\n -------\n figure\n the matplotlib figure\n\n \"\"\"\n\n chains, parameters, truth, extents, blind = self._sanitise(chains, parameters, truth,\n extents, color_p=True, blind=blind)\n names = [chain.name for chain in chains]\n\n if legend is None:\n legend = len(chains) > 1\n\n # If no chains have names, don't plot the legend\n legend = legend and len([n for n in names if n]) > 0\n\n # Calculate cmap extents\n unique_color_params = list(set([c.config[\"color_params\"] for c in chains if c.config[\"color_params\"] is not None]))\n num_cax = len(unique_color_params)\n color_param_extents = {}\n for u in unique_color_params:\n umin, umax = np.inf, -np.inf\n for chain in chains:\n if chain.config[\"color_params\"] == u:\n data = chain.get_color_data()\n if data is not None:\n umin = min(umin, data.min())\n umax = max(umax, data.max())\n color_param_extents[u] = (umin, umax)\n\n grow_size = 1.5\n if isinstance(figsize, float):\n grow_size *= figsize\n figsize = \"GROW\"\n\n if isinstance(figsize, str):\n if figsize.upper() == \"COLUMN\":\n figsize = (5 + (1 if num_cax > 0 else 0), 5)\n elif figsize.upper() == \"PAGE\":\n figsize = (10, 10)\n elif figsize.upper() == \"GROW\":\n figsize = (grow_size * len(parameters) + num_cax * 1.0, grow_size * len(parameters))\n else:\n raise ValueError(\"Unknown figure size %s\" % figsize)\n elif isinstance(figsize, float):\n figsize = (figsize * grow_size * len(parameters), figsize * grow_size * len(parameters))\n\n plot_hists = self.parent.config[\"plot_hists\"]\n flip = (len(parameters) == 2 and plot_hists and self.parent.config[\"flip\"])\n\n fig, axes, params1, params2, extents = self._get_figure(parameters, chains=chains, figsize=figsize, flip=flip,\n external_extents=extents, blind=blind)\n axl = axes.ravel().tolist()\n summary = self.parent.config[\"summary\"]\n\n if summary is None:\n summary = len(parameters) < 5 and len(self.parent.chains) == 1\n if len(chains) == 1:\n self._logger.debug(\"Plotting surfaces for chain of dimension %s\" %\n (chains[0].chain.shape,))\n else:\n self._logger.debug(\"Plotting surfaces for %d chains\" % len(chains))\n cbar_done = []\n\n chain_points = [c for c in chains if c.config[\"plot_point\"]]\n num_chain_points = len(chain_points)\n if num_chain_points:\n subgroup_names = list(set([c.name for c in chain_points]))\n subgroups = [[c for c in chain_points if c.name == n] for n in subgroup_names]\n markers = [group[0].config[\"marker_style\"] for group in subgroups] # Only one marker per group\n marker_sizes = [[g.config[\"marker_size\"] for g in group] for group in subgroups] # But size can diff\n marker_alphas = [group[0].config[\"marker_alpha\"] for group in subgroups] # Only one marker per group\n for i, p1 in enumerate(params1):\n for j, p2 in enumerate(params2):\n if i < j:\n continue\n ax = axes[i, j]\n do_flip = (flip and i == len(params1) - 1)\n\n # Plot the histograms\n if plot_hists and i == j:\n if do_flip:\n self._add_truth(ax, truth, p1)\n else:\n self._add_truth(ax, truth, None, py=p2)\n max_val = None\n\n # Plot each chain\n for chain in chains:\n if p1 not in chain.parameters:\n continue\n if not chain.config[\"plot_contour\"]:\n continue\n\n param_summary = summary and p1 not in blind\n m = self._plot_bars(ax, p1, chain, flip=do_flip, summary=param_summary)\n\n if max_val is None or m > max_val:\n max_val = m\n\n if num_chain_points and self.parent.config[\"global_point\"]:\n m = self._plot_point_histogram(ax, subgroups, p1, flip=do_flip)\n if max_val is None or m > max_val:\n max_val = m\n\n if max_val is not None:\n if do_flip:\n ax.set_xlim(0, 1.1 * max_val)\n else:\n ax.set_ylim(0, 1.1 * max_val)\n\n else:\n for chain in chains:\n if p1 not in chain.parameters or p2 not in chain.parameters:\n continue\n if not chain.config[\"plot_contour\"]:\n continue\n h = None\n if p1 in chain.parameters and p2 in chain.parameters:\n h = self._plot_contour(ax, chain, p1, p2, color_extents=color_param_extents)\n cp = chain.config[\"color_params\"]\n if h is not None and cp is not None and cp not in cbar_done:\n cbar_done.append(cp)\n aspect = figsize[1] / 0.15\n fraction = 0.85 / figsize[0]\n cbar = fig.colorbar(h, ax=axl, aspect=aspect, pad=0.03, fraction=fraction, drawedges=False)\n label = cp\n if label == \"weights\":\n label = \"Weights\"\n elif label == \"log_weights\":\n label = \"log(Weights)\"\n elif label == \"posterior\":\n label = \"log(Posterior)\"\n cbar.set_label(label, fontsize=14)\n cbar.solids.set(alpha=1)\n\n if num_chain_points:\n self._plot_points(ax, subgroups, markers, marker_sizes, marker_alphas, p1, p2)\n\n self._add_truth(ax, truth, p1, py=p2)\n\n colors = [c.config[\"color\"] for c in chains]\n plot_points = [c.config[\"plot_point\"] for c in chains]\n plot_contours = [c.config[\"plot_contour\"] for c in chains]\n linestyles = [c.config[\"linestyle\"] for c in chains]\n linewidths = [c.config[\"linewidth\"] for c in chains]\n marker_styles = [c.config[\"marker_style\"] for c in chains]\n marker_sizes = [c.config[\"marker_size\"] for c in chains]\n legend_kwargs = self.parent.config[\"legend_kwargs\"]\n legend_artists = self.parent.config[\"legend_artists\"]\n legend_color_text = self.parent.config[\"legend_color_text\"]\n legend_location = self.parent.config[\"legend_location\"]\n\n if legend_location is None:\n if not flip or len(parameters) > 2:\n legend_location = (0, -1)\n else:\n legend_location = (-1, 0)\n outside = (legend_location[0] >= legend_location[1])\n if names is not None and legend:\n ax = axes[legend_location[0], legend_location[1]]\n if \"markerfirst\" not in legend_kwargs:\n # If we have legend inside a used subplot, switch marker order\n legend_kwargs[\"markerfirst\"] = outside or not legend_artists\n linewidths2 = linewidths if legend_artists else [0]*len(linewidths)\n linestyles2 = linestyles if legend_artists else [\"-\"]*len(linestyles)\n marker_sizes2 = marker_sizes if legend_artists else [0]*len(linestyles)\n\n artists = []\n done_names = []\n final_colors = []\n for i, (n, c, ls, lw, marker, size, pp, pc) in enumerate(zip(names, colors, linestyles2, linewidths2,\n marker_styles, marker_sizes2, plot_points, plot_contours)):\n if n is None or n in done_names:\n continue\n done_names.append(n)\n final_colors.append(c)\n size = np.sqrt(size) # plot vs scatter use size differently, hence the sqrt\n if pc and not pp:\n artists.append(plt.Line2D((0, 1), (0, 0), color=c, ls=ls, lw=lw))\n elif not pc and pp:\n artists.append(plt.Line2D((0, 1), (0, 0), color=c, ls=ls, lw=0, marker=marker, markersize=size))\n else:\n artists.append(plt.Line2D((0, 1), (0, 0), color=c, ls=ls, lw=lw, marker=marker, markersize=size))\n\n leg = ax.legend(artists, done_names, **legend_kwargs)\n if legend_color_text:\n for text, c in zip(leg.get_texts(), final_colors):\n text.set_weight(\"medium\")\n text.set_color(c)\n if not outside:\n loc = legend_kwargs.get(\"loc\") or \"\"\n if \"right\" in loc.lower():\n vp = leg._legend_box._children[-1]._children[0]\n vp.align = \"right\"\n\n fig.canvas.draw()\n for ax in axes[-1, :]:\n offset = ax.get_xaxis().get_offset_text()\n ax.set_xlabel('{0} {1}'.format(ax.get_xlabel(), \"[{0}]\".format(offset.get_text()) if offset.get_text() else \"\"))\n offset.set_visible(False)\n for ax in axes[:, 0]:\n offset = ax.get_yaxis().get_offset_text()\n ax.set_ylabel('{0} {1}'.format(ax.get_ylabel(), \"[{0}]\".format(offset.get_text()) if offset.get_text() else \"\"))\n offset.set_visible(False)\n\n dpi = 300\n if watermark:\n if flip and len(parameters) == 2:\n ax = axes[-1, 0]\n else:\n ax = None\n self._add_watermark(fig, ax, figsize, watermark, dpi=dpi)\n\n if filename is not None:\n if isinstance(filename, str):\n filename = [filename]\n for f in filename:\n self._save_fig(fig, f, dpi)\n if display:\n plt.show()\n\n return fig", "response": "Plots the key - word entry for the current location."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef plot_walks(self, parameters=None, truth=None, extents=None, display=False,\n filename=None, chains=None, convolve=None, figsize=None,\n plot_weights=True, plot_posterior=True, log_weight=None): # pragma: no cover\n \"\"\" Plots the chain walk; the parameter values as a function of step index.\n\n This plot is more for a sanity or consistency check than for use with final results.\n Plotting this before plotting with :func:`plot` allows you to quickly see if the\n chains are well behaved, or if certain parameters are suspect\n or require a greater burn in period.\n\n The desired outcome is to see an unchanging distribution along the x-axis of the plot.\n If there are obvious tails or features in the parameters, you probably want\n to investigate.\n\n Parameters\n ----------\n parameters : list[str]|int, optional\n Specify a subset of parameters to plot. If not set, all parameters are plotted.\n If an integer is given, only the first so many parameters are plotted.\n truth : list[float]|dict[str], optional\n A list of truth values corresponding to parameters, or a dictionary of\n truth values keyed by the parameter.\n extents : list[tuple]|dict[str], optional\n A list of two-tuples for plot extents per parameter, or a dictionary of\n extents keyed by the parameter.\n display : bool, optional\n If set, shows the plot using ``plt.show()``\n filename : str, optional\n If set, saves the figure to the filename\n chains : int|str, list[str|int], optional\n Used to specify which chain to show if more than one chain is loaded in.\n Can be an integer, specifying the\n chain index, or a str, specifying the chain name.\n convolve : int, optional\n If set, overplots a smoothed version of the steps using ``convolve`` as\n the width of the smoothing filter.\n figsize : tuple, optional\n If set, sets the created figure size.\n plot_weights : bool, optional\n If true, plots the weight if they are available\n plot_posterior : bool, optional\n If true, plots the log posterior if they are available\n log_weight : bool, optional\n Whether to display weights in log space or not. If None, the value is\n inferred by the mean weights of the plotted chains.\n\n Returns\n -------\n figure\n the matplotlib figure created\n\n \"\"\"\n\n chains, parameters, truth, extents, _ = self._sanitise(chains, parameters, truth, extents)\n\n n = len(parameters)\n extra = 0\n if plot_weights:\n plot_weights = plot_weights and np.any([np.any(c.weights != 1.0) for c in chains])\n\n plot_posterior = plot_posterior and np.any([c.posterior is not None for c in chains])\n\n if plot_weights:\n extra += 1\n if plot_posterior:\n extra += 1\n\n if figsize is None:\n figsize = (8, 0.75 + (n + extra))\n\n fig, axes = plt.subplots(figsize=figsize, nrows=n + extra, squeeze=False, sharex=True)\n\n for i, axes_row in enumerate(axes):\n ax = axes_row[0]\n if i >= extra:\n p = parameters[i - n]\n for chain in chains:\n if p in chain.parameters:\n chain_row = chain.get_data(p)\n self._plot_walk(ax, p, chain_row, extents=extents.get(p), convolve=convolve, color=chain.config[\"color\"])\n if truth.get(p) is not None:\n self._plot_walk_truth(ax, truth.get(p))\n else:\n if i == 0 and plot_posterior:\n for chain in chains:\n if chain.posterior is not None:\n self._plot_walk(ax, \"$\\log(P)$\", chain.posterior - chain.posterior.max(),\n convolve=convolve, color=chain.config[\"color\"])\n else:\n if log_weight is None:\n log_weight = np.any([chain.weights.mean() < 0.1 for chain in chains])\n if log_weight:\n for chain in chains:\n self._plot_walk(ax, r\"$\\log_{10}(w)$\", np.log10(chain.weights),\n convolve=convolve, color=chain.config[\"color\"])\n else:\n for chain in chains:\n self._plot_walk(ax, \"$w$\", chain.weights,\n convolve=convolve, color=chain.config[\"color\"])\n\n if filename is not None:\n if isinstance(filename, str):\n filename = [filename]\n for f in filename:\n self._save_fig(fig, f, 300)\n if display:\n plt.show()\n return fig", "response": "Plots the chain walks for the specified parameters."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plot_distributions(self, parameters=None, truth=None, extents=None, display=False,\n filename=None, chains=None, col_wrap=4, figsize=None, blind=None): # pragma: no cover\n \"\"\" Plots the 1D parameter distributions for verification purposes.\n\n This plot is more for a sanity or consistency check than for use with final results.\n Plotting this before plotting with :func:`plot` allows you to quickly see if the\n chains give well behaved distributions, or if certain parameters are suspect\n or require a greater burn in period.\n\n\n Parameters\n ----------\n parameters : list[str]|int, optional\n Specify a subset of parameters to plot. If not set, all parameters are plotted.\n If an integer is given, only the first so many parameters are plotted.\n truth : list[float]|dict[str], optional\n A list of truth values corresponding to parameters, or a dictionary of\n truth values keyed by the parameter.\n extents : list[tuple]|dict[str], optional\n A list of two-tuples for plot extents per parameter, or a dictionary of\n extents keyed by the parameter.\n display : bool, optional\n If set, shows the plot using ``plt.show()``\n filename : str, optional\n If set, saves the figure to the filename\n chains : int|str, list[str|int], optional\n Used to specify which chain to show if more than one chain is loaded in.\n Can be an integer, specifying the\n chain index, or a str, specifying the chain name.\n col_wrap : int, optional\n How many columns to plot before wrapping.\n figsize : tuple(float)|float, optional\n Either a tuple specifying the figure size or a float scaling factor.\n blind : bool|string|list[string], optional\n Whether to blind axes values. Can be set to `True` to blind all parameters,\n or can pass in a string (or list of strings) which specify the parameters to blind.\n\n Returns\n -------\n figure\n the matplotlib figure created\n\n \"\"\"\n chains, parameters, truth, extents, blind = self._sanitise(chains, parameters, truth, extents, blind=blind)\n\n n = len(parameters)\n num_cols = min(n, col_wrap)\n num_rows = int(np.ceil(1.0 * n / col_wrap))\n\n if figsize is None:\n figsize = 1.0\n if isinstance(figsize, float):\n figsize_float = figsize\n figsize = (num_cols * 2 * figsize, num_rows * 2 * figsize)\n else:\n figsize_float = 1.0\n\n summary = self.parent.config[\"summary\"]\n label_font_size = self.parent.config[\"label_font_size\"]\n tick_font_size = self.parent.config[\"tick_font_size\"]\n max_ticks = self.parent.config[\"max_ticks\"]\n diagonal_tick_labels = self.parent.config[\"diagonal_tick_labels\"]\n\n if summary is None:\n summary = len(self.parent.chains) == 1\n\n hspace = (0.8 if summary else 0.5) / figsize_float\n fig, axes = plt.subplots(nrows=num_rows, ncols=num_cols, figsize=figsize, squeeze=False)\n fig.subplots_adjust(left=0.1, right=0.95, top=0.95, bottom=0.1, wspace=0.05, hspace=hspace)\n\n formatter = ScalarFormatter(useOffset=False)\n formatter.set_powerlimits((-3, 4))\n\n for i, ax in enumerate(axes.flatten()):\n if i >= len(parameters):\n ax.set_axis_off()\n continue\n p = parameters[i]\n\n ax.set_yticks([])\n if p in blind:\n ax.set_xticks([])\n else:\n if diagonal_tick_labels:\n _ = [l.set_rotation(45) for l in ax.get_xticklabels()]\n _ = [l.set_fontsize(tick_font_size) for l in ax.get_xticklabels()]\n ax.xaxis.set_major_locator(MaxNLocator(max_ticks, prune=\"lower\"))\n ax.xaxis.set_major_formatter(formatter)\n ax.set_xlim(extents.get(p) or self._get_parameter_extents(p, chains))\n max_val = None\n for chain in chains:\n if p in chain.parameters:\n param_summary = summary and p not in blind\n m = self._plot_bars(ax, p, chain, summary=param_summary)\n if max_val is None or m > max_val:\n max_val = m\n\n self._add_truth(ax, truth, None, py=p)\n ax.set_ylim(0, 1.1 * max_val)\n ax.set_xlabel(p, fontsize=label_font_size)\n\n if filename is not None:\n if isinstance(filename, str):\n filename = [filename]\n for f in filename:\n self._save_fig(fig, f, 300)\n if display:\n plt.show()\n return fig", "response": "Plots the 1D parameter distributions for verification purposes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nplotting the parameter summaries for the current system.", "response": "def plot_summary(self, parameters=None, truth=None, extents=None, display=False,\n filename=None, chains=None, figsize=1.0, errorbar=False, include_truth_chain=True,\n blind=None, watermark=None, extra_parameter_spacing=0.5,\n vertical_spacing_ratio=1.0, show_names=True): # pragma: no cover\n \"\"\" Plots parameter summaries\n\n This plot is more for a sanity or consistency check than for use with final results.\n Plotting this before plotting with :func:`plot` allows you to quickly see if the\n chains give well behaved distributions, or if certain parameters are suspect\n or require a greater burn in period.\n\n\n Parameters\n ----------\n parameters : list[str]|int, optional\n Specify a subset of parameters to plot. If not set, all parameters are plotted.\n If an integer is given, only the first so many parameters are plotted.\n truth : list[float]|list|list[float]|dict[str]|str, optional\n A list of truth values corresponding to parameters, or a dictionary of\n truth values keyed by the parameter. Each \"truth value\" can be either a float (will\n draw a vertical line), two floats (a shaded interval) or three floats (min, mean, max),\n which renders as a shaded interval with a line for the mean. Or, supply a string\n which matches a chain name, and the results for that chain will be used as the 'truth'\n extents : list[tuple]|dict[str], optional\n A list of two-tuples for plot extents per parameter, or a dictionary of\n extents keyed by the parameter.\n display : bool, optional\n If set, shows the plot using ``plt.show()``\n filename : str, optional\n If set, saves the figure to the filename\n chains : int|str, list[str|int], optional\n Used to specify which chain to show if more than one chain is loaded in.\n Can be an integer, specifying the\n chain index, or a str, specifying the chain name.\n figsize : float, optional\n Scale horizontal and vertical figure size.\n errorbar : bool, optional\n Whether to onle plot an error bar, instead of the marginalised distribution.\n include_truth_chain : bool, optional\n If you specify another chain as the truth chain, determine if it should still\n be plotted.\n blind : bool|string|list[string], optional\n Whether to blind axes values. Can be set to `True` to blind all parameters,\n or can pass in a string (or list of strings) which specify the parameters to blind.\n watermark : str, optional\n A watermark to add to the figure\n extra_parameter_spacing : float, optional\n Increase horizontal space for parameter values\n vertical_spacing_ratio : float, optional\n Increase vertical space for each model\n show_names : bool, optional\n Whether to show chain names or not. Defaults to `True`.\n\n Returns\n -------\n figure\n the matplotlib figure created\n\n \"\"\"\n wide_extents = not errorbar\n chains, parameters, truth, extents, blind = self._sanitise(chains, parameters, truth, extents, blind=blind, wide_extents=wide_extents)\n\n all_names = [c.name for c in self.parent.chains]\n\n # Check if we're using a chain for truth values\n if isinstance(truth, str):\n assert truth in all_names, \"Truth chain %s is not in the list of added chains %s\" % (truth, all_names)\n if not include_truth_chain:\n chains = [c for c in chains if c.name != truth]\n truth = self.parent.analysis.get_summary(chains=truth, parameters=parameters)\n\n max_param = self._get_size_of_texts(parameters)\n fid_dpi = 65 # Seriously I have no idea what value this should be\n param_width = extra_parameter_spacing + max(0.5, max_param / fid_dpi)\n\n if show_names:\n max_model_name = self._get_size_of_texts([chain.name for chain in chains])\n model_width = 0.25 + (max_model_name / fid_dpi)\n gridspec_kw = {'width_ratios': [model_width] + [param_width] * len(parameters), 'height_ratios': [1] * len(chains)}\n ncols = 1 + len(parameters)\n else:\n model_width = 0\n gridspec_kw = {'width_ratios': [param_width] * len(parameters), 'height_ratios': [1] * len(chains)}\n ncols = len(parameters)\n\n top_spacing = 0.3\n bottom_spacing = 0.2\n row_height = (0.5 if not errorbar else 0.3) * vertical_spacing_ratio\n width = param_width * len(parameters) + model_width\n height = top_spacing + bottom_spacing + row_height * len(chains)\n top_ratio = 1 - (top_spacing / height)\n bottom_ratio = bottom_spacing / height\n\n figsize = (width * figsize, height * figsize)\n fig, axes = plt.subplots(nrows=len(chains), ncols=ncols, figsize=figsize, squeeze=False, gridspec_kw=gridspec_kw)\n fig.subplots_adjust(left=0.05, right=0.95, top=top_ratio, bottom=bottom_ratio, wspace=0.0, hspace=0.0)\n label_font_size = self.parent.config[\"label_font_size\"]\n legend_color_text = self.parent.config[\"legend_color_text\"]\n\n max_vals = {}\n for i, row in enumerate(axes):\n chain = chains[i]\n\n cs, ws, ps, = chain.chain, chain.weights, chain.parameters\n gs, ns = chain.grid, chain.name\n\n colour = chain.config[\"color\"]\n\n # First one put name of model\n if show_names:\n ax_first = row[0]\n ax_first.set_axis_off()\n text_colour = \"k\" if not legend_color_text else colour\n ax_first.text(0, 0.5, ns, transform=ax_first.transAxes, fontsize=label_font_size, verticalalignment=\"center\", color=text_colour, weight=\"medium\")\n cols = row[1:]\n else:\n cols = row\n\n for ax, p in zip(cols, parameters):\n # Set up the frames\n if i > 0:\n ax.spines['top'].set_visible(False)\n if i < (len(chains) - 1):\n ax.spines['bottom'].set_visible(False)\n if i < (len(chains) - 1) or p in blind:\n ax.set_xticks([])\n ax.set_yticks([])\n ax.set_xlim(extents[p])\n\n # Put title in\n if i == 0:\n ax.set_title(r\"$%s$\" % p, fontsize=label_font_size)\n\n # Add truth values\n truth_value = truth.get(p)\n if truth_value is not None:\n if isinstance(truth_value, float) or isinstance(truth_value, int):\n truth_mean = truth_value\n truth_min, truth_max = None, None\n else:\n if len(truth_value) == 1:\n truth_mean = truth_value\n truth_min, truth_max = None, None\n elif len(truth_value) == 2:\n truth_min, truth_max = truth_value\n truth_mean = None\n else:\n truth_min, truth_mean, truth_max = truth_value\n if truth_mean is not None:\n ax.axvline(truth_mean, **self.parent.config_truth)\n if truth_min is not None and truth_max is not None:\n ax.axvspan(truth_min, truth_max, color=self.parent.config_truth[\"color\"], alpha=0.15, lw=0)\n # Skip if this chain doesnt have the parameter\n if p not in ps:\n continue\n\n # Plot the good stuff\n if errorbar:\n fv = self.parent.analysis.get_parameter_summary(chain, p)\n if fv[0] is not None and fv[2] is not None:\n diff = np.abs(np.diff(fv))\n ax.errorbar([fv[1]], 0, xerr=[[diff[0]], [diff[1]]], fmt='o', color=colour)\n else:\n m = self._plot_bars(ax, p, chain)\n if max_vals.get(p) is None or m > max_vals.get(p):\n max_vals[p] = m\n\n for i, row in enumerate(axes):\n index = 1 if show_names else 0\n for ax, p in zip(row[index:], parameters):\n if not errorbar:\n ax.set_ylim(0, 1.1 * max_vals[p])\n\n dpi = 300\n if watermark:\n ax = None\n self._add_watermark(fig, ax, figsize, watermark, dpi=dpi, size_scale=0.8)\n\n if filename is not None:\n if isinstance(filename, str):\n filename = [filename]\n for f in filename:\n self._save_fig(fig, f, dpi)\n if display:\n plt.show()\n\n return fig"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrun the Geweke diagnostic on the supplied chains.", "response": "def geweke(self, chain=None, first=0.1, last=0.5, threshold=0.05):\n \"\"\" Runs the Geweke diagnostic on the supplied chains.\n\n Parameters\n ----------\n chain : int|str, optional\n Which chain to run the diagnostic on. By default, this is `None`,\n which will run the diagnostic on all chains. You can also\n supply and integer (the chain index) or a string, for the chain\n name (if you set one).\n first : float, optional\n The amount of the start of the chain to use\n last : float, optional\n The end amount of the chain to use\n threshold : float, optional\n The p-value to use when testing for normality.\n\n Returns\n -------\n float\n whether or not the chains pass the test\n\n \"\"\"\n if chain is None:\n return np.all([self.geweke(k, threshold=threshold) for k in range(len(self.parent.chains))])\n\n index = self.parent._get_chain(chain)\n assert len(index) == 1, \"Please specify only one chain, have %d chains\" % len(index)\n chain = self.parent.chains[index[0]]\n\n num_walkers = chain.walkers\n assert num_walkers is not None and num_walkers > 0, \\\n \"You need to specify the number of walkers to use the Geweke diagnostic.\"\n name = chain.name\n data = chain.chain\n chains = np.split(data, num_walkers)\n n = 1.0 * chains[0].shape[0]\n n_start = int(np.floor(first * n))\n n_end = int(np.floor((1 - last) * n))\n mean_start = np.array([np.mean(c[:n_start, i])\n for c in chains for i in range(c.shape[1])])\n var_start = np.array([self._spec(c[:n_start, i]) / c[:n_start, i].size\n for c in chains for i in range(c.shape[1])])\n mean_end = np.array([np.mean(c[n_end:, i])\n for c in chains for i in range(c.shape[1])])\n var_end = np.array([self._spec(c[n_end:, i]) / c[n_end:, i].size\n for c in chains for i in range(c.shape[1])])\n zs = (mean_start - mean_end) / (np.sqrt(var_start + var_end))\n _, pvalue = normaltest(zs)\n print(\"Gweke Statistic for chain %s has p-value %e\" % (name, pvalue))\n return pvalue > threshold"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate a LaTeX table from parameter summaries.", "response": "def get_latex_table(self, parameters=None, transpose=False, caption=None,\n label=\"tab:model_params\", hlines=True, blank_fill=\"--\"): # pragma: no cover\n \"\"\" Generates a LaTeX table from parameter summaries.\n\n Parameters\n ----------\n parameters : list[str], optional\n A list of what parameters to include in the table. By default, includes all parameters\n transpose : bool, optional\n Defaults to False, which gives each column as a parameter, each chain (framework)\n as a row. You can swap it so that you have a parameter each row and a framework\n each column by setting this to True\n caption : str, optional\n If you want to generate a caption for the table through Python, use this.\n Defaults to an empty string\n label : str, optional\n If you want to generate a label for the table through Python, use this.\n Defaults to an empty string\n hlines : bool, optional\n Inserts ``\\\\hline`` before and after the header, and at the end of table.\n blank_fill : str, optional\n If a framework does not have a particular parameter, will fill that cell of\n the table with this string.\n\n Returns\n -------\n str\n the LaTeX table.\n \"\"\"\n if parameters is None:\n parameters = self.parent._all_parameters\n for p in parameters:\n assert isinstance(p, str), \\\n \"Generating a LaTeX table requires all parameters have labels\"\n num_parameters = len(parameters)\n num_chains = len(self.parent.chains)\n fit_values = self.get_summary(squeeze=False)\n if label is None:\n label = \"\"\n if caption is None:\n caption = \"\"\n\n end_text = \" \\\\\\\\ \\n\"\n if transpose:\n column_text = \"c\" * (num_chains + 1)\n else:\n column_text = \"c\" * (num_parameters + 1)\n\n center_text = \"\"\n hline_text = \"\\\\hline\\n\"\n if hlines:\n center_text += hline_text + \"\\t\\t\"\n if transpose:\n center_text += \" & \".join([\"Parameter\"] + [c.name for c in self.parent.chains]) + end_text\n if hlines:\n center_text += \"\\t\\t\" + hline_text\n for p in parameters:\n arr = [\"\\t\\t\" + p]\n for chain_res in fit_values:\n if p in chain_res:\n arr.append(self.get_parameter_text(*chain_res[p], wrap=True))\n else:\n arr.append(blank_fill)\n center_text += \" & \".join(arr) + end_text\n else:\n center_text += \" & \".join([\"Model\"] + parameters) + end_text\n if hlines:\n center_text += \"\\t\\t\" + hline_text\n for name, chain_res in zip([c.name for c in self.parent.chains], fit_values):\n arr = [\"\\t\\t\" + name]\n for p in parameters:\n if p in chain_res:\n arr.append(self.get_parameter_text(*chain_res[p], wrap=True))\n else:\n arr.append(blank_fill)\n center_text += \" & \".join(arr) + end_text\n if hlines:\n center_text += \"\\t\\t\" + hline_text\n final_text = get_latex_table_frame(caption, label) % (column_text, center_text)\n\n return final_text"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget a summary of the marginalised parameter distributions.", "response": "def get_summary(self, squeeze=True, parameters=None, chains=None):\n \"\"\" Gets a summary of the marginalised parameter distributions.\n\n Parameters\n ----------\n squeeze : bool, optional\n Squeeze the summaries. If you only have one chain, squeeze will not return\n a length one list, just the single summary. If this is false, you will\n get a length one list.\n parameters : list[str], optional\n A list of parameters which to generate summaries for.\n chains : list[int|str], optional\n A list of the chains to get a summary of.\n\n Returns\n -------\n list of dictionaries\n One entry per chain, parameter bounds stored in dictionary with parameter as key\n \"\"\"\n results = []\n if chains is None:\n chains = self.parent.chains\n else:\n if isinstance(chains, (int, str)):\n chains = [chains]\n chains = [self.parent.chains[i] for c in chains for i in self.parent._get_chain(c)]\n\n for chain in chains:\n res = {}\n params_to_find = parameters if parameters is not None else chain.parameters\n for p in params_to_find:\n if p not in chain.parameters:\n continue\n summary = self.get_parameter_summary(chain, p)\n res[p] = summary\n results.append(res)\n if squeeze and len(results) == 1:\n return results[0]\n return results"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the maximum posterior point in parameter space from the passed parameters.", "response": "def get_max_posteriors(self, parameters=None, squeeze=True, chains=None):\n \"\"\" Gets the maximum posterior point in parameter space from the passed parameters.\n \n Requires the chains to have set `posterior` values.\n\n Parameters\n ----------\n parameters : str|list[str]\n The parameters to find\n squeeze : bool, optional\n Squeeze the summaries. If you only have one chain, squeeze will not return\n a length one list, just the single summary. If this is false, you will\n get a length one list.\n chains : list[int|str], optional\n A list of the chains to get a summary of.\n\n Returns\n -------\n list of two-tuples\n One entry per chain, two-tuple represents the max-likelihood coordinate\n \"\"\"\n\n results = []\n if chains is None:\n chains = self.parent.chains\n else:\n if isinstance(chains, (int, str)):\n chains = [chains]\n chains = [self.parent.chains[i] for c in chains for i in self.parent._get_chain(c)]\n\n if isinstance(parameters, str):\n parameters = [parameters]\n\n for chain in chains:\n if chain.posterior_max_index is None:\n results.append(None)\n continue\n res = {}\n params_to_find = parameters if parameters is not None else chain.parameters\n for p in params_to_find:\n if p in chain.parameters:\n res[p] = chain.posterior_max_params[p]\n results.append(res)\n\n if squeeze and len(results) == 1:\n return results[0]\n return results"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_correlations(self, chain=0, parameters=None):\n parameters, cov = self.get_covariance(chain=chain, parameters=parameters)\n diag = np.sqrt(np.diag(cov))\n divisor = diag[None, :] * diag[:, None]\n correlations = cov / divisor\n return parameters, correlations", "response": "Takes a chain and returns the correlation between chain parameters and all parameters."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntaking a chain and returns the covariance between the parameters and the covariance matrix between the chain parameters.", "response": "def get_covariance(self, chain=0, parameters=None):\n \"\"\"\n Takes a chain and returns the covariance between chain parameters.\n\n Parameters\n ----------\n chain : int|str, optional\n The chain index or name. Defaults to first chain.\n parameters : list[str], optional\n The list of parameters to compute correlations. Defaults to all parameters\n for the given chain.\n\n Returns\n -------\n tuple\n The first index giving a list of parameter names, the second index being the\n 2D covariance matrix.\n \"\"\"\n index = self.parent._get_chain(chain)\n assert len(index) == 1, \"Please specify only one chain, have %d chains\" % len(index)\n chain = self.parent.chains[index[0]]\n if parameters is None:\n parameters = chain.parameters\n\n data = chain.get_data(parameters)\n cov = np.atleast_2d(np.cov(data, aweights=chain.weights, rowvar=False))\n\n return parameters, cov"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_correlation_table(self, chain=0, parameters=None, caption=\"Parameter Correlations\",\n label=\"tab:parameter_correlations\"):\n \"\"\"\n Gets a LaTeX table of parameter correlations.\n\n Parameters\n ----------\n chain : int|str, optional\n The chain index or name. Defaults to first chain.\n parameters : list[str], optional\n The list of parameters to compute correlations. Defaults to all parameters\n for the given chain.\n caption : str, optional\n The LaTeX table caption.\n label : str, optional\n The LaTeX table label.\n\n Returns\n -------\n str\n The LaTeX table ready to go!\n \"\"\"\n parameters, cor = self.get_correlations(chain=chain, parameters=parameters)\n return self._get_2d_latex_table(parameters, cor, caption, label)", "response": "Returns a LaTeX table of parameter correlations."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a LaTeX table of parameter covariance.", "response": "def get_covariance_table(self, chain=0, parameters=None, caption=\"Parameter Covariance\",\n label=\"tab:parameter_covariance\"):\n \"\"\"\n Gets a LaTeX table of parameter covariance.\n\n Parameters\n ----------\n chain : int|str, optional\n The chain index or name. Defaults to first chain.\n parameters : list[str], optional\n The list of parameters to compute correlations. Defaults to all parameters\n for the given chain.\n caption : str, optional\n The LaTeX table caption.\n label : str, optional\n The LaTeX table label.\n\n Returns\n -------\n str\n The LaTeX table ready to go!\n \"\"\"\n parameters, cov = self.get_covariance(chain=chain, parameters=parameters)\n return self._get_2d_latex_table(parameters, cov, caption, label)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating LaTeX appropriate text given the marginalised parameter bounds.", "response": "def get_parameter_text(self, lower, maximum, upper, wrap=False):\n \"\"\" Generates LaTeX appropriate text from marginalised parameter bounds.\n\n Parameters\n ----------\n lower : float\n The lower bound on the parameter\n maximum : float\n The value of the parameter with maximum probability\n upper : float\n The upper bound on the parameter\n wrap : bool\n Wrap output text in dollar signs for LaTeX\n\n Returns\n -------\n str\n The formatted text given the parameter bounds\n \"\"\"\n if lower is None or upper is None:\n return \"\"\n upper_error = upper - maximum\n lower_error = maximum - lower\n if upper_error != 0 and lower_error != 0:\n resolution = min(np.floor(np.log10(np.abs(upper_error))),\n np.floor(np.log10(np.abs(lower_error))))\n elif upper_error == 0 and lower_error != 0:\n resolution = np.floor(np.log10(np.abs(lower_error)))\n elif upper_error != 0 and lower_error == 0:\n resolution = np.floor(np.log10(np.abs(upper_error)))\n else:\n resolution = np.floor(np.log10(np.abs(maximum)))\n factor = 0\n fmt = \"%0.1f\"\n r = 1\n if np.abs(resolution) > 2:\n factor = -resolution\n if resolution == 2:\n fmt = \"%0.0f\"\n factor = -1\n r = 0\n if resolution == 1:\n fmt = \"%0.0f\"\n if resolution == -1:\n fmt = \"%0.2f\"\n r = 2\n elif resolution == -2:\n fmt = \"%0.3f\"\n r = 3\n upper_error *= 10 ** factor\n lower_error *= 10 ** factor\n maximum *= 10 ** factor\n upper_error = round(upper_error, r)\n lower_error = round(lower_error, r)\n maximum = round(maximum, r)\n if maximum == -0.0:\n maximum = 0.0\n if resolution == 2:\n upper_error *= 10 ** -factor\n lower_error *= 10 ** -factor\n maximum *= 10 ** -factor\n factor = 0\n fmt = \"%0.0f\"\n upper_error_text = fmt % upper_error\n lower_error_text = fmt % lower_error\n if upper_error_text == lower_error_text:\n text = r\"%s\\pm %s\" % (fmt, \"%s\") % (maximum, lower_error_text)\n else:\n text = r\"%s^{+%s}_{-%s}\" % (fmt, \"%s\", \"%s\") % \\\n (maximum, upper_error_text, lower_error_text)\n if factor != 0:\n text = r\"\\left( %s \\right) \\times 10^{%d}\" % (text, -factor)\n if wrap:\n text = \"$%s$\" % text\n return text"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_chain(self, chain, parameters=None, name=None, weights=None, posterior=None, walkers=None,\n grid=False, num_eff_data_points=None, num_free_params=None, color=None, linewidth=None,\n linestyle=None, kde=None, shade=None, shade_alpha=None, power=None, marker_style=None, marker_size=None,\n marker_alpha=None, plot_contour=None, plot_point=None, statistics=None, cloud=None,\n shade_gradient=None, bar_shade=None, bins=None, smooth=None, color_params=None,\n plot_color_params=None, cmap=None, num_cloud=None):\n \"\"\"\n Add a chain to the consumer.\n\n Parameters\n ----------\n chain : str|ndarray|dict\n The chain to load. Normally a ``numpy.ndarray``. If a string is found, it\n interprets the string as a filename and attempts to load it in. If a ``dict``\n is passed in, it assumes the dict has keys of parameter names and values of\n an array of samples. Notice that using a dictionary puts the order of\n parameters in the output under the control of the python ``dict.keys()`` function.\n If you passed ``grid`` is set, you can pass in the parameter ranges in list form.\n parameters : list[str], optional\n A list of parameter names, one for each column (dimension) in the chain. This parameter\n should remain ``None`` if a dictionary is given as ``chain``, as the parameter names\n are taken from the dictionary keys.\n name : str, optional\n The name of the chain. Used when plotting multiple chains at once.\n weights : ndarray, optional\n If given, uses this array to weight the samples in chain\n posterior : ndarray, optional\n If given, records the log posterior for each sample in the chain\n walkers : int, optional\n How many walkers went into creating the chain. Each walker should\n contribute the same number of steps, and should appear in contiguous\n blocks in the final chain.\n grid : boolean, optional\n Whether the input is a flattened chain from a grid search instead of a Monte-Carlo\n chains. Note that when this is set, `walkers` should not be set, and `weights` should\n be set to the posterior evaluation for the grid point. **Be careful** when using\n a coarse grid of setting a high smoothing value, as this may oversmooth the posterior\n surface and give unreasonably large parameter bounds.\n num_eff_data_points : int|float, optional\n The number of effective (independent) data points used in the model fitting. Not required\n for plotting, but required if loading in multiple chains to perform model comparison.\n num_free_params : int, optional\n The number of degrees of freedom in your model. Not required for plotting, but required if\n loading in multiple chains to perform model comparison. \n color : str(hex), optional\n Provide a colour for the chain. Can be used instead of calling `configure` for convenience.\n linewidth : float, optional\n Provide a line width to plot the contours. Can be used instead of calling `configure` for convenience.\n linestyle : str, optional\n Provide a line style to plot the contour. Can be used instead of calling `configure` for convenience.\n kde : bool|float, optional\n Set the `kde` value for this specific chain. Can be used instead of calling `configure` for convenience.\n shade : booloptional\n If set, overrides the default behaviour and plots filled contours or not. If a list of\n bools is passed, you can turn shading on or off for specific chains.\n shade_alpha : float, optional\n Filled contour alpha value. Can be used instead of calling `configure` for convenience.\n power : float, optional\n The power to raise the posterior surface to. Useful for inflating or deflating uncertainty for debugging.\n marker_style : str|, optional\n The marker style to use when plotting points. Defaults to `'.'`\n marker_size : numeric|, optional\n Size of markers, if plotted. Defaults to `4`.\n marker_alpha : numeric, optional\n The alpha values when plotting markers.\n plot_contour : bool, optional\n Whether to plot the whole contour (as opposed to a point). Defaults to true for less than\n 25 concurrent chains.\n plot_point : bool, optional\n Whether to plot a maximum likelihood point. Defaults to true for more then 24 chains.\n statistics : string, optional\n Which sort of statistics to use. Defaults to `\"max\"` for maximum likelihood\n statistics. Other available options are `\"mean\"`, `\"cumulative\"`, `\"max_symmetric\"`,\n `\"max_closest\"` and `\"max_central\"`. In the\n very, very rare case you want to enable different statistics for different\n chains, you can pass in a list of strings.\n cloud : bool, optional\n If set, overrides the default behaviour and plots the cloud or not shade_gradient :\n bar_shade : bool, optional\n If set to true, shades in confidence regions in under histogram. By default\n this happens if you less than 3 chains, but is disabled if you are comparing\n more chains. You can pass a list if you wish to shade some chains but not others.\n bins : int|float, optional\n The number of bins to use. By default uses :math:`\\frac{\\sqrt{n}}{10}`, where\n :math:`n` are the number of data points. Giving an integer will set the number\n of bins to the given value. Giving a float will scale the number of bins, such\n that giving ``bins=1.5`` will result in using :math:`\\frac{1.5\\sqrt{n}}{10}` bins.\n Note this parameter is most useful if `kde=False` is also passed, so you\n can actually see the bins and not a KDE. smooth : \n color_params : str, optional\n The name of the parameter to use for the colour scatter. Defaults to none, for no colour. If set\n to 'weights', 'log_weights', or 'posterior' (without the quotes), and that is not a parameter in the chain, \n it will respectively use the weights, log weights, or posterior, to colour the points.\n plot_color_params : bool, optional\n Whether or not the colour parameter should also be plotted as a posterior surface.\n cmaps : str, optional\n The matplotlib colourmap to use in the `colour_param`. If you have multiple `color_param`s, you can\n specific a different cmap for each variable. By default ChainConsumer will cycle between several\n cmaps.\n num_cloud : int, optional\n The number of scatter points to show when enabling `cloud` or setting one of the parameters\n to colour scatter. Defaults to 15k per chain.\n \n Returns\n -------\n ChainConsumer\n Itself, to allow chaining calls.\n \"\"\"\n is_dict = False\n assert chain is not None, \"You cannot have a chain of None\"\n if isinstance(chain, str):\n if chain.endswith(\"txt\"):\n chain = np.loadtxt(chain)\n else:\n chain = np.load(chain)\n elif isinstance(chain, dict):\n assert parameters is None, \\\n \"You cannot pass a dictionary and specify parameter names\"\n is_dict = True\n parameters = list(chain.keys())\n chain = np.array([chain[p] for p in parameters]).T\n elif isinstance(chain, list):\n chain = np.array(chain).T\n\n if grid:\n assert walkers is None, \"If grid is set, walkers should not be\"\n assert weights is not None, \"If grid is set, you need to supply weights\"\n if len(weights.shape) > 1:\n assert not is_dict, \"We cannot construct a meshgrid from a dictionary, as the parameters\" \\\n \"are no longer ordered. Please pass in a flattened array instead.\"\n self._logger.info(\"Constructing meshgrid for grid results\")\n meshes = np.meshgrid(*[u for u in chain.T], indexing=\"ij\")\n chain = np.vstack([m.flatten() for m in meshes]).T\n weights = weights.flatten()\n assert weights.size == chain[:,\n 0].size, \"Error, given weight array size disagrees with parameter sampling\"\n\n if len(chain.shape) == 1:\n chain = chain[None].T\n\n if name is None:\n name = \"Chain %d\" % len(self.chains)\n\n if power is not None:\n assert isinstance(power, int) or isinstance(power, float), \"Power should be numeric, but is %s\" % type(\n power)\n\n if self._default_parameters is None and parameters is not None:\n self._default_parameters = parameters\n\n if parameters is None:\n if self._default_parameters is not None:\n assert chain.shape[1] == len(self._default_parameters), \\\n \"Chain has %d dimensions, but default parameters have %d dimensions\" \\\n % (chain.shape[1], len(self._default_parameters))\n parameters = self._default_parameters\n self._logger.debug(\"Adding chain using default parameters\")\n else:\n self._logger.debug(\"Adding chain with no parameter names\")\n parameters = [\"%d\" % x for x in range(chain.shape[1])]\n else:\n self._logger.debug(\"Adding chain with defined parameters\")\n assert len(parameters) <= chain.shape[1], \\\n \"Have only %d columns in chain, but have been given %d parameters names! \" \\\n \"Please double check this.\" % (chain.shape[1], len(parameters))\n for p in parameters:\n if p not in self._all_parameters:\n self._all_parameters.append(p)\n\n # Sorry, no KDE for you on a grid.\n if grid:\n kde = None\n if color is not None:\n color = self.color_finder.get_formatted([color])[0]\n\n c = Chain(chain, parameters, name, weights=weights, posterior=posterior, walkers=walkers,\n grid=grid, num_free_params=num_free_params, num_eff_data_points=num_eff_data_points,\n color=color, linewidth=linewidth, linestyle=linestyle, kde=kde, shade_alpha=shade_alpha, power=power,\n marker_style=marker_style, marker_size=marker_size, marker_alpha=marker_alpha,\n plot_contour=plot_contour, plot_point=plot_point, statistics=statistics, cloud=cloud,\n shade=shade, shade_gradient=shade_gradient, bar_shade=bar_shade, bins=bins, smooth=smooth,\n color_params=color_params, plot_color_params=plot_color_params, cmap=cmap,\n num_cloud=num_cloud)\n self.chains.append(c)\n self._init_params()\n return self", "response": "This function adds a new chain to the consumer."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nremoves a chain from the ChainConsumer.", "response": "def remove_chain(self, chain=-1):\n \"\"\"\n Removes a chain from ChainConsumer. Calling this will require any configurations set to be redone!\n\n Parameters\n ----------\n chain : int|str, list[str|int]\n The chain(s) to remove. You can pass in either the chain index, or the chain name, to remove it.\n By default removes the last chain added.\n\n Returns\n -------\n ChainConsumer\n Itself, to allow chaining calls.\n \"\"\"\n if isinstance(chain, str) or isinstance(chain, int):\n chain = [chain]\n\n chain = sorted([i for c in chain for i in self._get_chain(c)])[::-1]\n assert len(chain) == len(list(set(chain))), \"Error, you are trying to remove a chain more than once.\"\n\n for index in chain:\n del self.chains[index]\n\n seen = set()\n self._all_parameters = [p for c in self.chains for p in c.parameters if not (p in seen or seen.add(p))]\n\n # Need to reconfigure\n self._init_params()\n\n return self"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef configure(self, statistics=\"max\", max_ticks=5, plot_hists=True, flip=True,\n serif=True, sigma2d=False, sigmas=None, summary=None, bins=None, rainbow=None,\n colors=None, linestyles=None, linewidths=None, kde=False, smooth=None,\n cloud=None, shade=None, shade_alpha=None, shade_gradient=None, bar_shade=None,\n num_cloud=None, color_params=None, plot_color_params=False, cmaps=None,\n plot_contour=None, plot_point=None, global_point=True, marker_style=None, marker_size=None, marker_alpha=None,\n usetex=True, diagonal_tick_labels=True, label_font_size=12, tick_font_size=10,\n spacing=None, contour_labels=None, contour_label_font_size=10,\n legend_kwargs=None, legend_location=None, legend_artists=None,\n legend_color_text=True, watermark_text_kwargs=None, summary_area=0.6827): # pragma: no cover\n r\"\"\" Configure the general plotting parameters common across the bar\n and contour plots.\n\n If you do not call this explicitly, the :func:`plot`\n method will invoke this method automatically.\n\n Please ensure that you call this method *after* adding all the relevant data to the\n chain consumer, as the consume changes configuration values depending on\n the supplied data.\n\n Parameters\n ----------\n statistics : string|list[str], optional\n Which sort of statistics to use. Defaults to `\"max\"` for maximum likelihood\n statistics. Other available options are `\"mean\"`, `\"cumulative\"`, `\"max_symmetric\"`,\n `\"max_closest\"` and `\"max_central\"`. In the\n very, very rare case you want to enable different statistics for different\n chains, you can pass in a list of strings.\n max_ticks : int, optional\n The maximum number of ticks to use on the plots\n plot_hists : bool, optional\n Whether to plot marginalised distributions or not\n flip : bool, optional\n Set to false if, when plotting only two parameters, you do not want it to\n rotate the histogram so that it is horizontal.\n sigma2d: bool, optional\n Defaults to `False`. When `False`, uses :math:`\\sigma` levels for 1D Gaussians - ie confidence\n levels of 68% and 95%. When `True`, uses the confidence levels for 2D Gaussians, where 1 and 2\n :math:`\\sigma` represents 39% and 86% confidence levels respectively.\n sigmas : np.array, optional\n The :math:`\\sigma` contour levels to plot. Defaults to [0, 1, 2, 3] for a single chain\n and [0, 1, 2] for multiple chains.\n serif : bool, optional\n Whether to display ticks and labels with serif font.\n summary : bool, optional\n If overridden, sets whether parameter summaries should be set as axis titles.\n Will not work if you have multiple chains\n bins : int|float,list[int|float], optional\n The number of bins to use. By default uses :math:`\\frac{\\sqrt{n}}{10}`, where\n :math:`n` are the number of data points. Giving an integer will set the number\n of bins to the given value. Giving a float will scale the number of bins, such\n that giving ``bins=1.5`` will result in using :math:`\\frac{1.5\\sqrt{n}}{10}` bins.\n Note this parameter is most useful if `kde=False` is also passed, so you\n can actually see the bins and not a KDE.\n rainbow : bool|list[bool], optional\n Set to True to force use of rainbow colours\n colors : str(hex)|list[str(hex)], optional\n Provide a list of colours to use for each chain. If you provide more chains\n than colours, you *will* get the rainbow colour spectrum. If you only pass\n one colour, all chains are set to this colour. This probably won't look good.\n linestyles : str|list[str], optional\n Provide a list of line styles to plot the contours and marginalised\n distributions with. By default, this will become a list of solid lines. If a\n string is passed instead of a list, this style is used for all chains.\n linewidths : float|list[float], optional\n Provide a list of line widths to plot the contours and marginalised\n distributions with. By default, this is a width of 1. If a float\n is passed instead of a list, this width is used for all chains.\n kde : bool|float|list[bool|float], optional\n Whether to use a Gaussian KDE to smooth marginalised posteriors. If false, uses\n bins and linear interpolation, so ensure you have plenty of samples if your\n distribution is highly non-gaussian. Due to the slowness of performing a\n KDE on all data, it is often useful to disable this before producing final\n plots. If float, scales the width of the KDE bandpass manually.\n smooth : int|list[int], optional\n Defaults to 3. How much to smooth the marginalised distributions using a gaussian filter.\n If ``kde`` is set to true, this parameter is ignored. Setting it to either\n ``0``, ``False`` disables smoothing. For grid data, smoothing\n is set to 0 by default, not 3.\n cloud : bool|list[bool], optional\n If set, overrides the default behaviour and plots the cloud or not\n shade : bool|list[bool] optional\n If set, overrides the default behaviour and plots filled contours or not. If a list of\n bools is passed, you can turn shading on or off for specific chains.\n shade_alpha : float|list[float], optional\n Filled contour alpha value override. Default is 1.0. If a list is passed, you can set the\n shade opacity for specific chains.\n shade_gradient : float|list[float], optional\n How much to vary colours in different contour levels.\n bar_shade : bool|list[bool], optional\n If set to true, shades in confidence regions in under histogram. By default\n this happens if you less than 3 chains, but is disabled if you are comparing\n more chains. You can pass a list if you wish to shade some chains but not others.\n num_cloud : int|list[int], optional\n The number of scatter points to show when enabling `cloud` or setting one of the parameters\n to colour scatter. Defaults to 15k per chain.\n color_params : str|list[str], optional\n The name of the parameter to use for the colour scatter. Defaults to none, for no colour. If set\n to 'weights', 'log_weights', or 'posterior' (without the quotes), and that is not a parameter in the chain, \n it will respectively use the weights, log weights, or posterior, to colour the points.\n plot_color_params : bool|list[bool], optional\n Whether or not the colour parameter should also be plotted as a posterior surface.\n cmaps : str|list[str], optional\n The matplotlib colourmap to use in the `colour_param`. If you have multiple `color_param`s, you can\n specific a different cmap for each variable. By default ChainConsumer will cycle between several\n cmaps.\n plot_contour : bool|list[bool], optional\n Whether to plot the whole contour (as opposed to a point). Defaults to true for less than\n 25 concurrent chains.\n plot_point : bool|list[bool], optional\n Whether to plot a maximum likelihood point. Defaults to true for more then 24 chains.\n global_point : bool, optional\n Whether the point which gets plotted is the global posterior maximum, or the marginalised 2D \n posterior maximum. Note that when you use marginalised 2D maximums for the points, you do not\n get the 1D histograms. Defaults to `True`, for a global maximum value.\n marker_style : str|list[str], optional\n The marker style to use when plotting points. Defaults to `'.'`\n marker_size : numeric|list[numeric], optional\n Size of markers, if plotted. Defaults to `4`.\n marker_alpha : numeric|list[numeric], optional\n The alpha values when plotting markers.\n usetex : bool, optional\n Whether or not to parse text as LaTeX in plots.\n diagonal_tick_labels : bool, optional\n Whether to display tick labels on a 45 degree angle.\n label_font_size : int|float, optional\n The font size for plot axis labels and axis titles if summaries are configured to display.\n tick_font_size : int|float, optional\n The font size for the tick labels in the plots.\n spacing : float, optional\n The amount of spacing to add between plots. Defaults to `None`, which equates to 1.0 for less\n than 6 dimensions and 0.0 for higher dimensions.\n contour_labels : string, optional\n If unset do not plot contour labels. If set to \"confidence\", label the using confidence\n intervals. If set to \"sigma\", labels using sigma.\n contour_label_font_size : int|float, optional\n The font size for contour labels, if they are enabled.\n legend_kwargs : dict, optional\n Extra arguments to pass to the legend api.\n legend_location : tuple(int,int), optional\n Specifies the subplot in which to locate the legend. By default, this will be (0, -1),\n corresponding to the top right subplot if there are more than two parameters,\n and the bottom left plot for only two parameters with flip on.\n For having the legend in the primary subplot\n in the bottom left, set to (-1,0).\n legend_artists : bool, optional\n Whether to include hide artists in the legend. If all linestyles and line widths are identical,\n this will default to false (as only the colours change). Otherwise it will be true.\n legend_color_text : bool, optional\n Whether to colour the legend text.\n watermark_text_kwargs : dict, optional\n Options to pass to the fontdict property when generating text for the watermark.\n summary_area : float, optional\n The confidence interval used when generating parameter summaries. Defaults to 1 sigma, aka 0.6827\n \n Returns\n -------\n ChainConsumer\n Itself, to allow chaining calls.\n \"\"\"\n # Dirty way of ensuring overrides happen when requested\n l = locals()\n explicit = []\n for k in l.keys():\n if l[k] is not None:\n explicit.append(k)\n if k.endswith(\"s\"):\n explicit.append(k[:-1])\n self._init_params()\n\n num_chains = len(self.chains)\n\n assert rainbow is None or colors is None, \\\n \"You cannot both ask for rainbow colours and then give explicit colours\"\n\n # Determine statistics\n assert statistics is not None, \"statistics should be a string or list of strings!\"\n if isinstance(statistics, str):\n assert statistics in list(Analysis.summaries), \"statistics %s not recognised. Should be in %s\" % (statistics, Analysis.summaries)\n statistics = [statistics.lower()] * len(self.chains)\n elif isinstance(statistics, list):\n for i, l in enumerate(statistics):\n statistics[i] = l.lower()\n else:\n raise ValueError(\"statistics is not a string or a list!\")\n\n # Determine KDEs\n if isinstance(kde, bool) or isinstance(kde, float):\n kde = [False if c.grid else kde for c in self.chains]\n\n kde_override = [c.kde for c in self.chains]\n kde = [c2 if c2 is not None else c1 for c1, c2 in zip(kde, kde_override)]\n\n # Determine bins\n if bins is None:\n bins = get_bins(self.chains)\n elif isinstance(bins, list):\n bins = [b2 if isinstance(b2, int) else np.floor(b2 * b1) for b1, b2 in zip(get_bins(self.chains), bins)]\n elif isinstance(bins, float):\n bins = [np.floor(b * bins) for b in get_bins(self.chains)]\n elif isinstance(bins, int):\n bins = [bins] * len(self.chains)\n else:\n raise ValueError(\"bins value is not a recognised class (float or int)\")\n\n # Determine smoothing\n if smooth is None:\n smooth = [0 if c.grid or k else 3 for c, k in zip(self.chains, kde)]\n else:\n if smooth is not None and not smooth:\n smooth = 0\n if isinstance(smooth, list):\n smooth = [0 if k else s for s, k in zip(smooth, kde)]\n else:\n smooth = [0 if k else smooth for k in kde]\n\n # Determine color parameters\n if color_params is None:\n color_params = [None] * num_chains\n else:\n if isinstance(color_params, str):\n color_params = [\n color_params if color_params in cs.parameters + [\"log_weights\", \"weights\", \"posterior\"] else None\n for cs in self.chains]\n color_params = [None if c == \"posterior\" and self.chains[i].posterior is None else c for i, c in\n enumerate(color_params)]\n elif isinstance(color_params, list) or isinstance(color_params, tuple):\n for c, chain in zip(color_params, self.chains):\n p = chain.parameters\n if c is not None:\n assert c in p, \"Color parameter %s not in parameters %s\" % (c, p)\n # Determine if we should plot color parameters\n if isinstance(plot_color_params, bool):\n plot_color_params = [plot_color_params] * len(color_params)\n\n # Determine cmaps\n if cmaps is None:\n param_cmaps = {}\n cmaps = []\n i = 0\n for cp in color_params:\n if cp is None:\n cmaps.append(None)\n elif cp in param_cmaps:\n cmaps.append(param_cmaps[cp])\n else:\n param_cmaps[cp] = self._cmaps[i]\n cmaps.append(self._cmaps[i])\n i = (i + 1) % len(self._cmaps)\n\n # Determine colours\n if colors is None:\n if rainbow:\n colors = self.color_finder.get_colormap(num_chains)\n else:\n if num_chains > len(self._all_colours):\n num_needed_colours = np.sum([c is None for c in color_params])\n colour_list = self.color_finder.get_colormap(num_needed_colours)\n else:\n colour_list = self._all_colours\n colors = []\n ci = 0\n for c in color_params:\n if c:\n colors.append('#000000')\n else:\n colors.append(colour_list[ci])\n ci += 1\n elif isinstance(colors, str):\n colors = [colors] * len(self.chains)\n colors = self.color_finder.get_formatted(colors)\n\n # Determine linestyles\n if linestyles is None:\n i = 0\n linestyles = []\n for c in color_params:\n if c is None:\n linestyles.append(self._linestyles[0])\n else:\n linestyles.append(self._linestyles[i])\n i = (i + 1) % len(self._linestyles)\n elif isinstance(linestyles, str):\n linestyles = [linestyles] * len(self.chains)\n\n # Determine linewidths\n if linewidths is None:\n linewidths = [1.0] * len(self.chains)\n elif isinstance(linewidths, float) or isinstance(linewidths, int):\n linewidths = [linewidths] * len(self.chains)\n\n # Determine clouds\n if cloud is None:\n cloud = False\n cloud = [cloud or c is not None for c in color_params]\n\n # Determine cloud points\n if num_cloud is None:\n num_cloud = 30000\n if isinstance(num_cloud, int) or isinstance(num_cloud, float):\n num_cloud = [int(num_cloud)] * num_chains\n\n # Should we shade the contours\n if shade is None:\n if shade_alpha is None:\n shade = num_chains <= 3\n else:\n shade = True\n if isinstance(shade, bool):\n # If not overridden, do not shade chains with colour scatter points\n shade = [shade and c is None for c in color_params]\n\n # Modify shade alpha based on how many chains we have\n if shade_alpha is None:\n if num_chains == 1:\n if contour_labels is not None:\n shade_alpha = 0.75\n else:\n shade_alpha = 1.0\n else:\n shade_alpha = 1.0 / num_chains\n # Decrease the shading amount if there are colour scatter points\n if isinstance(shade_alpha, float) or isinstance(shade_alpha, int):\n shade_alpha = [shade_alpha if c is None else 0.25 * shade_alpha for c in color_params]\n\n if shade_gradient is None:\n shade_gradient = 1.0\n if isinstance(shade_gradient, float):\n shade_gradient = [shade_gradient] * num_chains\n elif isinstance(shade_gradient, list):\n assert len(shade_gradient) == num_chains, \\\n \"Have %d shade_gradient but % chains\" % (len(shade_gradient), num_chains)\n\n contour_over_points = num_chains < 20\n\n if plot_contour is None:\n plot_contour = [contour_over_points if chain.posterior is not None else True for chain in self.chains]\n elif isinstance(plot_contour, bool):\n plot_contour = [plot_contour] * num_chains\n\n if plot_point is None:\n plot_point = [not contour_over_points] * num_chains\n elif isinstance(plot_point, bool):\n plot_point = [plot_point] * num_chains\n\n if marker_style is None:\n marker_style = ['.'] * num_chains\n elif isinstance(marker_style, str):\n marker_style = [marker_style] * num_chains\n\n if marker_size is None:\n marker_size = [4] * num_chains\n elif isinstance(marker_style, (int, float)):\n marker_size = [marker_size] * num_chains\n\n if marker_alpha is None:\n marker_alpha = [1.0] * num_chains\n elif isinstance(marker_alpha, (int, float)):\n marker_alpha = [marker_alpha] * num_chains\n\n # Figure out if we should display parameter summaries\n if summary is not None:\n summary = summary and num_chains == 1\n\n # Figure out bar shading\n if bar_shade is None:\n bar_shade = num_chains <= 3\n if isinstance(bar_shade, bool):\n bar_shade = [bar_shade] * num_chains\n\n # Figure out how many sigmas to plot\n if sigmas is None:\n if num_chains == 1:\n sigmas = np.array([0, 1, 2])\n else:\n sigmas = np.array([0, 1, 2])\n if sigmas[0] != 0:\n sigmas = np.concatenate(([0], sigmas))\n sigmas = np.sort(sigmas)\n\n if contour_labels is not None:\n assert isinstance(contour_labels, str), \"contour_labels parameter should be a string\"\n contour_labels = contour_labels.lower()\n assert contour_labels in [\"sigma\", \"confidence\"], \"contour_labels should be either sigma or confidence\"\n assert isinstance(contour_label_font_size, int) or isinstance(contour_label_font_size, float), \\\n \"contour_label_font_size needs to be numeric\"\n\n if legend_artists is None:\n legend_artists = len(set(linestyles)) > 1 or len(set(linewidths)) > 1\n\n if legend_kwargs is not None:\n assert isinstance(legend_kwargs, dict), \"legend_kwargs should be a dict\"\n else:\n legend_kwargs = {}\n\n if num_chains < 3:\n labelspacing = 0.5\n elif num_chains == 3:\n labelspacing = 0.2\n else:\n labelspacing = 0.15\n legend_kwargs_default = {\n \"labelspacing\": labelspacing,\n \"loc\": \"upper right\",\n \"frameon\": False,\n \"fontsize\": label_font_size,\n \"handlelength\": 1,\n \"handletextpad\": 0.2,\n \"borderaxespad\": 0.0\n }\n legend_kwargs_default.update(legend_kwargs)\n\n watermark_text_kwargs_default = {\n \"color\": \"#333333\",\n \"alpha\": 0.7,\n \"verticalalignment\": \"center\",\n \"horizontalalignment\": \"center\"\n }\n if watermark_text_kwargs is None:\n watermark_text_kwargs = {}\n watermark_text_kwargs_default.update(watermark_text_kwargs)\n\n assert isinstance(summary_area, float), \"summary_area needs to be a float, not %s!\" % type(summary_area)\n assert summary_area > 0, \"summary_area should be a positive number, instead is %s!\" % summary_area\n assert summary_area < 1, \"summary_area must be less than unity, instead is %s!\" % summary_area\n assert isinstance(global_point, bool), \"global_point should be a bool\"\n\n # List options\n for i, c in enumerate(self.chains):\n try:\n c.update_unset_config(\"statistics\", statistics[i], override=explicit)\n c.update_unset_config(\"color\", colors[i], override=explicit)\n c.update_unset_config(\"linestyle\", linestyles[i], override=explicit)\n c.update_unset_config(\"linewidth\", linewidths[i], override=explicit)\n c.update_unset_config(\"cloud\", cloud[i], override=explicit)\n c.update_unset_config(\"shade\", shade[i], override=explicit)\n c.update_unset_config(\"shade_alpha\", shade_alpha[i], override=explicit)\n c.update_unset_config(\"shade_gradient\", shade_gradient[i], override=explicit)\n c.update_unset_config(\"bar_shade\", bar_shade[i], override=explicit)\n c.update_unset_config(\"bins\", bins[i], override=explicit)\n c.update_unset_config(\"kde\", kde[i], override=explicit)\n c.update_unset_config(\"smooth\", smooth[i], override=explicit)\n c.update_unset_config(\"color_params\", color_params[i], override=explicit)\n c.update_unset_config(\"plot_color_params\", plot_color_params[i], override=explicit)\n c.update_unset_config(\"cmap\", cmaps[i], override=explicit)\n c.update_unset_config(\"num_cloud\", num_cloud[i], override=explicit)\n c.update_unset_config(\"marker_style\", marker_style[i], override=explicit)\n c.update_unset_config(\"marker_size\", marker_size[i], override=explicit)\n c.update_unset_config(\"marker_alpha\", marker_alpha[i], override=explicit)\n c.update_unset_config(\"plot_contour\", plot_contour[i], override=explicit)\n c.update_unset_config(\"plot_point\", plot_point[i], override=explicit)\n c.config[\"summary_area\"] = summary_area\n\n except IndentationError as e:\n print(\"Index error when assigning chain properties, make sure you \"\n \"have enough properties set for the number of chains you have loaded! \"\n \"See the stack trace for which config item has the wrong number of entries.\")\n raise e\n\n # Non list options\n self.config[\"sigma2d\"] = sigma2d\n self.config[\"sigmas\"] = sigmas\n self.config[\"summary\"] = summary\n self.config[\"flip\"] = flip\n self.config[\"serif\"] = serif\n self.config[\"plot_hists\"] = plot_hists\n self.config[\"max_ticks\"] = max_ticks\n self.config[\"usetex\"] = usetex\n self.config[\"diagonal_tick_labels\"] = diagonal_tick_labels\n self.config[\"label_font_size\"] = label_font_size\n self.config[\"tick_font_size\"] = tick_font_size\n self.config[\"spacing\"] = spacing\n self.config[\"contour_labels\"] = contour_labels\n self.config[\"contour_label_font_size\"] = contour_label_font_size\n self.config[\"legend_location\"] = legend_location\n self.config[\"legend_kwargs\"] = legend_kwargs_default\n self.config[\"legend_artists\"] = legend_artists\n self.config[\"legend_color_text\"] = legend_color_text\n self.config[\"watermark_text_kwargs\"] = watermark_text_kwargs_default\n self.config[\"global_point\"] = global_point\n\n self._configured = True\n return self", "response": "r Configure the general plotting parameters common across all barracatic and contour plots."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef configure_truth(self, **kwargs): # pragma: no cover\n if kwargs.get(\"ls\") is None and kwargs.get(\"linestyle\") is None:\n kwargs[\"ls\"] = \"--\"\n kwargs[\"dashes\"] = (3, 3)\n if kwargs.get(\"color\") is None:\n kwargs[\"color\"] = \"#000000\"\n self.config_truth = kwargs\n self._configured_truth = True\n return self", "response": "Configure the arguments passed to the AxvLine and AxhLine methods when plotting truth values."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef divide_chain(self, chain=0):\n indexes = self._get_chain(chain)\n con = ChainConsumer()\n\n for index in indexes:\n chain = self.chains[index]\n assert chain.walkers is not None, \"The chain you have selected was not added with any walkers!\"\n num_walkers = chain.walkers\n data = np.split(chain.chain, num_walkers)\n ws = np.split(chain.weights, num_walkers)\n for j, (c, w) in enumerate(zip(data, ws)):\n con.add_chain(c, weights=w, name=\"Chain %d\" % j, parameters=chain.parameters)\n return con", "response": "Divide a chain into several ChainConsumer instances."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating motif score threshold for a given FPR.", "response": "def threshold(args):\n \"\"\"Calculate motif score threshold for a given FPR.\"\"\"\n if args.fpr < 0 or args.fpr > 1:\n print(\"Please specify a FPR between 0 and 1\")\n sys.exit(1)\n\n motifs = read_motifs(args.pwmfile)\n \n s = Scanner()\n s.set_motifs(args.pwmfile)\n s.set_threshold(args.fpr, filename=args.inputfile)\n\n print(\"Motif\\tScore\\tCutoff\")\n for motif in motifs:\n min_score = motif.pwm_min_score()\n max_score = motif.pwm_max_score()\n opt_score = s.threshold[motif.id]\n if opt_score is None:\n opt_score = motif.pwm_max_score()\n threshold = (opt_score - min_score) / (max_score - min_score)\n print(\"{0}\\t{1}\\t{2}\".format(\n motif.id, opt_score, threshold))"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts two arrays of values to an array of labels and array of scores.", "response": "def values_to_labels(fg_vals, bg_vals):\n \"\"\"\n Convert two arrays of values to an array of labels and an array of scores.\n\n Parameters\n ----------\n fg_vals : array_like\n The list of values for the positive set.\n\n bg_vals : array_like\n The list of values for the negative set.\n\n Returns\n -------\n y_true : array\n Labels.\n y_score : array\n Values.\n \"\"\" \n y_true = np.hstack((np.ones(len(fg_vals)), np.zeros(len(bg_vals))))\n y_score = np.hstack((fg_vals, bg_vals))\n \n return y_true, y_score"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the recall at a specific FDR.", "response": "def recall_at_fdr(fg_vals, bg_vals, fdr_cutoff=0.1):\n \"\"\"\n Computes the recall at a specific FDR (default 10%).\n\n Parameters\n ----------\n fg_vals : array_like\n The list of values for the positive set.\n\n bg_vals : array_like\n The list of values for the negative set.\n \n fdr : float, optional\n The FDR (between 0.0 and 1.0).\n \n Returns\n -------\n recall : float\n The recall at the specified FDR.\n \"\"\"\n if len(fg_vals) == 0:\n return 0.0\n \n y_true, y_score = values_to_labels(fg_vals, bg_vals)\n \n precision, recall, _ = precision_recall_curve(y_true, y_score)\n fdr = 1 - precision\n cutoff_index = next(i for i, x in enumerate(fdr) if x <= fdr_cutoff)\n return recall[cutoff_index]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef matches_at_fpr(fg_vals, bg_vals, fpr=0.01):\n fg_vals = np.array(fg_vals)\n s = scoreatpercentile(bg_vals, 100 - fpr * 100)\n \n return [sum(fg_vals >= s), sum(bg_vals >= s)]", "response": "Computes the hypergeometric p - value at a specific FPR."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef phyper_at_fpr(fg_vals, bg_vals, fpr=0.01):\n fg_vals = np.array(fg_vals)\n s = scoreatpercentile(bg_vals, 100 - fpr * 100)\n \n table = [\n [sum(fg_vals >= s), sum(bg_vals >= s)],\n [sum(fg_vals < s), sum(bg_vals < s)],\n ]\n \n return fisher_exact(table, alternative=\"greater\")[1]", "response": "Computes the hypergeometric p - value at a specific FPR."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncomputes the fraction positives at a specific FPR.", "response": "def fraction_fpr(fg_vals, bg_vals, fpr=0.01):\n \"\"\"\n Computes the fraction positives at a specific FPR (default 1%).\n\n Parameters\n ----------\n fg_vals : array_like\n The list of values for the positive set.\n\n bg_vals : array_like\n The list of values for the negative set.\n \n fpr : float, optional\n The FPR (between 0.0 and 1.0).\n \n Returns\n -------\n fraction : float\n The fraction positives at the specified FPR.\n \"\"\"\n fg_vals = np.array(fg_vals)\n s = scoreatpercentile(bg_vals, 100 - 100 * fpr)\n return len(fg_vals[fg_vals >= s]) / float(len(fg_vals))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the motif score at a specific FPR.", "response": "def score_at_fpr(fg_vals, bg_vals, fpr=0.01):\n \"\"\"\n Returns the motif score at a specific FPR (default 1%).\n\n Parameters\n ----------\n fg_vals : array_like\n The list of values for the positive set.\n\n bg_vals : array_like\n The list of values for the negative set.\n \n fpr : float, optional\n The FPR (between 0.0 and 1.0).\n \n Returns\n -------\n score : float\n The motif score at the specified FPR.\n \"\"\"\n bg_vals = np.array(bg_vals)\n return scoreatpercentile(bg_vals, 100 - 100 * fpr)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef enr_at_fpr(fg_vals, bg_vals, fpr=0.01):\n pos = np.array(fg_vals)\n neg = np.array(bg_vals)\n s = scoreatpercentile(neg, 100 - fpr * 100)\n neg_matches = float(len(neg[neg >= s]))\n if neg_matches == 0:\n return float(\"inf\")\n return len(pos[pos >= s]) / neg_matches * len(neg) / float(len(pos))", "response": "Computes the enrichment at a specific FPR."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef max_enrichment(fg_vals, bg_vals, minbg=2):\n scores = np.hstack((fg_vals, bg_vals))\n idx = np.argsort(scores)\n x = np.hstack((np.ones(len(fg_vals)), np.zeros(len(bg_vals))))\n xsort = x[idx]\n l_fg = len(fg_vals)\n l_bg = len(bg_vals)\n m = 0\n s = 0\n for i in range(len(scores), 0, -1):\n bgcount = float(len(xsort[i:][xsort[i:] == 0])) \n if bgcount >= minbg:\n enr = (len(xsort[i:][xsort[i:] == 1]) / l_fg) / (bgcount / l_bg)\n if enr > m:\n m = enr\n s = scores[idx[i]]\n return m", "response": "Computes the maximum enrichment of a single set of values."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the Mean Normalized Conditional Probability of a set of positive and negative sets.", "response": "def mncp(fg_vals, bg_vals):\n \"\"\"\n Computes the Mean Normalized Conditional Probability (MNCP).\n\n MNCP is described in Clarke & Granek, Bioinformatics, 2003.\n\n Parameters\n ----------\n fg_vals : array_like\n The list of values for the positive set.\n\n bg_vals : array_like\n The list of values for the negative set.\n \n Returns\n -------\n score : float\n MNCP score\n \"\"\"\n fg_len = len(fg_vals)\n total_len = len(fg_vals) + len(bg_vals)\n\n if not isinstance(fg_vals, np.ndarray):\n fg_vals = np.array(fg_vals)\n if not isinstance(bg_vals, np.ndarray):\n bg_vals = np.array(bg_vals)\n \n fg_rank = stats.rankdata(fg_vals)\n total_rank = stats.rankdata(np.hstack((fg_vals, bg_vals)))\n\n slopes = []\n for i in range(len(fg_vals)):\n slope = ((fg_len - fg_rank[i] + 1) / fg_len ) / (\n (total_len - total_rank[i] + 1)/ total_len)\n slopes.append(slope)\n \n return np.mean(slopes)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the Precision - Recall Area Under Curve of a node.", "response": "def pr_auc(fg_vals, bg_vals):\n \"\"\"\n Computes the Precision-Recall Area Under Curve (PR AUC)\n\n Parameters\n ----------\n fg_vals : array_like\n list of values for positive set\n\n bg_vals : array_like\n list of values for negative set\n \n Returns\n -------\n score : float\n PR AUC score\n \"\"\"\n # Create y_labels\n y_true, y_score = values_to_labels(fg_vals, bg_vals)\n \n return average_precision_score(y_true, y_score)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the ROC Area Under Curve ROC AUC score for a single node.", "response": "def roc_auc(fg_vals, bg_vals):\n \"\"\"\n Computes the ROC Area Under Curve (ROC AUC)\n\n Parameters\n ----------\n fg_vals : array_like\n list of values for positive set\n\n bg_vals : array_like\n list of values for negative set\n \n Returns\n -------\n score : float\n ROC AUC score\n \"\"\"\n # Create y_labels\n y_true, y_score = values_to_labels(fg_vals, bg_vals)\n \n return roc_auc_score(y_true, y_score)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing the ROC Area Under Curve until a certain FPR value.", "response": "def roc_auc_xlim(x_bla, y_bla, xlim=0.1):\n \"\"\"\n Computes the ROC Area Under Curve until a certain FPR value.\n\n Parameters\n ----------\n fg_vals : array_like\n list of values for positive set\n\n bg_vals : array_like\n list of values for negative set\n\n xlim : float, optional\n FPR value\n \n Returns\n -------\n score : float\n ROC AUC score\n \"\"\"\n x = x_bla[:]\n y = y_bla[:]\n\n x.sort()\n y.sort()\n\n u = {}\n for i in x + y:\n u[i] = 1\n\n vals = sorted(u.keys())\n \n len_x = float(len(x))\n len_y = float(len(y))\n \n new_x = []\n new_y = []\n \n x_p = 0\n y_p = 0\n for val in vals[::-1]:\n while len(x) > 0 and x[-1] >= val:\n x.pop()\n x_p += 1\n while len(y) > 0 and y[-1] >= val:\n y.pop()\n y_p += 1\n new_y.append((len_x - x_p) / len_x)\n new_x.append((len_y - y_p) / len_y)\n \n #print new_x\n #print new_y\n new_x = 1 - np.array(new_x)\n new_y = 1 - np.array(new_y)\n #plot(new_x, new_y)\n #show()\n\n x = new_x\n y = new_y\n\n if len(x) != len(y):\n raise ValueError(\"Unequal!\")\n\n if not xlim:\n xlim = 1.0\n\n auc = 0.0\n bla = zip(stats.rankdata(x), range(len(x)))\n bla = sorted(bla, key=lambda x: x[1])\n \n prev_x = x[bla[0][1]]\n prev_y = y[bla[0][1]]\n index = 1\n\n while index < len(bla) and x[bla[index][1]] <= xlim:\n\n _, i = bla[index]\n \n auc += y[i] * (x[i] - prev_x) - ((x[i] - prev_x) * (y[i] - prev_y) / 2.0)\n prev_x = x[i]\n prev_y = y[i]\n index += 1\n \n if index < len(bla):\n (rank, i) = bla[index]\n auc += prev_y * (xlim - prev_x) + ((y[i] - prev_y)/(x[i] - prev_x) * (xlim -prev_x) * (xlim - prev_x)/2)\n \n return auc"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef roc_values(fg_vals, bg_vals):\n if len(fg_vals) == 0:\n return 0\n \n y_true, y_score = values_to_labels(fg_vals, bg_vals)\n \n fpr, tpr, _thresholds = roc_curve(y_true, y_score)\n \n return fpr, tpr", "response": "Returns fpr and tpr of the ROC curve."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef max_fmeasure(fg_vals, bg_vals):\n x, y = roc_values(fg_vals, bg_vals)\n x, y = x[1:], y[1:] # don't include origin\n \n p = y / (y + x)\n filt = np.logical_and((p * y) > 0, (p + y) > 0)\n p = p[filt]\n y = y[filt]\n \n f = (2 * p * y) / (p + y)\n if len(f) > 0:\n #return np.nanmax(f), np.nanmax(y[f == np.nanmax(f)])\n return np.nanmax(f)\n else:\n return None", "response": "Computes the maximum F - measure."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the Kolmogorov - Smirnov p - value of position distribution.", "response": "def ks_pvalue(fg_pos, bg_pos=None):\n \"\"\"\n Computes the Kolmogorov-Smirnov p-value of position distribution.\n\n Parameters\n ----------\n fg_pos : array_like\n The list of values for the positive set.\n\n bg_pos : array_like, optional\n The list of values for the negative set.\n \n Returns\n -------\n p : float\n KS p-value.\n \"\"\"\n if len(fg_pos) == 0:\n return 1.0\n a = np.array(fg_pos, dtype=\"float\") / max(fg_pos)\n p = kstest(a, \"uniform\")[1]\n return p"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ks_significance(fg_pos, bg_pos=None):\n p = ks_pvalue(fg_pos, max(fg_pos))\n if p > 0:\n return -np.log10(p)\n else:\n return np.inf", "response": "Computes the log10 of Kolmogorov - Smirnov p - value of position distribution."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nloads and shape data for training and test sets of Keras + Pescador.", "response": "def setup_data():\n \"\"\"Load and shape data for training with Keras + Pescador.\n\n Returns\n -------\n input_shape : tuple, len=3\n Shape of each sample; adapts to channel configuration of Keras.\n\n X_train, y_train : np.ndarrays\n Images and labels for training.\n\n X_test, y_test : np.ndarrays\n Images and labels for test.\n \"\"\"\n # The data, shuffled and split between train and test sets\n (x_train, y_train), (x_test, y_test) = mnist.load_data()\n\n if K.image_data_format() == 'channels_first':\n x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n input_shape = (1, img_rows, img_cols)\n else:\n x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n input_shape = (img_rows, img_cols, 1)\n\n x_train = x_train.astype('float32')\n x_test = x_test.astype('float32')\n x_train /= 255\n x_test /= 255\n print('x_train shape:', x_train.shape)\n print(x_train.shape[0], 'train samples')\n print(x_test.shape[0], 'test samples')\n\n # convert class vectors to binary class matrices\n y_train = keras.utils.to_categorical(y_train, num_classes)\n y_test = keras.utils.to_categorical(y_test, num_classes)\n\n return input_shape, (x_train, y_train), (x_test, y_test)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef build_model(input_shape):\n model = Sequential()\n\n model.add(Conv2D(32, kernel_size=(3, 3),\n activation='relu',\n input_shape=input_shape))\n model.add(Conv2D(64, kernel_size=(3, 3),\n activation='relu'))\n model.add(MaxPooling2D(pool_size=(2, 2)))\n model.add(Dropout(0.25))\n\n model.add(Flatten())\n model.add(Dense(128, activation='relu'))\n model.add(Dropout(0.5))\n model.add(Dense(num_classes, activation='softmax'))\n\n model.compile(loss=keras.losses.categorical_crossentropy,\n optimizer=keras.optimizers.Adadelta(),\n metrics=['accuracy'])\n return model", "response": "Create a compiled Keras model."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef additive_noise(stream, key='X', scale=1e-1):\n '''Add noise to a data stream.\n\n Parameters\n ----------\n stream : iterable\n A stream that yields data objects.\n\n key : string, default='X'\n Name of the field to add noise.\n\n scale : float, default=0.1\n Scale factor for gaussian noise.\n\n Yields\n ------\n data : dict\n Updated data objects in the stream.\n '''\n for data in stream:\n noise_shape = data[key].shape\n noise = scale * np.random.randn(*noise_shape)\n data[key] = data[key] + noise\n yield data", "response": "Add noise to a data stream."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_denovo_params(user_params=None):\n config = MotifConfig()\n\n if user_params is None:\n user_params = {}\n params = config.get_default_params()\n params.update(user_params)\n\n if params.get(\"torque\"):\n logger.debug(\"Using torque\")\n else:\n logger.debug(\"Using multiprocessing\")\n\n params[\"background\"] = [x.strip() for x in params[\"background\"].split(\",\")]\n\n logger.debug(\"Parameters:\")\n for param, value in params.items():\n logger.debug(\" %s: %s\", param, value)\n\n # Maximum time?\n \n if params[\"max_time\"]:\n try:\n max_time = params[\"max_time\"] = float(params[\"max_time\"])\n except Exception:\n logger.debug(\"Could not parse max_time value, setting to no limit\")\n params[\"max_time\"] = -1\n\n if params[\"max_time\"] > 0:\n logger.debug(\"Time limit for motif prediction: %0.2f hours\", max_time)\n params[\"max_time\"] = 3600 * params[\"max_time\"]\n logger.debug(\"Max_time in seconds %0.0f\", max_time)\n else:\n logger.debug(\"No time limit for motif prediction\")\n\n return params", "response": "Return default GimmeMotifs parameters. \n\n Defaults will be replaced with parameters defined in user_params.\n\n Parameters\n ----------\n user_params : dict, optional\n User-defined parameters.\n\n Returns\n -------\n params : dict"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef rankagg_R(df, method=\"stuart\"):\n tmpdf = NamedTemporaryFile()\n tmpscript = NamedTemporaryFile(mode=\"w\") \n tmpranks = NamedTemporaryFile()\n\n df.to_csv(tmpdf.name, sep=\"\\t\",index=False) \n \n script = ''' \nlibrary(RobustRankAggreg); \na = read.table(\"{}\", header=TRUE); \nx = lapply(a, as.vector); \nresult = aggregateRanks(x, method=\"{}\"); \nresult$p.adjust = p.adjust(result$Score); \n write.table(result, file=\"{}\", sep=\"\\t\", quote=FALSE, row.names=FALSE); \n'''.format(tmpdf.name, method, tmpranks.name) \n tmpscript.write(script) \n tmpscript.flush() \n \n p = sp.Popen([\"Rscript\", tmpscript.name], stdout=sp.PIPE, stderr=sp.PIPE)\n stderr, stdout = p.communicate()\n df = pd.read_table(tmpranks.name, index_col=0)\n return df[\"p.adjust\"]", "response": "Return aggregated ranks as implemented in the RobustRankAgg R package."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns aggregated ranks. Implementation is ported from the RobustRankAggreg R package References: Kolde et al., 2012, DOI: 10.1093/bioinformatics/btr709 Stuart et al., 2003, DOI: 10.1126/science.1087447 Parameters ---------- df : pandas.DataFrame DataFrame with values to be ranked and aggregated Returns ------- pandas.DataFrame with aggregated ranks", "response": "def rankagg(df, method=\"stuart\"):\n \"\"\"Return aggregated ranks.\n\n Implementation is ported from the RobustRankAggreg R package\n \n References: \n Kolde et al., 2012, DOI: 10.1093/bioinformatics/btr709\n Stuart et al., 2003, DOI: 10.1126/science.1087447\n\n Parameters\n ----------\n df : pandas.DataFrame\n DataFrame with values to be ranked and aggregated\n\n Returns\n -------\n pandas.DataFrame with aggregated ranks\n \"\"\"\n rmat = pd.DataFrame(index=df.iloc[:,0])\n\n step = 1 / rmat.shape[0]\n for col in df.columns:\n rmat[col] = pd.DataFrame({col:np.arange(step, 1 + step, step)}, index=df[col]).loc[rmat.index]\n rmat = rmat.apply(sorted, 1, result_type=\"expand\")\n p = rmat.apply(qStuart, 1)\n df = pd.DataFrame(\n {\"p.adjust\":multipletests(p, method=\"h\")[1]}, \n index=rmat.index).sort_values('p.adjust') \n \n return df[\"p.adjust\"]"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nyields data while optionally burning compute cycles.", "response": "def data_gen(n_ops=100):\n \"\"\"Yield data, while optionally burning compute cycles.\n\n Parameters\n ----------\n n_ops : int, default=100\n Number of operations to run between yielding data.\n\n Returns\n -------\n data : dict\n A object which looks like it might come from some\n machine learning problem, with X as features, and y as targets.\n \"\"\"\n while True:\n X = np.random.uniform(size=(64, 64))\n yield dict(X=costly_function(X, n_ops),\n y=np.random.randint(10, size=(1,)))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef mp_calc_stats(motifs, fg_fa, bg_fa, bg_name=None):\n try:\n stats = calc_stats(motifs, fg_fa, bg_fa, ncpus=1)\n except Exception as e:\n raise\n sys.stderr.write(\"ERROR: {}\\n\".format(str(e)))\n stats = {}\n\n if not bg_name:\n bg_name = \"default\"\n\n return bg_name, stats", "response": "Parallel calculation of motif statistics."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef pp_predict_motifs(fastafile, outfile, analysis=\"small\", organism=\"hg18\", single=False, background=\"\", tools=None, job_server=None, ncpus=8, max_time=-1, stats_fg=None, stats_bg=None):\n if tools is None:\n tools = {}\n\n config = MotifConfig()\n\n if not tools:\n tools = dict([(x,1) for x in config.get_default_params[\"tools\"].split(\",\")])\n \n #logger = logging.getLogger('gimme.prediction.pp_predict_motifs')\n\n wmin = 5 \n step = 1\n if analysis in [\"large\",\"xl\"]:\n step = 2\n wmin = 6\n \n analysis_max = {\"xs\":5,\"small\":8, \"medium\":10,\"large\":14, \"xl\":20}\n wmax = analysis_max[analysis]\n\n if analysis == \"xs\":\n sys.stderr.write(\"Setting analysis xs to small\")\n analysis = \"small\"\n\n \n if not job_server:\n n_cpus = int(config.get_default_params()[\"ncpus\"])\n job_server = Pool(processes=n_cpus, maxtasksperchild=1000) \n \n jobs = {}\n \n result = PredictionResult(\n outfile, \n fg_file=stats_fg, \n background=stats_bg,\n job_server=job_server,\n )\n \n # Dynamically load all tools\n toolio = [x[1]() for x in inspect.getmembers(\n tool_classes, \n lambda x: \n inspect.isclass(x) and \n issubclass(x, tool_classes.MotifProgram)\n ) if x[0] != 'MotifProgram']\n \n # TODO:\n # Add warnings for running time: Weeder, GADEM\n \n ### Add all jobs to the job_server ###\n params = {\n 'analysis': analysis, \n 'background':background, \n \"single\":single, \n \"organism\":organism\n }\n \n # Tools that don't use a specified width usually take longer\n # ie. GADEM, XXmotif, MEME\n # Start these first.\n for t in [tool for tool in toolio if not tool.use_width]:\n if t.name in tools and tools[t.name]:\n logger.debug(\"Starting %s job\", t.name)\n job_name = t.name\n jobs[job_name] = job_server.apply_async(\n _run_tool,\n (job_name, t, fastafile, params), \n callback=result.add_motifs)\n else:\n logger.debug(\"Skipping %s\", t.name)\n\n for t in [tool for tool in toolio if tool.use_width]:\n if t.name in tools and tools[t.name]:\n for i in range(wmin, wmax + 1, step):\n logger.debug(\"Starting %s job, width %s\", t.name, i)\n job_name = \"%s_width_%s\" % (t.name, i)\n my_params = params.copy()\n my_params['width'] = i\n jobs[job_name] = job_server.apply_async(\n _run_tool,\n (job_name, t, fastafile, my_params), \n callback=result.add_motifs)\n else:\n logger.debug(\"Skipping %s\", t.name)\n \n logger.info(\"all jobs submitted\")\n for job in jobs.values():\n job.get()\n\n result.wait_for_stats()\n ### Wait until all jobs are finished or the time runs out ###\n# start_time = time() \n# try:\n# # Run until all jobs are finished\n# while len(result.finished) < len(jobs.keys()) and (not(max_time) or time() - start_time < max_time):\n# pass\n# if len(result.finished) < len(jobs.keys()):\n# logger.info(\"Maximum allowed running time reached, destroying remaining jobs\")\n# job_server.terminate()\n# result.submit_remaining_stats()\n# ### Or the user gets impatient... ###\n# except KeyboardInterrupt:\n# # Destroy all running jobs\n# logger.info(\"Caught interrupt, destroying all running jobs\")\n# job_server.terminate()\n# result.submit_remaining_stats()\n# \n# \n# if stats_fg and stats_bg:\n# logger.info(\"waiting for motif statistics\")\n# n = 0\n# last_len = 0 \n# \n# \n# while len(set(result.stats.keys())) < len(set([str(m) for m in result.motifs])):\n# if n >= 30:\n# logger.debug(\"waited long enough\")\n# logger.debug(\"motifs: %s, stats: %s\", len(result.motifs), len(result.stats.keys()))\n# for i,motif in enumerate(result.motifs):\n# if \"{}_{}\".format(motif.id, motif.to_consensus()) not in result.stats:\n# logger.debug(\"deleting %s\", motif)\n# del result.motifs[i]\n# break\n# sleep(2)\n# if len(result.stats.keys()) == last_len:\n# n += 1\n# else:\n# last_len = len(result.stats.keys())\n# n = 0\n# \n return result", "response": "Predict motifs using multiprocessing."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict_motifs(infile, bgfile, outfile, params=None, stats_fg=None, stats_bg=None):\n\n # Parse parameters\n required_params = [\"tools\", \"available_tools\", \"analysis\", \n \"genome\", \"use_strand\", \"max_time\"]\n if params is None:\n params = parse_denovo_params()\n else:\n for p in required_params:\n if p not in params:\n params = parse_denovo_params()\n break\n \n # Define all tools\n tools = dict(\n [\n (x.strip(), x in [y.strip() for y in params[\"tools\"].split(\",\")]) \n for x in params[\"available_tools\"].split(\",\")\n ]\n )\n\n # Predict the motifs\n analysis = params[\"analysis\"]\n logger.info(\"starting motif prediction (%s)\", analysis)\n logger.info(\"tools: %s\", \n \", \".join([x for x in tools.keys() if tools[x]]))\n result = pp_predict_motifs(\n infile, \n outfile, \n analysis, \n params.get(\"genome\", None), \n params[\"use_strand\"], \n bgfile, \n tools, \n None, \n #logger=logger, \n max_time=params[\"max_time\"], \n stats_fg=stats_fg, \n stats_bg=stats_bg\n )\n\n motifs = result.motifs\n logger.info(\"predicted %s motifs\", len(motifs))\n logger.debug(\"written to %s\", outfile)\n\n if len(motifs) == 0:\n logger.info(\"no motifs found\")\n result.motifs = []\n \n return result", "response": "Predict motifs for a single file."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_motifs(self, args):\n self.lock.acquire()\n # Callback function for motif programs\n if args is None or len(args) != 2 or len(args[1]) != 3:\n try:\n job = args[0]\n logger.warn(\"job %s failed\", job)\n self.finished.append(job)\n except Exception:\n logger.warn(\"job failed\") \n return\n \n job, (motifs, stdout, stderr) = args\n\n logger.info(\"%s finished, found %s motifs\", job, len(motifs))\n \n for motif in motifs:\n if self.do_counter:\n self.counter += 1 \n motif.id = \"gimme_{}_\".format(self.counter) + motif.id\n f = open(self.outfile, \"a\")\n f.write(\"%s\\n\" % motif.to_pfm())\n f.close()\n self.motifs.append(motif)\n \n if self.do_stats and len(motifs) > 0:\n #job_id = \"%s_%s\" % (motif.id, motif.to_consensus())\n logger.debug(\"Starting stats job of %s motifs\", len(motifs))\n for bg_name, bg_fa in self.background.items():\n job = self.job_server.apply_async(\n mp_calc_stats, \n (motifs, self.fg_fa, bg_fa, bg_name), \n callback=self.add_stats\n )\n self.stat_jobs.append(job)\n \n logger.debug(\"stdout %s: %s\", job, stdout)\n logger.debug(\"stdout %s: %s\", job, stderr)\n self.finished.append(job)\n self.lock.release()", "response": "Add motifs to the result object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwait for all jobs to finish.", "response": "def wait_for_stats(self):\n \"\"\"Make sure all jobs are finished.\"\"\"\n logging.debug(\"waiting for statistics to finish\")\n for job in self.stat_jobs:\n job.get()\n sleep(2)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_stats(self, args):\n bg_name, stats = args\n logger.debug(\"Stats: %s %s\", bg_name, stats)\n \n for motif_id in stats.keys():\n if motif_id not in self.stats:\n self.stats[motif_id] = {}\n \n self.stats[motif_id][bg_name] = stats[motif_id]", "response": "Callback to add motif statistics."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprepares a narrowPeak file for de novo motif prediction.", "response": "def prepare_denovo_input_narrowpeak(inputfile, params, outdir):\n \"\"\"Prepare a narrowPeak file for de novo motif prediction.\n\n All regions to same size; split in test and validation set;\n converted to FASTA.\n\n Parameters\n ----------\n inputfile : str\n BED file with input regions.\n\n params : dict\n Dictionary with parameters.\n\n outdir : str\n Output directory to save files.\n \"\"\"\n\n bedfile = os.path.join(outdir, \"input.from.narrowpeak.bed\")\n p = re.compile(r'^(#|track|browser)')\n width = int(params[\"width\"])\n logger.info(\"preparing input (narrowPeak to BED, width %s)\", width)\n warn_no_summit = True\n with open(bedfile, \"w\") as f_out:\n with open(inputfile) as f_in:\n for line in f_in:\n if p.search(line):\n continue\n vals = line.strip().split(\"\\t\")\n start, end = int(vals[1]), int(vals[2])\n summit = int(vals[9])\n if summit == -1:\n if warn_no_summit:\n logger.warn(\"No summit present in narrowPeak file, using the peak center.\")\n warn_no_summit = False\n summit = (end - start) // 2\n\n start = start + summit - (width // 2)\n end = start + width\n f_out.write(\"{}\\t{}\\t{}\\t{}\\n\".format(\n vals[0],\n start,\n end,\n vals[6]\n ))\n \n prepare_denovo_input_bed(bedfile, params, outdir)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nprepare a BED file for de novo motif prediction.", "response": "def prepare_denovo_input_bed(inputfile, params, outdir):\n \"\"\"Prepare a BED file for de novo motif prediction.\n\n All regions to same size; split in test and validation set;\n converted to FASTA.\n\n Parameters\n ----------\n inputfile : str\n BED file with input regions.\n\n params : dict\n Dictionary with parameters.\n\n outdir : str\n Output directory to save files.\n \"\"\"\n logger.info(\"preparing input (BED)\")\n \n # Create BED file with regions of equal size\n width = int(params[\"width\"])\n bedfile = os.path.join(outdir, \"input.bed\")\n write_equalwidth_bedfile(inputfile, width, bedfile)\n \n abs_max = int(params[\"abs_max\"])\n fraction = float(params[\"fraction\"])\n pred_bedfile = os.path.join(outdir, \"prediction.bed\")\n val_bedfile = os.path.join(outdir, \"validation.bed\")\n # Split input into prediction and validation set\n logger.debug(\n \"Splitting %s into prediction set (%s) and validation set (%s)\",\n bedfile, pred_bedfile, val_bedfile)\n divide_file(bedfile, pred_bedfile, val_bedfile, fraction, abs_max)\n\n config = MotifConfig()\n \n genome = Genome(params[\"genome\"])\n for infile in [pred_bedfile, val_bedfile]:\n genome.track2fasta(\n infile, \n infile.replace(\".bed\", \".fa\"), \n )\n\n # Create file for location plots\n lwidth = int(params[\"lwidth\"])\n extend = (lwidth - width) // 2\n \n genome.track2fasta(\n val_bedfile, \n os.path.join(outdir, \"localization.fa\"), \n extend_up=extend, \n extend_down=extend, \n stranded=params[\"use_strand\"], \n )"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef prepare_denovo_input_fa(inputfile, params, outdir):\n fraction = float(params[\"fraction\"])\n abs_max = int(params[\"abs_max\"])\n\n logger.info(\"preparing input (FASTA)\")\n\n pred_fa = os.path.join(outdir, \"prediction.fa\")\n val_fa = os.path.join(outdir, \"validation.fa\")\n loc_fa = os.path.join(outdir, \"localization.fa\")\n\n # Split inputfile in prediction and validation set\n logger.debug(\n \"Splitting %s into prediction set (%s) and validation set (%s)\",\n inputfile, pred_fa, val_fa)\n\n divide_fa_file(inputfile, pred_fa, val_fa, fraction, abs_max)\n\n # File for location plots\n shutil.copy(val_fa, loc_fa)\n seqs = Fasta(loc_fa).seqs\n lwidth = len(seqs[0])\n all_same_width = not(False in [len(seq) == lwidth for seq in seqs])\n if not all_same_width:\n logger.warn(\n \"PLEASE NOTE: FASTA file contains sequences of different lengths. \"\n \"Positional preference plots might be incorrect!\")", "response": "Prepare the de novo motif prediction and validation files for prediction and validation."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_background(bg_type, fafile, outfile, genome=\"hg18\", width=200, nr_times=10, custom_background=None):\n width = int(width)\n config = MotifConfig()\n fg = Fasta(fafile)\n\n if bg_type in [\"genomic\", \"gc\"]:\n if not genome:\n logger.error(\"Need a genome to create background\")\n sys.exit(1)\n \n if bg_type == \"random\":\n f = MarkovFasta(fg, k=1, n=nr_times * len(fg))\n logger.debug(\"Random background: %s\", outfile)\n elif bg_type == \"genomic\":\n logger.debug(\"Creating genomic background\")\n f = RandomGenomicFasta(genome, width, nr_times * len(fg))\n elif bg_type == \"gc\":\n logger.debug(\"Creating GC matched background\")\n f = MatchedGcFasta(fafile, genome, nr_times * len(fg))\n logger.debug(\"GC matched background: %s\", outfile)\n elif bg_type == \"promoter\":\n fname = Genome(genome).filename\n gene_file = fname.replace(\".fa\", \".annotation.bed.gz\")\n if not gene_file:\n gene_file = os.path.join(config.get_gene_dir(), \"%s.bed\" % genome)\n if not os.path.exists(gene_file):\n print(\"Could not find a gene file for genome {}\")\n print(\"Did you use the --annotation flag for genomepy?\")\n print(\"Alternatively make sure there is a file called {}.bed in {}\".format(genome, config.get_gene_dir()))\n raise ValueError()\n\n logger.info(\n \"Creating random promoter background (%s, using genes in %s)\",\n genome, gene_file)\n f = PromoterFasta(gene_file, genome, width, nr_times * len(fg))\n logger.debug(\"Random promoter background: %s\", outfile)\n elif bg_type == \"custom\":\n bg_file = custom_background\n if not bg_file:\n raise IOError(\n \"Background file not specified!\")\n\n if not os.path.exists(bg_file):\n raise IOError(\n \"Custom background file %s does not exist!\",\n bg_file)\n else:\n logger.info(\"Copying custom background file %s to %s.\",\n bg_file, outfile)\n f = Fasta(bg_file)\n l = np.median([len(seq) for seq in f.seqs])\n if l < (width * 0.95) or l > (width * 1.05):\n logger.warn(\n \"The custom background file %s contains sequences with a \"\n \"median length of %s, while GimmeMotifs predicts motifs in sequences \"\n \"of length %s. This will influence the statistics! It is recommended \"\n \"to use background sequences of the same length.\", \n bg_file, l, width)\n \n f.writefasta(outfile)\n return len(f)", "response": "Create a new background of a specific type."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef create_backgrounds(outdir, background=None, genome=\"hg38\", width=200, custom_background=None):\n if background is None:\n background = [\"random\"]\n nr_sequences = {}\n\n # Create background for motif prediction\n if \"gc\" in background:\n pred_bg = \"gc\"\n else:\n pred_bg = background[0]\n \n create_background(\n pred_bg, \n os.path.join(outdir, \"prediction.fa\"), \n os.path.join(outdir, \"prediction.bg.fa\"), \n genome=genome, \n width=width,\n custom_background=custom_background)\n\n # Get background fasta files for statistics\n bg_info = {}\n nr_sequences = {} \n for bg in background:\n fname = os.path.join(outdir, \"bg.{}.fa\".format(bg))\n nr_sequences[bg] = create_background(\n bg, \n os.path.join(outdir, \"validation.fa\"), \n fname, \n genome=genome, \n width=width,\n custom_background=custom_background)\n\n bg_info[bg] = fname\n return bg_info", "response": "Create different backgrounds for motif prediction and validation."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nfiltering significant motifs based on several statistics.", "response": "def _is_significant(stats, metrics=None):\n \"\"\"Filter significant motifs based on several statistics.\n \n Parameters\n ----------\n stats : dict\n Statistics disctionary object.\n\n metrics : sequence\n Metric with associated minimum values. The default is\n ((\"max_enrichment\", 3), (\"roc_auc\", 0.55), (\"enr_at_fpr\", 0.55))\n \n Returns\n -------\n significant : bool\n \"\"\"\n if metrics is None:\n metrics = ((\"max_enrichment\", 3), (\"roc_auc\", 0.55), (\"enr_at_fpr\", 0.55))\n \n for stat_name, min_value in metrics:\n if stats.get(stat_name, 0) < min_value:\n return False\n\n return True"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef filter_significant_motifs(fname, result, bg, metrics=None):\n sig_motifs = []\n with open(fname, \"w\") as f:\n for motif in result.motifs:\n stats = result.stats.get(\n \"%s_%s\" % (motif.id, motif.to_consensus()), {}).get(bg, {}\n ) \n if _is_significant(stats, metrics):\n f.write(\"%s\\n\" % motif.to_pfm())\n sig_motifs.append(motif)\n \n logger.info(\"%s motifs are significant\", len(sig_motifs))\n logger.debug(\"written to %s\", fname)\n \n return sig_motifs", "response": "Filter significant motifs based on several statistics."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the best motif for a single cluster.", "response": "def best_motif_in_cluster(single_pwm, clus_pwm, clusters, fg_fa, background, stats=None, metrics=(\"roc_auc\", \"recall_at_fdr\")):\n \"\"\"Return the best motif per cluster for a clustering results.\n\n The motif can be either the average motif or one of the clustered motifs.\n\n Parameters\n ----------\n single_pwm : str\n Filename of motifs.\n\n clus_pwm : str\n Filename of motifs.\n\n clusters : \n Motif clustering result.\n\n fg_fa : str\n Filename of FASTA file.\n\n background : dict\n Dictionary for background file names.\n\n stats : dict, optional\n If statistics are not supplied they will be computed.\n\n metrics : sequence, optional\n Metrics to use for motif evaluation. Default are \"roc_auc\" and \n \"recall_at_fdr\".\n \n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n # combine original and clustered motifs\n motifs = read_motifs(single_pwm) + read_motifs(clus_pwm)\n motifs = dict([(str(m), m) for m in motifs])\n\n # get the statistics for those motifs that were not yet checked\n clustered_motifs = []\n for clus,singles in clusters:\n for motif in set([clus] + singles):\n if str(motif) not in stats:\n clustered_motifs.append(motifs[str(motif)])\n \n new_stats = {}\n for bg, bg_fa in background.items():\n for m,s in calc_stats(clustered_motifs, fg_fa, bg_fa).items():\n if m not in new_stats:\n new_stats[m] = {}\n new_stats[m][bg] = s\n stats.update(new_stats)\n \n rank = rank_motifs(stats, metrics)\n\n # rank the motifs\n best_motifs = []\n for clus, singles in clusters:\n if len(singles) > 1:\n eval_motifs = singles\n if clus not in motifs:\n eval_motifs.append(clus)\n eval_motifs = [motifs[str(e)] for e in eval_motifs]\n best_motif = sorted(eval_motifs, key=lambda x: rank[str(x)])[-1]\n best_motifs.append(best_motif)\n else:\n best_motifs.append(clus)\n for bg in background:\n stats[str(best_motifs[-1])][bg][\"num_cluster\"] = len(singles)\n\n best_motifs = sorted(best_motifs, key=lambda x: rank[str(x)], reverse=True)\n return best_motifs"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrenaming motifs to GimmeMotifs_1.. GimmeMotifs_N.", "response": "def rename_motifs(motifs, stats=None):\n \"\"\"Rename motifs to GimmeMotifs_1..GimmeMotifs_N.\n \n If stats object is passed, stats will be copied.\"\"\"\n final_motifs = []\n for i, motif in enumerate(motifs):\n old = str(motif)\n motif.id = \"GimmeMotifs_{}\".format(i + 1)\n final_motifs.append(motif)\n if stats:\n stats[str(motif)] = stats[old].copy()\n \n if stats:\n return final_motifs, stats\n else:\n return final_motifs"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nregisters a class as a decorator for the database.", "response": "def register_db(cls, dbname):\n \"\"\"Register method to keep list of dbs.\"\"\"\n def decorator(subclass):\n \"\"\"Register as decorator function.\"\"\"\n cls._dbs[dbname] = subclass\n subclass.name = dbname\n return subclass\n return decorator"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrunning a single motif activity prediction algorithm.", "response": "def moap(inputfile, method=\"hypergeom\", scoring=None, outfile=None, motiffile=None, pwmfile=None, genome=None, fpr=0.01, ncpus=None,\n subsample=None):\n \"\"\"Run a single motif activity prediction algorithm.\n \n Parameters\n ----------\n inputfile : str\n :1File with regions (chr:start-end) in first column and either cluster \n name in second column or a table with values.\n \n method : str, optional\n Motif activity method to use. Any of 'hypergeom', 'lasso', \n 'lightningclassification', 'lightningregressor', 'bayesianridge', \n 'rf', 'xgboost'. Default is 'hypergeom'. \n \n scoring: str, optional\n Either 'score' or 'count'\n \n outfile : str, optional\n Name of outputfile to save the fitted activity values.\n \n motiffile : str, optional\n Table with motif scan results. First column should be exactly the same\n regions as in the inputfile.\n \n pwmfile : str, optional\n File with motifs in pwm format. Required when motiffile is not \n supplied.\n \n genome : str, optional\n Genome name, as indexed by gimme. Required when motiffile is not\n supplied\n \n fpr : float, optional\n FPR for motif scanning\n \n ncpus : int, optional\n Number of threads to use. Default is the number specified in the config.\n \n Returns\n -------\n pandas DataFrame with motif activity\n \"\"\"\n\n if scoring and scoring not in ['score', 'count']:\n raise ValueError(\"valid values are 'score' and 'count'\")\n \n config = MotifConfig()\n\n if inputfile.endswith(\"feather\"):\n df = pd.read_feather(inputfile)\n df = df.set_index(df.columns[0])\n else:\n # read data\n df = pd.read_table(inputfile, index_col=0, comment=\"#\")\n \n clf = Moap.create(method, ncpus=ncpus)\n\n if clf.ptype == \"classification\":\n if df.shape[1] != 1:\n raise ValueError(\"1 column expected for {}\".format(method))\n else:\n if np.dtype('object') in set(df.dtypes):\n raise ValueError(\n \"columns should all be numeric for {}\".format(method))\n\n if motiffile is None:\n if genome is None:\n raise ValueError(\"need a genome\")\n \n pwmfile = pwmfile_location(pwmfile)\n try:\n motifs = read_motifs(pwmfile)\n except:\n sys.stderr.write(\"can't read motifs from {}\".format(pwmfile))\n raise\n\n # initialize scanner\n s = Scanner(ncpus=ncpus)\n sys.stderr.write(pwmfile + \"\\n\")\n s.set_motifs(pwmfile)\n s.set_genome(genome)\n s.set_background(genome=genome)\n\n # scan for motifs\n sys.stderr.write(\"scanning for motifs\\n\")\n motif_names = [m.id for m in read_motifs(pwmfile)]\n scores = []\n if method == 'classic' or scoring == \"count\":\n s.set_threshold(fpr=fpr)\n for row in s.count(list(df.index)):\n scores.append(row)\n else:\n for row in s.best_score(list(df.index), normalize=True):\n scores.append(row)\n\n motifs = pd.DataFrame(scores, index=df.index, columns=motif_names)\n else:\n motifs = pd.read_table(motiffile, index_col=0, comment=\"#\") \n\n if outfile and os.path.exists(outfile):\n out = pd.read_table(outfile, index_col=0, comment=\"#\")\n ncols = df.shape[1]\n if ncols == 1:\n ncols = len(df.iloc[:,0].unique())\n \n if out.shape[0] == motifs.shape[1] and out.shape[1] == ncols:\n logger.warn(\"%s output already exists... skipping\", method)\n return out\n \n if subsample is not None:\n n = int(subsample * df.shape[0])\n logger.debug(\"Subsampling %d regions\", n)\n df = df.sample(n)\n \n motifs = motifs.loc[df.index]\n \n if method == \"lightningregressor\":\n outdir = os.path.dirname(outfile)\n tmpname = os.path.join(outdir, \".lightning.tmp\")\n clf.fit(motifs, df, tmpdir=tmpname)\n shutil.rmtree(tmpname)\n else:\n clf.fit(motifs, df)\n \n if outfile:\n with open(outfile, \"w\") as f:\n f.write(\"# maelstrom - GimmeMotifs version {}\\n\".format(__version__))\n f.write(\"# method: {} with motif {}\\n\".format(method, scoring))\n if genome:\n f.write(\"# genome: {}\\n\".format(genome))\n if motiffile:\n f.write(\"# motif table: {}\\n\".format(motiffile))\n f.write(\"# {}\\n\".format(clf.act_description))\n \n with open(outfile, \"a\") as f:\n clf.act_.to_csv(f, sep=\"\\t\")\n\n return clf.act_"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a Moap instance based on the predictor name.", "response": "def create(cls, name, ncpus=None):\n \"\"\"Create a Moap instance based on the predictor name.\n\n Parameters\n ----------\n name : str\n Name of the predictor (eg. Xgboost, BayesianRidge, ...)\n \n ncpus : int, optional\n Number of threads. Default is the number specified in the config.\n \n Returns\n -------\n moap : Moap instance\n moap instance.\n \"\"\"\n try:\n return cls._predictors[name.lower()](ncpus=ncpus)\n except KeyError:\n raise Exception(\"Unknown class\")"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef register_predictor(cls, name):\n def decorator(subclass):\n \"\"\"Register as decorator function.\"\"\"\n cls._predictors[name.lower()] = subclass\n subclass.name = name.lower()\n return subclass\n return decorator", "response": "Register a predictor with the given name."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlists available classification predictors.", "response": "def list_classification_predictors(self):\n \"\"\"List available classification predictors.\"\"\"\n preds = [self.create(x) for x in self._predictors.keys()]\n return [x.name for x in preds if x.ptype == \"classification\"]"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef iterate(self, max_iter=None):\n '''Instantiate an iterator.\n\n Parameters\n ----------\n max_iter : None or int > 0\n Maximum number of iterations to yield.\n If ``None``, exhaust the stream.\n\n Yields\n ------\n obj : Objects yielded by the streamer provided on init.\n\n See Also\n --------\n cycle : force an infinite stream.\n\n '''\n # Use self as context manager / calls __enter__() => _activate()\n with self as active_streamer:\n for n, obj in enumerate(active_streamer.stream_):\n if max_iter is not None and n >= max_iter:\n break\n yield obj", "response": "Instantiate an iterator.\n\n Parameters\n ----------\n max_iter : None or int > 0\n Maximum number of iterations to yield.\n If ``None``, exhaust the stream.\n\n Yields\n ------\n obj : Objects yielded by the streamer provided on init.\n\n See Also\n --------\n cycle : force an infinite stream."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cycle(self, max_iter=None):\n '''Iterate from the streamer infinitely.\n\n This function will force an infinite stream, restarting\n the streamer even if a StopIteration is raised.\n\n Parameters\n ----------\n max_iter : None or int > 0\n Maximum number of iterations to yield.\n If `None`, iterate indefinitely.\n\n Yields\n ------\n obj : Objects yielded by the streamer provided on init.\n '''\n\n count = 0\n while True:\n for obj in self.iterate():\n count += 1\n if max_iter is not None and count > max_iter:\n return\n yield obj", "response": "Iterate from the streamer infinitely."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating motif enrichment metrics.", "response": "def calc_stats_iterator(motifs, fg_file, bg_file, genome=None, stats=None, ncpus=None):\n \"\"\"Calculate motif enrichment metrics.\n\n Parameters\n ----------\n motifs : str, list or Motif instance\n A file with motifs in pwm format, a list of Motif instances or a \n single Motif instance.\n\n fg_file : str\n Filename of a FASTA, BED or region file with positive sequences.\n\n bg_file : str\n Filename of a FASTA, BED or region file with negative sequences.\n\n genome : str, optional\n Genome or index directory in case of BED/regions.\n \n stats : list, optional\n Names of metrics to calculate. See gimmemotifs.rocmetrics.__all__ \n for available metrics.\n\n ncpus : int, optional\n Number of cores to use.\n\n Returns\n -------\n result : dict\n Dictionary with results where keys are motif ids and the values are\n dictionary with metric name and value pairs.\n \"\"\"\n if not stats:\n stats = rocmetrics.__all__\n \n if isinstance(motifs, Motif):\n all_motifs = [motifs]\n else:\n if type([]) == type(motifs):\n all_motifs = motifs\n else:\n all_motifs = read_motifs(motifs, fmt=\"pwm\")\n \n if ncpus is None:\n ncpus = int(MotifConfig().get_default_params()[\"ncpus\"])\n chunksize = 240\n\n for i in range(0, len(all_motifs), chunksize):\n result = {}\n logger.debug(\"chunk %s of %s\",\n (i / chunksize) + 1, len(all_motifs) // chunksize + 1)\n motifs = all_motifs[i:i + chunksize]\n \n fg_total = scan_to_best_match(fg_file, motifs, ncpus=ncpus, genome=genome)\n bg_total = scan_to_best_match(bg_file, motifs, ncpus=ncpus, genome=genome)\n \n logger.debug(\"calculating statistics\")\n \n if ncpus == 1:\n it = _single_stats(motifs, stats, fg_total, bg_total) \n else:\n it = _mp_stats(motifs, stats, fg_total, bg_total, ncpus) \n \n for motif_id, s, ret in it:\n if motif_id not in result:\n result[motif_id] = {}\n result[motif_id][s] = ret\n yield result"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calc_stats(motifs, fg_file, bg_file, genome=None, stats=None, ncpus=None):\n result = {}\n for batch_result in calc_stats_iterator(motifs, fg_file, bg_file, genome=genome, stats=stats, ncpus=ncpus):\n for motif_id in batch_result:\n if motif_id not in result:\n result[motif_id] = {}\n for s,ret in batch_result[motif_id].items():\n result[motif_id][s] = ret\n return result", "response": "Calculate motif enrichment metrics."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef rank_motifs(stats, metrics=(\"roc_auc\", \"recall_at_fdr\")):\n rank = {}\n combined_metrics = []\n motif_ids = stats.keys()\n background = list(stats.values())[0].keys()\n for metric in metrics:\n mean_metric_stats = [np.mean(\n [stats[m][bg][metric] for bg in background]) for m in motif_ids]\n ranked_metric_stats = rankdata(mean_metric_stats)\n combined_metrics.append(ranked_metric_stats)\n \n for motif, val in zip(motif_ids, np.mean(combined_metrics, 0)):\n rank[motif] = val\n\n return rank", "response": "Determine mean rank of motifs based on metrics."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwriting motif statistics to text file", "response": "def write_stats(stats, fname, header=None):\n \"\"\"write motif statistics to text file.\"\"\"\n # Write stats output to file\n\n for bg in list(stats.values())[0].keys():\n f = open(fname.format(bg), \"w\")\n if header:\n f.write(header)\n \n stat_keys = sorted(list(list(stats.values())[0].values())[0].keys())\n f.write(\"{}\\t{}\\n\".format(\"Motif\", \"\\t\".join(stat_keys)))\n\n for motif in stats:\n m_stats = stats.get(str(motif), {}).get(bg)\n if m_stats:\n f.write(\"{}\\t{}\\n\".format(\n \"_\".join(motif.split(\"_\")[:-1]),\n \"\\t\".join([str(m_stats[k]) for k in stat_keys])\n ))\n else:\n logger.warn(\"No stats for motif {0}, skipping this motif!\".format(motif.id))\n #motifs.remove(motif)\n f.close()\n\n return"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_roc_values(motif, fg_file, bg_file):\n #print(calc_stats(motif, fg_file, bg_file, stats=[\"roc_values\"], ncpus=1))\n #[\"roc_values\"])\n \n try:\n# fg_result = motif.pwm_scan_score(Fasta(fg_file), cutoff=0.0, nreport=1)\n# fg_vals = [sorted(x)[-1] for x in fg_result.values()]\n#\n# bg_result = motif.pwm_scan_score(Fasta(bg_file), cutoff=0.0, nreport=1)\n# bg_vals = [sorted(x)[-1] for x in bg_result.values()]\n\n# (x, y) = roc_values(fg_vals, bg_vals)\n stats = calc_stats(motif, fg_file, bg_file, stats=[\"roc_values\"], ncpus=1)\n (x,y) = list(stats.values())[0][\"roc_values\"]\n return None,x,y\n except Exception as e:\n print(motif)\n print(motif.id)\n raise\n error = e\n return error,[],[]", "response": "Calculate ROC AUC values for ROC plots."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate ROC plots for all motifs.", "response": "def create_roc_plots(pwmfile, fgfa, background, outdir):\n \"\"\"Make ROC plots for all motifs.\"\"\"\n motifs = read_motifs(pwmfile, fmt=\"pwm\", as_dict=True)\n ncpus = int(MotifConfig().get_default_params()['ncpus'])\n pool = Pool(processes=ncpus)\n jobs = {}\n for bg,fname in background.items():\n for m_id, m in motifs.items():\n\n k = \"{}_{}\".format(str(m), bg)\n jobs[k] = pool.apply_async(\n get_roc_values,\n (motifs[m_id], fgfa, fname,)\n )\n imgdir = os.path.join(outdir, \"images\")\n if not os.path.exists(imgdir):\n os.mkdir(imgdir)\n \n roc_img_file = os.path.join(outdir, \"images\", \"{}_roc.{}.png\")\n\n for motif in motifs.values():\n for bg in background:\n k = \"{}_{}\".format(str(motif), bg)\n error, x, y = jobs[k].get()\n if error:\n logger.error(\"Error in thread: %s\", error)\n logger.error(\"Motif: %s\", motif)\n sys.exit(1)\n roc_plot(roc_img_file.format(motif.id, bg), x, y)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate text report of motifs with statistics and database match.", "response": "def _create_text_report(inputfile, motifs, closest_match, stats, outdir):\n \"\"\"Create text report of motifs with statistics and database match.\"\"\"\n my_stats = {}\n for motif in motifs:\n match = closest_match[motif.id]\n my_stats[str(motif)] = {}\n for bg in list(stats.values())[0].keys():\n if str(motif) not in stats:\n logger.error(\"####\")\n logger.error(\"{} not found\".format(str(motif)))\n for s in sorted(stats.keys()):\n logger.error(s)\n logger.error(\"####\")\n else:\n my_stats[str(motif)][bg] = stats[str(motif)][bg].copy()\n my_stats[str(motif)][bg][\"best_match\"] = \"_\".join(match[0].split(\"_\")[:-1])\n my_stats[str(motif)][bg][\"best_match_pvalue\"] = match[1][-1]\n \n header = (\"# GimmeMotifs version {}\\n\"\n \"# Inputfile: {}\\n\"\n ).format(__version__, inputfile)\n\n write_stats(my_stats, os.path.join(outdir, \"stats.{}.txt\"), header=header)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a graphical report from a gimme_motifs input file pwm and background.", "response": "def _create_graphical_report(inputfile, pwm, background, closest_match, outdir, stats, best_id=None):\n \"\"\"Create main gimme_motifs output html report.\"\"\"\n if best_id is None:\n best_id = {}\n\n logger.debug(\"Creating graphical report\")\n \n class ReportMotif(object):\n \"\"\"Placeholder for motif stats.\"\"\"\n pass\n\n config = MotifConfig()\n \n imgdir = os.path.join(outdir, \"images\")\n if not os.path.exists(imgdir):\n os.mkdir(imgdir)\n \n motifs = read_motifs(pwm, fmt=\"pwm\")\n \n roc_img_file = \"%s_roc.%s\"\n\n dbpwm = config.get_default_params()[\"motif_db\"]\n pwmdir = config.get_motif_dir()\n\n dbmotifs = read_motifs(os.path.join(pwmdir, dbpwm), as_dict=True)\n \n report_motifs = []\n for motif in motifs:\n \n rm = ReportMotif()\n rm.id = motif.id\n rm.id_href = {\"href\": \"#%s\" % motif.id}\n rm.id_name = {\"name\": motif.id}\n rm.img = {\"src\": os.path.join(\"images\", \"%s.png\" % motif.id)}\n motif.to_img(os.path.join(outdir, \"images/{}.png\".format(motif.id)), fmt=\"PNG\")\n \n # TODO: fix best ID\n rm.best = \"Gimme\"#best_id[motif.id]\n\n rm.consensus = motif.to_consensus()\n rm.stars = int(np.mean(\n [stats[str(motif)][bg].get(\"stars\", 0) for bg in background]\n ) + 0.5)\n\n rm.bg = {}\n for bg in background:\n rm.bg[bg] = {}\n this_stats = stats.get(str(motif), {}).get(bg)\n # TODO: fix these stats\n rm.bg[bg][\"e\"] = \"%0.2f\" % this_stats.get(\"enr_at_fpr\", 1.0)\n rm.bg[bg][\"p\"] = \"%0.2f\" % this_stats.get(\"phyper_at_fpr\", 1.0)\n rm.bg[bg][\"auc\"] = \"%0.3f\" % this_stats.get(\"roc_auc\", 0.5)\n rm.bg[bg][\"mncp\"] = \"%0.3f\" % this_stats.get(\"mncp\", 1.0)\n rm.bg[bg][\"roc_img\"] = {\"src\": \"images/\" + os.path.basename(roc_img_file % (motif.id, bg)) + \".png\"}\n rm.bg[bg][u\"roc_img_link\"] = {u\"href\": \"images/\" + os.path.basename(roc_img_file % (motif.id, bg)) + \".png\"}\n\n rm.histogram_img = {\"data\":\"images/%s_histogram.svg\" % motif.id}\n rm.histogram_link= {\"href\":\"images/%s_histogram.svg\" % motif.id}\n \n match_id = closest_match[motif.id][0]\n dbmotifs[match_id].to_img(os.path.join(outdir, \"images/{}.png\".format(match_id)), fmt=\"PNG\")\n \n rm.match_img = {\"src\": \"images/{}.png\".format(match_id)}\n rm.match_id = closest_match[motif.id][0]\n rm.match_pval = \"%0.2e\" % closest_match[motif.id][1][-1]\n\n report_motifs.append(rm)\n \n total_report = os.path.join(outdir, \"motif_report.html\")\n\n star_img = os.path.join(config.get_template_dir(), \"star.png\")\n shutil.copyfile(star_img, os.path.join(outdir, \"images\", \"star.png\"))\n\n env = jinja2.Environment(loader=jinja2.FileSystemLoader([config.get_template_dir()]))\n template = env.get_template(\"report_template.jinja.html\")\n # TODO: title\n result = template.render(\n motifs=report_motifs, \n inputfile=inputfile, \n date=datetime.today().strftime(\"%d/%m/%Y\"), \n version=__version__,\n bg_types=list(background.keys()))\n\n with open(total_report, \"wb\") as f:\n f.write(result.encode('utf-8'))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_denovo_motif_report(inputfile, pwmfile, fgfa, background, locfa, outdir, params, stats=None):\n logger.info(\"creating reports\")\n\n motifs = read_motifs(pwmfile, fmt=\"pwm\")\n \n # ROC plots\n create_roc_plots(pwmfile, fgfa, background, outdir)\n \n # Closest match in database\n mc = MotifComparer()\n closest_match = mc.get_closest_match(motifs)\n \n if stats is None:\n stats = {}\n for bg, bgfa in background.items():\n for m, s in calc_stats(motifs, fgfa, bgfa).items():\n if m not in stats:\n stats[m] = {}\n stats[m][bg] = s\n\n stats = add_star(stats)\n\n if not params:\n params = {}\n cutoff_fpr = params.get('cutoff_fpr', 0.9)\n lwidth = np.median([len(seq) for seq in Fasta(locfa).seqs])\n\n # Location plots\n logger.debug(\"Creating localization plots\")\n for motif in motifs:\n logger.debug(\" {} {}\".format(motif.id, motif))\n outfile = os.path.join(outdir, \"images/{}_histogram.svg\".format(motif.id))\n motif_localization(locfa, motif, lwidth, outfile, cutoff=cutoff_fpr)\n\n # Create reports\n _create_text_report(inputfile, motifs, closest_match, stats, outdir)\n _create_graphical_report(inputfile, pwmfile, background, closest_match, outdir, stats)", "response": "Create a single motif report."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef axes_off(ax):\n ax.set_frame_on(False)\n ax.axes.get_yaxis().set_visible(False)\n ax.axes.get_xaxis().set_visible(False)", "response": "Get rid of all axis ticks, lines, etc."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nplotting list of motifs with database match and p - value", "response": "def match_plot(plotdata, outfile):\n \"\"\"Plot list of motifs with database match and p-value\n \"param plotdata: list of (motif, dbmotif, pval)\n \"\"\"\n fig_h = 2 \n fig_w = 7\n\n nrows = len(plotdata)\n ncols = 2\n fig = plt.figure(figsize=(fig_w, nrows * fig_h))\n \n for i, (motif, dbmotif, pval) in enumerate(plotdata):\n text = \"Motif: %s\\nBest match: %s\\np-value: %0.2e\" % (motif.id, dbmotif.id, pval)\n \n\n grid = ImageGrid(fig, (nrows, ncols, i * 2 + 1), \n nrows_ncols = (2,1),\n axes_pad=0, \n )\n\n for j in range(2): \n axes_off(grid[j])\n\n tmp = NamedTemporaryFile(dir=mytmpdir(), suffix=\".png\")\n motif.to_img(tmp.name, fmt=\"PNG\", height=6)\n grid[0].imshow(plt.imread(tmp.name), interpolation=\"none\")\n tmp = NamedTemporaryFile(dir=mytmpdir(), suffix=\".png\")\n dbmotif.to_img(tmp.name, fmt=\"PNG\")\n grid[1].imshow(plt.imread(tmp.name), interpolation=\"none\")\n\n ax = plt.subplot(nrows, ncols, i * 2 + 2)\n axes_off(ax)\n\n ax.text(0, 0.5, text,\n horizontalalignment='left',\n verticalalignment='center') \n \n plt.savefig(outfile, dpi=300, bbox_inches='tight')\n plt.close(fig)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nplot a phylogenetic tree", "response": "def motif_tree_plot(outfile, tree, data, circle=True, vmin=None, vmax=None, dpi=300):\n \"\"\"\n Plot a \"phylogenetic\" tree \n \"\"\"\n try:\n from ete3 import Tree, faces, AttrFace, TreeStyle, NodeStyle\n except ImportError:\n print(\"Please install ete3 to use this functionality\")\n sys.exit(1)\n\n # Define the tree\n t, ts = _get_motif_tree(tree, data, circle, vmin, vmax)\n \n # Save image\n t.render(outfile, tree_style=ts, w=100, dpi=dpi, units=\"mm\");\n \n # Remove the bottom (empty) half of the figure\n if circle:\n img = Image.open(outfile)\n size = img.size[0]\n spacer = 50\n img.crop((0,0,size,size/2 + spacer)).save(outfile)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef check_bed_file(fname):\n if not os.path.exists(fname):\n logger.error(\"Inputfile %s does not exist!\", fname)\n sys.exit(1)\n\n for i, line in enumerate(open(fname)):\n if line.startswith(\"#\") or line.startswith(\"track\") or line.startswith(\"browser\"):\n # comment or BED specific stuff\n pass\n else:\n vals = line.strip().split(\"\\t\")\n if len(vals) < 3:\n logger.error(\"Expecting tab-seperated values (chromosomestartend) on line %s of file %s\", i + 1, fname)\n sys.exit(1)\n try:\n start, end = int(vals[1]), int(vals[2])\n except ValueError:\n logger.error(\"No valid integer coordinates on line %s of file %s\", i + 1, fname)\n sys.exit(1)\n if len(vals) > 3:\n try:\n float(vals[3])\n except ValueError:\n pass", "response": "Check if the inputfile is a valid BED - file"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef check_denovo_input(inputfile, params):\n background = params[\"background\"]\n \n input_type = determine_file_type(inputfile)\n \n if input_type == \"fasta\":\n valid_bg = FA_VALID_BGS \n elif input_type in [\"bed\", \"narrowpeak\"]:\n genome = params[\"genome\"]\n valid_bg = BED_VALID_BGS \n if \"genomic\" in background or \"gc\" in background:\n Genome(genome)\n # is it a valid bed-file etc.\n check_bed_file(inputfile) # bed-specific, will also work for narrowPeak\n else:\n sys.stderr.write(\"Format of inputfile {} not recognized.\\n\".format(inputfile))\n sys.stderr.write(\"Input should be FASTA, BED or narrowPeak.\\n\")\n sys.stderr.write(\"See https://genome.ucsc.edu/FAQ/FAQformat.html for specifications.\\n\")\n sys.exit(1)\n\n for bg in background:\n if not bg in valid_bg:\n logger.info(\"Input type is %s, ignoring background type '%s'\", \n input_type, bg)\n background = [bg for bg in background if bg in valid_bg]\n\n if len(background) == 0:\n logger.error(\"No valid backgrounds specified!\")\n sys.exit(1)\n\n return input_type, background", "response": "Check if an input file is valid and return the input type and backgrounds."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef scan_to_best_match(fname, motifs, ncpus=None, genome=None, score=False):\n # Initialize scanner\n s = Scanner(ncpus=ncpus)\n s.set_motifs(motifs)\n s.set_threshold(threshold=0.0)\n if genome:\n s.set_genome(genome)\n \n if isinstance(motifs, six.string_types):\n motifs = read_motifs(motifs)\n\n logger.debug(\"scanning %s...\", fname)\n result = dict([(m.id, []) for m in motifs])\n if score:\n it = s.best_score(fname)\n else:\n it = s.best_match(fname)\n for scores in it:\n for motif,score in zip(motifs, scores):\n result[motif.id].append(score)\n\n # Close the pool and reclaim memory\n del s\n \n return result", "response": "Scan a FASTA file with motifs and return a dictionary with the best match per motif."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the background for FPR and z - score calculations.", "response": "def set_background(self, fname=None, genome=None, length=200, nseq=10000):\n \"\"\"Set the background to use for FPR and z-score calculations.\n\n Background can be specified either as a genome name or as the \n name of a FASTA file.\n \n Parameters\n ----------\n fname : str, optional\n Name of FASTA file to use as background.\n\n genome : str, optional\n Name of genome to use to retrieve random sequences.\n\n length : int, optional\n Length of genomic sequences to retrieve. The default\n is 200.\n\n nseq : int, optional\n Number of genomic sequences to retrieve.\n \"\"\"\n length = int(length)\n\n if genome and fname:\n raise ValueError(\"Need either genome or filename for background.\")\n\n if fname:\n if not os.path.exists(fname):\n raise IOError(\"Background file {} does not exist!\".format(fname))\n\n self.background = Fasta(fname)\n self.background_hash = file_checksum(fname)\n return\n \n if not genome:\n if self.genome:\n genome = self.genome\n logger.info(\"Using default background: genome {} with length {}\".format(\n genome, length))\n else:\n raise ValueError(\"Need either genome or filename for background.\")\n \n \n logger.info(\"Using background: genome {} with length {}\".format(genome, length))\n with Cache(CACHE_DIR) as cache:\n self.background_hash = \"{}\\{}\".format(genome, int(length))\n fa = cache.get(self.background_hash)\n if not fa:\n fa = RandomGenomicFasta(genome, length, nseq)\n cache.set(self.background_hash, fa)\n self.background = fa"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting motif scanning threshold based on background sequences.", "response": "def set_threshold(self, fpr=None, threshold=None):\n \"\"\"Set motif scanning threshold based on background sequences.\n\n Parameters\n ----------\n fpr : float, optional\n Desired FPR, between 0.0 and 1.0.\n\n threshold : float or str, optional\n Desired motif threshold, expressed as the fraction of the \n difference between minimum and maximum score of the PWM.\n Should either be a float between 0.0 and 1.0 or a filename\n with thresholds as created by 'gimme threshold'.\n\n \"\"\"\n if threshold and fpr:\n raise ValueError(\"Need either fpr or threshold.\")\n \n if fpr:\n fpr = float(fpr)\n if not (0.0 < fpr < 1.0):\n raise ValueError(\"Parameter fpr should be between 0 and 1\")\n \n if not self.motifs:\n raise ValueError(\"please run set_motifs() first\")\n\n thresholds = {}\n motifs = read_motifs(self.motifs)\n \n if threshold is not None:\n self.threshold = parse_threshold_values(self.motifs, threshold) \n return\n \n if not self.background:\n try:\n self.set_background()\n except:\n raise ValueError(\"please run set_background() first\")\n \n seqs = self.background.seqs\n\n with Cache(CACHE_DIR) as cache:\n scan_motifs = []\n for motif in motifs:\n k = \"{}|{}|{:.4f}\".format(motif.hash(), self.background_hash, fpr)\n \n threshold = cache.get(k)\n if threshold is None:\n scan_motifs.append(motif)\n else:\n if np.isclose(threshold, motif.pwm_max_score()):\n thresholds[motif.id] = None \n elif np.isclose(threshold, motif.pwm_min_score()):\n thresholds[motif.id] = 0.0\n else:\n thresholds[motif.id] = threshold\n \n if len(scan_motifs) > 0:\n logger.info(\"Determining FPR-based threshold\")\n for motif, threshold in self._threshold_from_seqs(scan_motifs, seqs, fpr):\n k = \"{}|{}|{:.4f}\".format(motif.hash(), self.background_hash, fpr)\n cache.set(k, threshold)\n if np.isclose(threshold, motif.pwm_max_score()):\n thresholds[motif.id] = None\n elif np.isclose(threshold, motif.pwm_min_score()):\n thresholds[motif.id] = 0.0\n else:\n thresholds[motif.id] = threshold\n self.threshold_str = \"{}_{}_{}\".format(fpr, threshold, self.background_hash)\n self.threshold = thresholds"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncount the number of matches above the cutoff returns an iterator of lists containing integer counts", "response": "def count(self, seqs, nreport=100, scan_rc=True):\n \"\"\"\n count the number of matches above the cutoff\n returns an iterator of lists containing integer counts\n \"\"\"\n for matches in self.scan(seqs, nreport, scan_rc):\n counts = [len(m) for m in matches]\n yield counts"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncounts the number of matches above the cutoff returns an iterator of lists containing integer counts", "response": "def total_count(self, seqs, nreport=100, scan_rc=True):\n \"\"\"\n count the number of matches above the cutoff\n returns an iterator of lists containing integer counts\n \"\"\"\n \n count_table = [counts for counts in self.count(seqs, nreport, scan_rc)]\n return np.sum(np.array(count_table), 0)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning an iterator of lists containing the scores of the best match of each motif in each sequence", "response": "def best_score(self, seqs, scan_rc=True, normalize=False):\n \"\"\"\n give the score of the best match of each motif in each sequence\n returns an iterator of lists containing floats\n \"\"\"\n self.set_threshold(threshold=0.0)\n if normalize and len(self.meanstd) == 0:\n self.set_meanstd()\n means = np.array([self.meanstd[m][0] for m in self.motif_ids])\n stds = np.array([self.meanstd[m][1] for m in self.motif_ids])\n\n for matches in self.scan(seqs, 1, scan_rc):\n scores = np.array([sorted(m, key=lambda x: x[0])[0][0] for m in matches if len(m) > 0])\n if normalize:\n scores = (scores - means) / stds\n yield scores"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef best_match(self, seqs, scan_rc=True):\n self.set_threshold(threshold=0.0)\n for matches in self.scan(seqs, 1, scan_rc):\n yield [m[0] for m in matches]", "response": "gives the best match of each motif in each sequence"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef scan(self, seqs, nreport=100, scan_rc=True, normalize=False):\n\n if not self.threshold:\n sys.stderr.write(\n \"Using default threshold of 0.95. \"\n \"This is likely not optimal!\\n\"\n )\n self.set_threshold(threshold=0.95)\n\n seqs = as_fasta(seqs, genome=self.genome)\n \n it = self._scan_sequences(seqs.seqs, \n nreport, scan_rc)\n \n if normalize:\n if len(self.meanstd) == 0:\n self.set_meanstd()\n mean_std = [self.meanstd.get(m_id) for m_id in self.motif_ids]\n means = [x[0] for x in mean_std]\n stds = [x[1] for x in mean_std]\n\n\n for result in it:\n if normalize:\n zresult = [] \n for i,mrow in enumerate(result):\n mrow = [((x[0] - means[i]) / stds[i], x[1], x[2]) for x in mrow]\n zresult.append(mrow)\n yield zresult\n else:\n yield result", "response": "Scan a set of regions and sequences and yield motifs."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef roc(args):\n outputfile = args.outfile\n # Default extension for image\n if outputfile and not outputfile.endswith(\".png\"):\n outputfile += \".png\"\n \n motifs = read_motifs(args.pwmfile, fmt=\"pwm\")\n\n ids = []\n if args.ids:\n ids = args.ids.split(\",\")\n else:\n ids = [m.id for m in motifs]\n motifs = [m for m in motifs if (m.id in ids)]\n \n stats = [\n \"phyper_at_fpr\",\n \"roc_auc\", \n \"pr_auc\", \n \"enr_at_fpr\",\n \"recall_at_fdr\", \n \"roc_values\",\n \"matches_at_fpr\",\n ]\n \n plot_x = []\n plot_y = []\n legend = []\n \n f_out = sys.stdout\n if args.outdir:\n if not os.path.exists(args.outdir):\n os.makedirs(args.outdir)\n f_out = open(args.outdir + \"/gimme.roc.report.txt\", \"w\")\n \n # Print the metrics\n f_out.write(\"Motif\\t# matches\\t# matches background\\tP-value\\tlog10 P-value\\tROC AUC\\tPR AUC\\tEnr. at 1% FPR\\tRecall at 10% FDR\\n\")\n \n \n for motif_stats in calc_stats_iterator(motifs, args.sample, args.background, \n genome=args.genome, stats=stats, ncpus=args.ncpus):\n \n for motif in motifs:\n if str(motif) in motif_stats:\n if outputfile:\n x, y = motif_stats[str(motif)][\"roc_values\"]\n plot_x.append(x)\n plot_y.append(y)\n legend.append(motif.id)\n log_pvalue = np.inf\n if motif_stats[str(motif)][\"phyper_at_fpr\"] > 0:\n log_pvalue = -np.log10(motif_stats[str(motif)][\"phyper_at_fpr\"])\n f_out.write(\"{}\\t{:d}\\t{:d}\\t{:.2e}\\t{:.3f}\\t{:.3f}\\t{:.3f}\\t{:.2f}\\t{:0.4f}\\n\".format(\n motif.id, \n motif_stats[str(motif)][\"matches_at_fpr\"][0], \n motif_stats[str(motif)][\"matches_at_fpr\"][1], \n motif_stats[str(motif)][\"phyper_at_fpr\"], \n log_pvalue, \n motif_stats[str(motif)][\"roc_auc\"], \n motif_stats[str(motif)][\"pr_auc\"], \n motif_stats[str(motif)][\"enr_at_fpr\"], \n motif_stats[str(motif)][\"recall_at_fdr\"],\n ))\n f_out.close() \n \n if args.outdir:\n html_report(\n args.outdir,\n args.outdir + \"/gimme.roc.report.txt\",\n args.pwmfile,\n 0.01,\n )\n\n # Plot the ROC curve\n if outputfile:\n roc_plot(outputfile, plot_x, plot_y, ids=legend)", "response": "Calculate ROC_AUC and other metrics and optionally plot ROC curve."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ssd(p1, p2):\n return 2 - np.sum([(a-b)**2 for a,b in zip(p1,p2)])", "response": "Calculates motif position similarity based on sum of squared distances."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef seqcor(m1, m2, seq=None):\n l1 = len(m1)\n l2 = len(m2)\n\n l = max(l1, l2)\n\n if seq is None:\n seq = RCDB \n \n L = len(seq)\n\n # Scan RC de Bruijn sequence\n result1 = pfmscan(seq, m1.pwm, m1.pwm_min_score(), len(seq), False, True)\n result2 = pfmscan(seq, m2.pwm, m2.pwm_min_score(), len(seq), False, True)\n \n # Reverse complement of motif 2\n result3 = pfmscan(seq, m2.rc().pwm, m2.rc().pwm_min_score(), len(seq), False, True)\n \n result1 = np.array(result1)\n result2 = np.array(result2)\n result3 = np.array(result3)\n\n # Return maximum correlation\n c = []\n for i in range(l1 - l1 // 3):\n c.append([1 - distance.correlation(result1[:L-l-i],result2[i:L-l]), i, 1])\n c.append([1 - distance.correlation(result1[:L-l-i],result3[i:L-l]), i, -1])\n for i in range(l2 - l2 // 3):\n c.append([1 - distance.correlation(result1[i:L-l],result2[:L-l-i]), -i, 1])\n c.append([1 - distance.correlation(result1[i:L-l],result3[:L-l-i]), -i, -1])\n \n return sorted(c, key=lambda x: x[0])[-1]", "response": "Calculates similarity based on Pearson correlation of scores of two Motif objects."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef compare_motifs(self, m1, m2, match=\"total\", metric=\"wic\", combine=\"mean\", pval=False):\n if metric == \"seqcor\":\n return seqcor(m1, m2)\n elif match == \"partial\":\n if pval:\n return self.pvalue(m1, m2, \"total\", metric, combine, self.max_partial(m1.pwm, m2.pwm, metric, combine))\n elif metric in [\"pcc\", \"ed\", \"distance\", \"wic\", \"chisq\", \"ssd\"]:\n return self.max_partial(m1.pwm, m2.pwm, metric, combine)\n else:\n return self.max_partial(m1.pfm, m2.pfm, metric, combine)\n\n elif match == \"total\":\n if pval:\n return self.pvalue(m1, m2, match, metric, combine, self.max_total(m1.pwm, m2.pwm, metric, combine))\n elif metric in [\"pcc\", 'akl']:\n # Slightly randomize the weight matrix\n return self.max_total(m1.wiggle_pwm(), m2.wiggle_pwm(), metric, combine)\n elif metric in [\"ed\", \"distance\", \"wic\", \"chisq\", \"pcc\", \"ssd\"]:\n return self.max_total(m1.pwm, m2.pwm, metric, combine)\n else:\n return self.max_total(m1.pfm, m2.pfm, metric, combine)\n \n elif match == \"subtotal\":\n if metric in [\"pcc\", \"ed\", \"distance\", \"wic\", \"chisq\", \"ssd\"]:\n return self.max_subtotal(m1.pwm, m2.pwm, metric, combine)\n else:\n return self.max_subtotal(m1.pfm, m2.pfm, metric, combine)", "response": "Compare two motifs.\n \n The similarity metric can be any of seqcor, pcc, ed, distance, wic, \n chisq, akl or ssd. If match is 'total' the similarity score is \n calculated for the whole match, including positions that are not \n present in both motifs. If match is partial or subtotal, only the\n matching psotiions are used to calculate the score. The score of\n individual position is combined using either the mean or the sum.\n\n Note that the match and combine parameters have no effect on the seqcor\n similarity metric. \n\n Parameters\n ----------\n m1 : Motif instance\n Motif instance 1.\n\n m2 : Motif instance\n Motif instance 2.\n\n match : str, optional\n Match can be \"partial\", \"subtotal\" or \"total\". Not all metrics use \n this.\n\n metric : str, optional\n Distance metric.\n\n combine : str, optional\n Combine positional scores using \"mean\" or \"sum\". Not all metrics\n use this.\n\n pval : bool, optional\n Calculate p-vale of match.\n \n Returns\n -------\n score, position, strand"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_all_scores(self, motifs, dbmotifs, match, metric, combine, \n pval=False, parallel=True, trim=None, ncpus=None):\n \"\"\"Pairwise comparison of a set of motifs compared to reference motifs.\n\n Parameters\n ----------\n motifs : list\n List of Motif instances.\n\n dbmotifs : list\n List of Motif instances.\n\n match : str\n Match can be \"partial\", \"subtotal\" or \"total\". Not all metrics use \n this.\n\n metric : str\n Distance metric.\n\n combine : str\n Combine positional scores using \"mean\" or \"sum\". Not all metrics\n use this.\n\n pval : bool , optional\n Calculate p-vale of match.\n \n parallel : bool , optional\n Use multiprocessing for parallel execution. True by default.\n\n trim : float or None\n If a float value is specified, motifs are trimmed used this IC \n cutoff before comparison.\n\n ncpus : int or None\n Specifies the number of cores to use for parallel execution.\n\n Returns\n -------\n scores : dict\n Dictionary with scores.\n \"\"\"\n # trim motifs first, if specified\n if trim:\n for m in motifs:\n m.trim(trim)\n for m in dbmotifs:\n m.trim(trim)\n \n # hash of result scores\n scores = {}\n \n if parallel: \n # Divide the job into big chunks, to keep parallel overhead to minimum\n # Number of chunks = number of processors available\n if ncpus is None:\n ncpus = int(MotifConfig().get_default_params()[\"ncpus\"])\n\n pool = Pool(processes=ncpus, maxtasksperchild=1000)\n \n batch_len = len(dbmotifs) // ncpus\n if batch_len <= 0:\n batch_len = 1\n jobs = []\n for i in range(0, len(dbmotifs), batch_len): \n # submit jobs to the job server\n \n p = pool.apply_async(_get_all_scores, \n args=(self, motifs, dbmotifs[i: i + batch_len], match, metric, combine, pval))\n jobs.append(p)\n \n pool.close()\n for job in jobs:\n # Get the job result\n result = job.get()\n # and update the result score\n for m1,v in result.items():\n for m2, s in v.items():\n if m1 not in scores:\n scores[m1] = {}\n scores[m1][m2] = s\n \n pool.join()\n else:\n # Do the whole thing at once if we don't want parallel\n scores = _get_all_scores(self, motifs, dbmotifs, match, metric, combine, pval)\n \n return scores", "response": "Pairwise comparison of a set of motifs compared to reference motifs."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_closest_match(self, motifs, dbmotifs=None, match=\"partial\", metric=\"wic\",combine=\"mean\", parallel=True, ncpus=None):\n\n if dbmotifs is None:\n pwm = self.config.get_default_params()[\"motif_db\"]\n pwmdir = self.config.get_motif_dir()\n dbmotifs = os.path.join(pwmdir, pwm)\n \n motifs = parse_motifs(motifs)\n dbmotifs = parse_motifs(dbmotifs)\n\n dbmotif_lookup = dict([(m.id, m) for m in dbmotifs])\n\n scores = self.get_all_scores(motifs, dbmotifs, match, metric, combine, parallel=parallel, ncpus=ncpus)\n for motif in scores:\n scores[motif] = sorted(\n scores[motif].items(), \n key=lambda x:x[1][0]\n )[-1]\n \n for motif in motifs:\n dbmotif, score = scores[motif.id]\n pval, pos, orient = self.compare_motifs(\n motif, dbmotif_lookup[dbmotif], match, metric, combine, True)\n \n scores[motif.id] = [dbmotif, (list(score) + [pval])]\n \n return scores", "response": "Returns a dict of motifs with the closest match."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef list_regions(service):\n for region in service.regions():\n print '%(name)s: %(endpoint)s' % {\n 'name': region.name,\n 'endpoint': region.endpoint,\n }", "response": "List the regions for the service"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef elb_table(balancers):\n t = prettytable.PrettyTable(['Name', 'DNS', 'Ports', 'Zones', 'Created'])\n t.align = 'l'\n for b in balancers:\n ports = ['%s: %s -> %s' % (l[2], l[0], l[1]) for l in b.listeners]\n ports = '\\n'.join(ports)\n zones = '\\n'.join(b.availability_zones)\n t.add_row([b.name, b.dns_name, ports, zones, b.created_time])\n return t", "response": "Print nice looking table of information from list of load balancers"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nprinting nice looking table of information from list of instances", "response": "def ec2_table(instances):\n \"\"\"\n Print nice looking table of information from list of instances\n \"\"\"\n t = prettytable.PrettyTable(['ID', 'State', 'Monitored', 'Image', 'Name', 'Type', 'SSH key', 'DNS'])\n t.align = 'l'\n for i in instances:\n name = i.tags.get('Name', '')\n t.add_row([i.id, i.state, i.monitored, i.image_id, name, i.instance_type, i.key_name, i.dns_name])\n return t"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprinting nice looking table of information from images", "response": "def ec2_image_table(images):\n \"\"\"\n Print nice looking table of information from images\n \"\"\"\n t = prettytable.PrettyTable(['ID', 'State', 'Name', 'Owner', 'Root device', 'Is public', 'Description'])\n t.align = 'l'\n for i in images:\n t.add_row([i.id, i.state, i.name, i.ownerId, i.root_device_type, i.is_public, i.description])\n return t"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ec2_fab(service, args):\n instance_ids = args.instances\n instances = service.list(elb=args.elb, instance_ids=instance_ids)\n hosts = service.resolve_hosts(instances)\n\n fab.env.hosts = hosts\n fab.env.key_filename = settings.get('SSH', 'KEY_FILE')\n fab.env.user = settings.get('SSH', 'USER', getpass.getuser())\n fab.env.parallel = True\n\n fabfile = find_fabfile(args.file)\n if not fabfile:\n print 'Couldn\\'t find any fabfiles!'\n return\n\n fab.env.real_fabile = fabfile\n docstring, callables, default = load_fabfile(fabfile)\n fab_state.commands.update(callables)\n\n commands_to_run = parse_arguments(args.methods)\n for name, args, kwargs, arg_hosts, arg_roles, arg_exclude_hosts in commands_to_run:\n fab.execute(name,\n hosts=arg_hosts,\n roles=arg_roles,\n exclude_hosts=arg_exclude_hosts,\n *args, **kwargs)", "response": "Run Fabric commands against EC2 instances"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nyields data from an input stream into one batch of size buffer_size.", "response": "def buffer_stream(stream, buffer_size, partial=False, axis=None):\n '''Buffer \"data\" from an stream into one data object.\n\n Parameters\n ----------\n stream : stream\n The stream to buffer\n\n buffer_size : int > 0\n The number of examples to retain per batch.\n\n partial : bool, default=False\n If True, yield a final partial batch on under-run.\n\n axis : int or None\n If `None` (default), concatenate data along a new 0th axis.\n Otherwise, concatenation is performed along the specified axis.\n\n This is primarily useful when combining data that already has a\n dimension for buffer index, e.g., when buffering buffers.\n\n Yields\n ------\n batch\n A batch of size at most `buffer_size`\n\n Raises\n ------\n DataError\n If the stream contains items that are not data-like.\n '''\n\n data = []\n count = 0\n\n for item in stream:\n data.append(item)\n count += 1\n\n if count < buffer_size:\n continue\n try:\n yield __stack_data(data, axis=axis)\n except (TypeError, AttributeError):\n raise DataError(\"Malformed data stream: {}\".format(data))\n finally:\n data = []\n count = 0\n\n if data and partial:\n yield __stack_data(data, axis=axis)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreformats data as tuples.", "response": "def tuples(stream, *keys):\n \"\"\"Reformat data as tuples.\n\n Parameters\n ----------\n stream : iterable\n Stream of data objects.\n\n *keys : strings\n Keys to use for ordering data.\n\n Yields\n ------\n items : tuple of np.ndarrays\n Data object reformated as a tuple.\n\n Raises\n ------\n DataError\n If the stream contains items that are not data-like.\n KeyError\n If a data object does not contain the requested key.\n \"\"\"\n if not keys:\n raise PescadorError('Unable to generate tuples from '\n 'an empty item set')\n for data in stream:\n try:\n yield tuple(data[key] for key in keys)\n except TypeError:\n raise DataError(\"Malformed data stream: {}\".format(data))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreformat data objects as keras - compatible tuples.", "response": "def keras_tuples(stream, inputs=None, outputs=None):\n \"\"\"Reformat data objects as keras-compatible tuples.\n\n For more detail: https://keras.io/models/model/#fit\n\n Parameters\n ----------\n stream : iterable\n Stream of data objects.\n\n inputs : string or iterable of strings, None\n Keys to use for ordered input data.\n If not specified, returns `None` in its place.\n\n outputs : string or iterable of strings, default=None\n Keys to use for ordered output data.\n If not specified, returns `None` in its place.\n\n Yields\n ------\n x : np.ndarray, list of np.ndarray, or None\n If `inputs` is a string, `x` is a single np.ndarray.\n If `inputs` is an iterable of strings, `x` is a list of np.ndarrays.\n If `inputs` is a null type, `x` is None.\n y : np.ndarray, list of np.ndarray, or None\n If `outputs` is a string, `y` is a single np.ndarray.\n If `outputs` is an iterable of strings, `y` is a list of np.ndarrays.\n If `outputs` is a null type, `y` is None.\n\n Raises\n ------\n DataError\n If the stream contains items that are not data-like.\n \"\"\"\n flatten_inputs, flatten_outputs = False, False\n if inputs and isinstance(inputs, six.string_types):\n inputs = [inputs]\n flatten_inputs = True\n if outputs and isinstance(outputs, six.string_types):\n outputs = [outputs]\n flatten_outputs = True\n\n inputs, outputs = (inputs or []), (outputs or [])\n if not inputs + outputs:\n raise PescadorError('At least one key must be given for '\n '`inputs` or `outputs`')\n\n for data in stream:\n try:\n x = list(data[key] for key in inputs) or None\n if len(inputs) == 1 and flatten_inputs:\n x = x[0]\n\n y = list(data[key] for key in outputs) or None\n if len(outputs) == 1 and flatten_outputs:\n y = y[0]\n\n yield (x, y)\n except TypeError:\n raise DataError(\"Malformed data stream: {}\".format(data))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a histogram of motif location.", "response": "def location(args):\n \"\"\"\n Creates histrogram of motif location.\n\n Parameters\n ----------\n args : argparse object\n Command line arguments.\n \"\"\"\n fastafile = args.fastafile\n pwmfile = args.pwmfile\n\n lwidth = args.width\n if not lwidth:\n f = Fasta(fastafile)\n lwidth = len(f.items()[0][1])\n f = None\n\n jobs = []\n motifs = pwmfile_to_motifs(pwmfile)\n ids = [motif.id for motif in motifs]\n if args.ids:\n ids = args.ids.split(\",\")\n \n n_cpus = int(MotifConfig().get_default_params()[\"ncpus\"])\n pool = Pool(processes=n_cpus, maxtasksperchild=1000) \n for motif in motifs:\n if motif.id in ids:\n outfile = os.path.join(\"%s_histogram\" % motif.id)\n jobs.append(\n pool.apply_async(\n motif_localization, \n (fastafile,motif,lwidth,outfile, args.cutoff)\n ))\n \n for job in jobs:\n job.get()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind location of executable.", "response": "def which(fname):\n \"\"\"Find location of executable.\"\"\"\n if \"PATH\" not in os.environ or not os.environ[\"PATH\"]:\n path = os.defpath\n else:\n path = os.environ[\"PATH\"]\n\n for p in [fname] + [os.path.join(x, fname) for x in path.split(os.pathsep)]:\n p = os.path.abspath(p)\n if os.access(p, os.X_OK) and not os.path.isdir(p):\n return p\n\n p = sp.Popen(\"locate %s\" % fname, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)\n (stdout, stderr) = p.communicate()\n if not stderr:\n for p in stdout.decode().split(\"\\n\"):\n if (os.path.basename(p) == fname) and (\n os.access(p, os.X_OK)) and (\n not os.path.isdir(p)):\n return p"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfind all files in a directory by extension.", "response": "def find_by_ext(dirname, ext):\n \"\"\"Find all files in a directory by extension.\"\"\"\n # Get all fasta-files\n try: \n files = os.listdir(dirname) \n except OSError: \n if os.path.exists(dirname): \n cmd = \"find {0} -maxdepth 1 -name \\\"*\\\"\".format(dirname) \n p = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE) \n stdout, _stderr = p.communicate() \n files = [os.path.basename(fname) for fname in stdout.decode().splitlines()] \n else: \n raise \n \n retfiles = [os.path.join(dirname, fname) for fname in files if \n os.path.splitext(fname)[-1] in ext] \n \n return retfiles"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns list of Motif instances from default motif database.", "response": "def default_motifs():\n \"\"\"Return list of Motif instances from default motif database.\"\"\"\n config = MotifConfig()\n d = config.get_motif_dir()\n m = config.get_default_params()['motif_db']\n\n if not d or not m:\n raise ValueError(\"default motif database not configured\")\n\n fname = os.path.join(d, m)\n with open(fname) as f:\n motifs = read_motifs(f)\n \n return motifs"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef motif_from_align(align):\n width = len(align[0])\n nucs = {\"A\":0,\"C\":1,\"G\":2,\"T\":3}\n pfm = [[0 for _ in range(4)] for _ in range(width)]\n for row in align:\n for i in range(len(row)):\n pfm[i][nucs[row[i]]] += 1\n m = Motif(pfm)\n m.align = align[:]\n return m", "response": "Convert an alignment to a motif."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting a consensus sequence to a motif.", "response": "def motif_from_consensus(cons, n=12):\n \"\"\"Convert consensus sequence to motif.\n\n Converts a consensus sequences using the nucleotide IUPAC alphabet to a \n motif. \n\n Parameters\n ----------\n cons : str\n Consensus sequence using the IUPAC alphabet.\n n : int , optional\n Count used to convert the sequence to a PFM.\n \n Returns\n -------\n m : Motif instance\n Motif created from the consensus.\n \"\"\"\n width = len(cons)\n nucs = {\"A\":0,\"C\":1,\"G\":2,\"T\":3}\n pfm = [[0 for _ in range(4)] for _ in range(width)]\n m = Motif()\n for i,char in enumerate(cons):\n for nuc in m.iupac[char.upper()]:\n pfm[i][nucs[nuc]] = n / len(m.iupac[char.upper()])\n m = Motif(pfm)\n m.id = cons\n return m"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse motifs in a variety of formats to return a list of motifs.", "response": "def parse_motifs(motifs):\n \"\"\"Parse motifs in a variety of formats to return a list of motifs.\n\n Parameters\n ----------\n\n motifs : list or str\n Filename of motif, list of motifs or single Motif instance.\n\n Returns\n -------\n\n motifs : list\n List of Motif instances.\n \"\"\"\n if isinstance(motifs, six.string_types):\n with open(motifs) as f:\n if motifs.endswith(\"pwm\") or motifs.endswith(\"pfm\"):\n motifs = read_motifs(f, fmt=\"pwm\")\n elif motifs.endswith(\"transfac\"):\n motifs = read_motifs(f, fmt=\"transfac\")\n else: \n motifs = read_motifs(f)\n elif isinstance(motifs, Motif):\n motifs = [motifs]\n else:\n if not isinstance(list(motifs)[0], Motif):\n raise ValueError(\"Not a list of motifs\")\n \n return list(motifs)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _read_motifs_from_filehandle(handle, fmt):\n if fmt.lower() == \"pwm\":\n motifs = _read_motifs_pwm(handle)\n if fmt.lower() == \"transfac\":\n motifs = _read_motifs_transfac(handle)\n if fmt.lower() == \"xxmotif\":\n motifs = _read_motifs_xxmotif(handle)\n if fmt.lower() == \"align\":\n motifs = _read_motifs_align(handle)\n if fmt.lower() == \"jaspar\":\n motifs = _read_motifs_jaspar(handle)\n \n if handle.name:\n base = os.path.splitext(handle.name)[0]\n map_file = base + \".motif2factors.txt\"\n if os.path.exists(map_file):\n m2f_direct = {}\n m2f_indirect = {}\n for line in open(map_file):\n try:\n motif,*factor_info = line.strip().split(\"\\t\")\n if len(factor_info) == 1:\n m2f_direct[motif] = factor_info[0].split(\",\")\n elif len(factor_info) == 3:\n if factor_info[2] == \"Y\":\n m2f_direct[motif] = m2f_direct.get(motif, []) + [factor_info[0]]\n else:\n m2f_indirect[motif] = m2f_indirect.get(motif, []) + [factor_info[0]]\n except:\n pass\n for motif in motifs:\n if motif.id in m2f_direct:\n motif.factors[DIRECT_NAME] = m2f_direct[motif.id]\n if motif.id in m2f_indirect:\n motif.factors[INDIRECT_NAME] = m2f_indirect[motif.id]\n for motif in motifs:\n for n in [DIRECT_NAME, INDIRECT_NAME]:\n motif.factors[n] = list(set(motif.factors[n]))\n return motifs", "response": "Reads motifs from a file - like object."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread motifs from a file or stream or file - like object.", "response": "def read_motifs(infile=None, fmt=\"pwm\", as_dict=False):\n \"\"\" \n Read motifs from a file or stream or file-like object.\n\n Parameters\n ----------\n infile : string or file-like object, optional\n Motif database, filename of motif file or file-like object. If infile \n is not specified the default motifs as specified in the config file \n will be returned.\n\n fmt : string, optional\n Motif format, can be 'pwm', 'transfac', 'xxmotif', 'jaspar' or 'align'.\n \n as_dict : boolean, optional\n Return motifs as a dictionary with motif_id, motif pairs.\n \n Returns\n -------\n motifs : list\n List of Motif instances. If as_dict is set to True, motifs is a \n dictionary.\n \"\"\"\n if infile is None or isinstance(infile, six.string_types): \n infile = pwmfile_location(infile)\n with open(infile) as f:\n motifs = _read_motifs_from_filehandle(f, fmt)\n else:\n motifs = _read_motifs_from_filehandle(infile, fmt)\n\n if as_dict:\n motifs = {m.id:m for m in motifs}\n\n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef information_content(self):\n ic = 0\n for row in self.pwm:\n ic += 2.0 + np.sum([row[x] * log(row[x])/log(2) for x in range(4) if row[x] > 0])\n return ic", "response": "Return the total information content of the motif."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the minimum PWM score.", "response": "def pwm_min_score(self):\n \"\"\"Return the minimum PWM score.\n\n Returns\n -------\n score : float\n Minimum PWM score.\n \"\"\"\n if self.min_score is None:\n score = 0\n for row in self.pwm:\n score += log(min(row) / 0.25 + 0.01)\n self.min_score = score\n \n return self.min_score"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pwm_max_score(self):\n if self.max_score is None:\n score = 0\n for row in self.pwm:\n score += log(max(row) / 0.25 + 0.01)\n self.max_score = score\n \n return self.max_score", "response": "Return the maximum PWM score."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the log - oddds score for a specific k - mer.", "response": "def score_kmer(self, kmer):\n \"\"\"Calculate the log-odds score for a specific k-mer.\n\n Parameters\n ----------\n kmer : str\n String representing a kmer. Should be the same length as the motif.\n \n Returns\n -------\n score : float\n Log-odd score.\n \"\"\"\n if len(kmer) != len(self.pwm):\n raise Exception(\"incorrect k-mer length\")\n \n score = 0.0\n d = {\"A\":0, \"C\":1, \"G\":2, \"T\":3}\n for nuc, row in zip(kmer.upper(), self.pwm):\n score += log(row[d[nuc]] / 0.25 + 0.01)\n\n return score"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts a PFM with counts to a PFM with fractions.", "response": "def pfm_to_pwm(self, pfm, pseudo=0.001):\n \"\"\"Convert PFM with counts to a PFM with fractions.\n\n Parameters\n ----------\n pfm : list\n 2-dimensional list with counts.\n pseudo : float\n Pseudocount used in conversion.\n \n Returns\n -------\n pwm : list\n 2-dimensional list with fractions.\n \"\"\"\n return [[(x + pseudo)/(float(np.sum(row)) + pseudo * 4) for x in row] for row in pfm]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning motif formatted in MotEvo format.", "response": "def to_motevo(self):\n \"\"\"Return motif formatted in MotEvo (TRANSFAC-like) format\n \n Returns\n -------\n m : str\n String of motif in MotEvo format.\n \"\"\"\n m = \"//\\n\"\n m += \"NA {}\\n\".format(self.id)\n m += \"P0\\tA\\tC\\tG\\tT\\n\"\n for i, row in enumerate(self.pfm):\n m += \"{}\\t{}\\n\".format(i, \"\\t\".join([str(int(x)) for x in row]))\n m += \"//\"\n return m"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef to_transfac(self):\n m = \"%s\\t%s\\t%s\\n\" % (\"DE\", self.id, \"unknown\")\n for i, (row, cons) in enumerate(zip(self.pfm, self.to_consensus())):\n m += \"%i\\t%s\\t%s\\n\" % (i, \"\\t\".join([str(int(x)) for x in row]), cons)\n m += \"XX\"\n return m", "response": "Return motif formatted in TRANSFAC format\n \n Returns ------- str str\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef to_meme(self):\n motif_id = self.id.replace(\" \", \"_\")\n m = \"MOTIF %s\\n\" % motif_id\n m += \"BL MOTIF %s width=0 seqs=0\\n\"% motif_id\n m += \"letter-probability matrix: alength= 4 w= %s nsites= %s E= 0\\n\" % (len(self), np.sum(self.pfm[0]))\n m +=\"\\n\".join([\"\\t\".join([\"%s\" % x for x in row]) for row in self.pwm])\n return m", "response": "Return motif formatted in MEME format\n \n Returns ------- str str\n String of motif in MEME format\n \n "} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ic_pos(self, row1, row2=None):\n if row2 is None:\n row2 = [0.25,0.25,0.25,0.25]\n\n score = 0\n for a,b in zip(row1, row2):\n if a > 0:\n score += a * log(a / b) / log(2)\n return score", "response": "Calculate the information content of one position."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pcc_pos(self, row1, row2):\n mean1 = np.mean(row1)\n mean2 = np.mean(row2)\n\n a = 0\n x = 0\n y = 0\n for n1, n2 in zip(row1, row2):\n a += (n1 - mean1) * (n2 - mean2)\n x += (n1 - mean1) ** 2\n y += (n2 - mean2) ** 2\n \n if a == 0:\n return 0\n else:\n return a / sqrt(x * y)", "response": "Calculate the Pearson correlation coefficient of one position\n compared to another position\n ."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef rc(self):\n m = Motif()\n m.pfm = [row[::-1] for row in self.pfm[::-1]]\n m.pwm = [row[::-1] for row in self.pwm[::-1]]\n m.id = self.id + \"_revcomp\"\n return m", "response": "Return the reverse complemented motif."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nremove all positions at the specified edge_ic_cutoff from the motif.", "response": "def trim(self, edge_ic_cutoff=0.4):\n \"\"\"Trim positions with an information content lower than the threshold.\n\n The default threshold is set to 0.4. The Motif will be changed in-place.\n\n Parameters\n ----------\n edge_ic_cutoff : float, optional\n Information content threshold. All motif positions at the flanks \n with an information content lower thab this will be removed.\n\n Returns\n -------\n m : Motif instance\n \"\"\"\n pwm = self.pwm[:]\n while len(pwm) > 0 and self.ic_pos(pwm[0]) < edge_ic_cutoff:\n pwm = pwm[1:]\n self.pwm = self.pwm[1:]\n self.pfm = self.pfm[1:]\n while len(pwm) > 0 and self.ic_pos(pwm[-1]) < edge_ic_cutoff:\n pwm = pwm[:-1]\n self.pwm = self.pwm[:-1]\n self.pfm = self.pfm[:-1]\n \n self.consensus = None \n self.min_score = None\n self.max_score = None\n self.wiggled_pwm = None\n \n return self"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nscanning FASTA with the motif as a consensus sequence.", "response": "def consensus_scan(self, fa):\n \"\"\"Scan FASTA with the motif as a consensus sequence.\n\n Parameters\n ----------\n fa : Fasta object\n Fasta object to scan\n \n Returns\n -------\n matches : dict\n Dictionaru with matches.\n \"\"\"\n regexp = \"\".join([\"[\" + \"\".join(self.iupac[x.upper()]) + \"]\" for x in self.to_consensusv2()])\n p = re.compile(regexp)\n matches = {}\n for name,seq in fa.items():\n matches[name] = [] \n for match in p.finditer(seq):\n middle = (match.span()[1] + match.span()[0]) / 2\n matches[name].append(middle)\n return matches"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef pwm_scan(self, fa, cutoff=0.9, nreport=50, scan_rc=True):\n c = self.pwm_min_score() + (\n self.pwm_max_score() - self.pwm_min_score()) * cutoff \n pwm = self.pwm\n matches = {}\n for name, seq in fa.items():\n matches[name] = [] \n result = pfmscan(seq.upper(), pwm, c, nreport, scan_rc)\n for _,pos,_ in result:\n matches[name].append(pos)\n return matches", "response": "Scan the PWM sequences with this motif."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nscanning all PWM sequences with this motif.", "response": "def pwm_scan_all(self, fa, cutoff=0.9, nreport=50, scan_rc=True):\n \"\"\"Scan sequences with this motif.\n\n Scan sequences from a FASTA object with this motif. Less efficient \n than using a Scanner object. By setting the cutoff to 0.0 and \n nreport to 1, the best match for every sequence will be returned.\n The score, position and strand for every match is returned.\n\n Parameters\n ----------\n fa : Fasta object\n Fasta object to scan.\n cutoff : float , optional\n Cutoff to use for motif scanning. This cutoff is not specifically\n optimized and the strictness will vary a lot with motif lengh.\n nreport : int , optional\n Maximum number of matches to report.\n scan_rc : bool , optional\n Scan the reverse complement. True by default.\n \n Returns\n -------\n matches : dict\n Dictionary with motif matches. The score, position and strand for \n every match is returned.\n \"\"\"\n c = self.pwm_min_score() + (self.pwm_max_score() - self.pwm_min_score()) * cutoff \n pwm = self.pwm\n matches = {}\n for name, seq in fa.items():\n matches[name] = [] \n result = pfmscan(seq.upper(), pwm, c, nreport, scan_rc)\n for score,pos,strand in result:\n matches[name].append((pos,score,strand))\n return matches"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nscans a FASTA file with this motif and save it to a GFF file.", "response": "def pwm_scan_to_gff(self, fa, gfffile, cutoff=0.9, nreport=50, scan_rc=True, append=False):\n \"\"\"Scan sequences with this motif and save to a GFF file.\n\n Scan sequences from a FASTA object with this motif. Less efficient \n than using a Scanner object. By setting the cutoff to 0.0 and \n nreport to 1, the best match for every sequence will be returned.\n The output is save to a file in GFF format.\n\n Parameters\n ----------\n fa : Fasta object\n Fasta object to scan.\n gfffile : str\n Filename of GFF output file.\n cutoff : float , optional\n Cutoff to use for motif scanning. This cutoff is not specifically\n optimized and the strictness will vary a lot with motif lengh.\n nreport : int , optional\n Maximum number of matches to report.\n scan_rc : bool , optional\n Scan the reverse complement. True by default.\n append : bool , optional\n Append to GFF file instead of overwriting it. False by default.\n \"\"\"\n if append:\n out = open(gfffile, \"a\")\n else: \n out = open(gfffile, \"w\")\n\n c = self.pwm_min_score() + (self.pwm_max_score() - self.pwm_min_score()) * cutoff \n pwm = self.pwm\n\n strandmap = {-1:\"-\",\"-1\":\"-\",\"-\":\"-\",\"1\":\"+\",1:\"+\",\"+\":\"+\"}\n gff_line = (\"{}\\tpfmscan\\tmisc_feature\\t{}\\t{}\\t{:.3f}\\t{}\\t.\\t\"\n \"motif_name \\\"{}\\\" ; motif_instance \\\"{}\\\"\\n\")\n for name, seq in fa.items():\n result = pfmscan(seq.upper(), pwm, c, nreport, scan_rc)\n for score, pos, strand in result:\n out.write(gff_line.format( \n name, \n pos, \n pos + len(pwm), \n score, \n strandmap[strand], \n self.id, \n seq[pos:pos + len(pwm)]\n ))\n out.close()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the average of two motifs.", "response": "def average_motifs(self, other, pos, orientation, include_bg=False):\n \"\"\"Return the average of two motifs.\n\n Combine this motif with another motif and return the average as a new\n Motif object. The position and orientatien need to be supplied. The pos\n parameter is the position of the second motif relative to this motif.\n \n For example, take the following two motifs:\n Motif 1: CATGYT\n Motif 2: GGCTTGY\n\n With position -2, the motifs are averaged as follows:\n xxCATGYT\n GGCTTGYx\n\n Parameters\n ----------\n other : Motif object\n Other Motif object.\n pos : int\n Position of the second motif relative to this motif.\n orientation : int\n Orientation, should be 1 or -1. If the orientation is -1 then the \n reverse complement of the other motif is used for averaging.\n include_bg : bool , optional\n Extend both motifs with background frequencies (0.25) before\n averaging. False by default.\n \n Returns\n -------\n motif : motif object\n New Motif object containing average motif.\n \"\"\"\n # xxCATGYT\n # GGCTTGYx\n # pos = -2\n pfm1 = self.pfm[:]\n pfm2 = other.pfm[:]\n\n if orientation < 0:\n pfm2 = [row[::-1] for row in pfm2[::-1]]\n \n pfm1_count = float(np.sum(pfm1[0]))\n pfm2_count = float(np.sum(pfm2[0]))\n \n if include_bg:\n if len(pfm1) > len(pfm2) + pos:\n pfm2 += [[pfm2_count / 4.0 for x in range(4)] for i in range(-(len(pfm1) - len(pfm2) - pos), 0)]\n elif len(pfm2) + pos > len(pfm1):\n pfm1 += [[pfm1_count / 4.0 for x in range(4)] for i in range(-(len(pfm2) - len(pfm1) + pos), 0)]\n \n if pos < 0:\n pfm1 = [[pfm1_count / 4.0 for x in range(4)] for i in range(-pos)] + pfm1\n elif pos > 0:\n pfm2 = [[pfm2_count / 4.0 for x in range(4)] for i in range(pos)] + pfm2\n \n else:\n if len(pfm1) > len(pfm2) + pos:\n pfm2 += [[pfm1[i][x] / pfm1_count * (pfm2_count) for x in range(4)] for i in range(-(len(pfm1) - len(pfm2) - pos), 0)]\n elif len(pfm2) + pos > len(pfm1):\n pfm1 += [[pfm2[i][x] / pfm2_count * (pfm1_count) for x in range(4)] for i in range(-(len(pfm2) - len(pfm1) + pos), 0)]\n \n if pos < 0:\n pfm1 = [[pfm2[i][x] / pfm2_count * (pfm1_count) for x in range(4)] for i in range(-pos)] + pfm1\n elif pos > 0:\n pfm2 = [[pfm1[i][x] / pfm1_count * (pfm2_count) for x in range(4)] for i in range(pos)] + pfm2\n\n pfm = [[a + b for a,b in zip(x,y)] for x,y in zip(pfm1, pfm2)]\n \n m = Motif(pfm)\n m.id = m.to_consensus()\n return m"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _pwm_to_str(self, precision=4):\n if not self.pwm:\n return \"\"\n \n fmt = \"{{:.{:d}f}}\".format(precision)\n return \"\\n\".join(\n [\"\\t\".join([fmt.format(p) for p in row])\n for row in self.pwm]\n )", "response": "Return string representation of the pwm."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_pwm(self, precision=4, extra_str=\"\"):\n motif_id = self.id\n \n if extra_str:\n motif_id += \"_%s\" % extra_str\n\n if not self.pwm:\n self.pwm = [self.iupac_pwm[char]for char in self.consensus.upper()]\n\n return \">%s\\n%s\" % (\n motif_id, \n self._pwm_to_str(precision)\n )", "response": "Return a string representation of the Amino Acid motif."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to_img(self, fname, fmt=\"PNG\", add_left=0, seqlogo=None, height=6):\n if not seqlogo:\n seqlogo = self.seqlogo\n if not seqlogo:\n raise ValueError(\"seqlogo not specified or configured\")\n \n #TODO: split to_align function\n \n VALID_FORMATS = [\"EPS\", \"GIF\", \"PDF\", \"PNG\"]\n N = 1000\n fmt = fmt.upper()\n if not fmt in VALID_FORMATS:\n sys.stderr.write(\"Invalid motif format\\n\")\n return\n \n if fname[-4:].upper() == (\".%s\" % fmt):\n fname = fname[:-4]\n seqs = []\n if add_left == 0:\n seqs = [\"\" for i in range(N)]\n else:\n for nuc in [\"A\", \"C\", \"T\", \"G\"]:\n seqs += [nuc * add_left for i in range(N // 4)]\n\n for pos in range(len(self.pwm)):\n vals = [self.pwm[pos][0] * N]\n for i in range(1,4):\n vals.append(vals[i-1] + self.pwm[pos][i] * N)\n if vals[3] - N != 0:\n #print \"Motif weights don't add up to 1! Error of %s%%\" % ((vals[3] - n)/ n * 100)\n vals[3] = N\n for i in range(N):\n if i <= vals[0]:\n seqs[i] += \"A\"\n elif i <= vals[1]:\n seqs[i] += \"C\"\n elif i <= vals[2]:\n seqs[i] += \"G\"\n elif i <= vals[3]:\n seqs[i] += \"T\"\n \n f = NamedTemporaryFile(mode=\"w\", dir=mytmpdir())\n for seq in seqs:\n f.write(\"%s\\n\" % seq)\n f.flush()\n makelogo = \"{0} -f {1} -F {2} -c -a -h {3} -w {4} -o {5} -b -n -Y\" \n cmd = makelogo.format(\n seqlogo, \n f.name, \n fmt, \n height,\n len(self) + add_left, \n fname)\n sp.call(cmd, shell=True)", "response": "Create a sequence logo and save it to a file."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a new Motif instance with shuffled positions.", "response": "def randomize(self):\n \"\"\"Create a new motif with shuffled positions.\n\n Shuffle the positions of this motif and return a new Motif instance.\n\n Returns\n -------\n m : Motif instance\n Motif instance with shuffled positions.\n \"\"\"\n random_pfm = [[c for c in row] for row in self.pfm]\n random.shuffle(random_pfm)\n m = Motif(pfm=random_pfm)\n m.id = \"random\"\n return m"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef maelstrom(args):\n infile = args.inputfile\n genome = args.genome\n outdir = args.outdir\n pwmfile = args.pwmfile\n methods = args.methods\n ncpus = args.ncpus\n \n if not os.path.exists(infile):\n raise ValueError(\"file {} does not exist\".format(infile))\n\n if methods:\n methods = [x.strip() for x in methods.split(\",\")]\n\n run_maelstrom(infile, genome, outdir, pwmfile, methods=methods, ncpus=ncpus)", "response": "Run the maelstrom method."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef zmq_send_data(socket, data, flags=0, copy=True, track=False):\n\n header, payload = [], []\n\n for key in sorted(data.keys()):\n arr = data[key]\n\n if not isinstance(arr, np.ndarray):\n raise DataError('Only ndarray types can be serialized')\n\n header.append(dict(dtype=str(arr.dtype),\n shape=arr.shape,\n key=key,\n aligned=arr.flags['ALIGNED']))\n # Force contiguity\n payload.append(arr)\n\n # Send the header\n msg = [json.dumps(header).encode('ascii')]\n msg.extend(payload)\n\n return socket.send_multipart(msg, flags, copy=copy, track=track)", "response": "Send data to a socket"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef zmq_recv_data(socket, flags=0, copy=True, track=False):\n\n data = dict()\n\n msg = socket.recv_multipart(flags=flags, copy=copy, track=track)\n\n headers = json.loads(msg[0].decode('ascii'))\n\n if len(headers) == 0:\n raise StopIteration\n\n for header, payload in zip(headers, msg[1:]):\n data[header['key']] = np.frombuffer(buffer(payload),\n dtype=header['dtype'])\n data[header['key']].shape = header['shape']\n if six.PY2:\n # Legacy python won't let us preserve alignment, skip this step\n continue\n data[header['key']].flags['ALIGNED'] = header['aligned']\n\n return data", "response": "Receive data over a socket."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nyielding the data from the ZMQStreamer.", "response": "def iterate(self, max_iter=None):\n \"\"\"\n Note: A ZMQStreamer does not activate its stream,\n but allows the zmq_worker to do that.\n\n Yields\n ------\n data : dict\n Data drawn from `streamer(max_iter)`.\n \"\"\"\n context = zmq.Context()\n\n if six.PY2:\n warnings.warn('zmq_stream cannot preserve numpy array alignment '\n 'in Python 2', RuntimeWarning)\n\n try:\n socket = context.socket(zmq.PAIR)\n\n port = socket.bind_to_random_port('tcp://*',\n min_port=self.min_port,\n max_port=self.max_port,\n max_tries=self.max_tries)\n terminate = mp.Event()\n\n worker = mp.Process(target=SafeFunction(zmq_worker),\n args=[port, self.streamer, terminate],\n kwargs=dict(copy=self.copy,\n max_iter=max_iter))\n\n worker.daemon = True\n worker.start()\n\n # Yield from the queue as long as it's open\n while True:\n yield zmq_recv_data(socket)\n\n except StopIteration:\n pass\n\n except:\n # pylint: disable-msg=W0702\n six.reraise(*sys.exc_info())\n\n finally:\n terminate.set()\n worker.join(self.timeout)\n if worker.is_alive():\n worker.terminate()\n context.destroy()"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmasks all lowercase nucleotides with N s", "response": "def hardmask(self):\n \"\"\" Mask all lowercase nucleotides with N's \"\"\"\n p = re.compile(\"a|c|g|t|n\")\n for seq_id in self.fasta_dict.keys():\n self.fasta_dict[seq_id] = p.sub(\"N\", self.fasta_dict[seq_id])\n return self"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning n random sequences from this Fasta object", "response": "def get_random(self, n, l=None):\n \"\"\" Return n random sequences from this Fasta object \"\"\"\n random_f = Fasta()\n if l:\n ids = self.ids[:]\n random.shuffle(ids)\n i = 0\n while (i < n) and (len(ids) > 0):\n seq_id = ids.pop()\n if (len(self[seq_id]) >= l):\n start = random.randint(0, len(self[seq_id]) - l)\n random_f[\"random%s\" % (i + 1)] = self[seq_id][start:start+l]\n i += 1\n if len(random_f) != n:\n sys.stderr.write(\"Not enough sequences of required length\")\n return\n else:\n return random_f\n\n else:\n choice = random.sample(self.ids, n)\n for i in range(n):\n random_f[choice[i]] = self[choice[i]]\n return random_f"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef writefasta(self, fname):\n f = open(fname, \"w\")\n fa_str = \"\\n\".join([\">%s\\n%s\" % (id, self._format_seq(seq)) for id, seq in self.items()])\n f.write(fa_str)\n f.close()", "response": "Write the sequence list to a FASTA formatted file"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncluster motifs into a set of sequences.", "response": "def cluster_motifs(motifs, match=\"total\", metric=\"wic\", combine=\"mean\", pval=True, threshold=0.95, trim_edges=False, edge_ic_cutoff=0.2, include_bg=True, progress=True, ncpus=None):\n \"\"\" \n Clusters a set of sequence motifs. Required arg 'motifs' is a file containing\n positional frequency matrices or an array with motifs.\n\n Optional args:\n\n 'match', 'metric' and 'combine' specify the method used to compare and score\n the motifs. By default the WIC score is used (metric='wic'), using the the\n score over the whole alignment (match='total'), with the total motif score\n calculated as the mean score of all positions (combine='mean').\n 'match' can be either 'total' for the total alignment or 'subtotal' for the \n maximum scoring subsequence of the alignment.\n 'metric' can be any metric defined in MotifComparer, currently: 'pcc', 'ed',\n 'distance', 'wic' or 'chisq' \n 'combine' determines how the total score is calculated from the score of \n individual positions and can be either 'sum' or 'mean'\n \n 'pval' can be True or False and determines if the score should be converted to \n an empirical p-value\n\n 'threshold' determines the score (or p-value) cutoff\n\n If 'trim_edges' is set to True, all motif edges with an IC below \n 'edge_ic_cutoff' will be removed before clustering\n\n When computing the average of two motifs 'include_bg' determines if, at a \n position only present in one motif, the information in that motif should\n be kept, or if it should be averaged with background frequencies. Should\n probably be left set to True.\n\n \"\"\"\n\n \n # First read pfm or pfm formatted motiffile\n if type([]) != type(motifs):\n motifs = read_motifs(motifs, fmt=\"pwm\")\n \n mc = MotifComparer()\n\n # Trim edges with low information content\n if trim_edges:\n for motif in motifs:\n motif.trim(edge_ic_cutoff)\n \n # Make a MotifTree node for every motif\n nodes = [MotifTree(m) for m in motifs]\n \n # Determine all pairwise scores and maxscore per motif\n scores = {}\n motif_nodes = dict([(n.motif.id,n) for n in nodes])\n motifs = [n.motif for n in nodes]\n \n if progress:\n sys.stderr.write(\"Calculating initial scores\\n\")\n result = mc.get_all_scores(motifs, motifs, match, metric, combine, pval, parallel=True, ncpus=ncpus)\n \n for m1, other_motifs in result.items():\n for m2, score in other_motifs.items():\n if m1 == m2:\n if pval:\n motif_nodes[m1].maxscore = 1 - score[0]\n else:\n motif_nodes[m1].maxscore = score[0]\n else:\n if pval:\n score = [1 - score[0]] + score[1:]\n scores[(motif_nodes[m1],motif_nodes[m2])] = score\n \n cluster_nodes = [node for node in nodes]\n ave_count = 1\n \n total = len(cluster_nodes)\n\n while len(cluster_nodes) > 1:\n l = sorted(scores.keys(), key=lambda x: scores[x][0])\n i = -1\n (n1, n2) = l[i]\n while n1 not in cluster_nodes or n2 not in cluster_nodes:\n i -= 1\n (n1,n2) = l[i]\n \n if len(n1.motif) > 0 and len(n2.motif) > 0:\n (score, pos, orientation) = scores[(n1,n2)]\n ave_motif = n1.motif.average_motifs(n2.motif, pos, orientation, include_bg=include_bg)\n \n ave_motif.trim(edge_ic_cutoff)\n \n # Check if the motif is not empty\n if len(ave_motif) == 0:\n ave_motif = Motif([[0.25,0.25,0.25,0.25]])\n\n ave_motif.id = \"Average_%s\" % ave_count\n ave_count += 1\n \n new_node = MotifTree(ave_motif)\n if pval:\n new_node.maxscore = 1 - mc.compare_motifs(new_node.motif, new_node.motif, match, metric, combine, pval)[0]\n else:\n new_node.maxscore = mc.compare_motifs(new_node.motif, new_node.motif, match, metric, combine, pval)[0]\n \n new_node.mergescore = score\n #print \"%s + %s = %s with score %s\" % (n1.motif.id, n2.motif.id, ave_motif.id, score)\n n1.parent = new_node\n n2.parent = new_node\n new_node.left = n1\n new_node.right = n2\n \n cmp_nodes = dict([(node.motif, node) for node in nodes if not node.parent])\n \n if progress:\n progress = (1 - len(cmp_nodes) / float(total)) * 100\n sys.stderr.write('\\rClustering [{0}{1}] {2}%'.format(\n '#' * (int(progress) // 10), \n \" \" * (10 - int(progress) // 10), \n int(progress)))\n \n result = mc.get_all_scores(\n [new_node.motif], \n list(cmp_nodes.keys()), \n match, \n metric, \n combine, \n pval, \n parallel=True)\n \n for motif, n in cmp_nodes.items():\n x = result[new_node.motif.id][motif.id]\n if pval:\n x = [1 - x[0]] + x[1:]\n scores[(new_node, n)] = x\n \n nodes.append(new_node)\n \n cluster_nodes = [node for node in nodes if not node.parent]\n \n if progress:\n sys.stderr.write(\"\\n\") \n root = nodes[-1]\n for node in [node for node in nodes if not node.left]:\n node.parent.checkMerge(root, threshold)\n \n return root"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef batch_length(batch):\n '''Determine the number of samples in a batch.\n\n Parameters\n ----------\n batch : dict\n A batch dictionary. Each value must implement `len`.\n All values must have the same `len`.\n\n Returns\n -------\n n : int >= 0 or None\n The number of samples in this batch.\n If the batch has no fields, n is None.\n\n Raises\n ------\n PescadorError\n If some two values have unequal length\n '''\n n = None\n\n for value in six.itervalues(batch):\n if n is None:\n n = len(value)\n\n elif len(value) != n:\n raise PescadorError('Unequal field lengths')\n\n return n", "response": "Determine the number of samples in a batch."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nactivate a number of streams", "response": "def _activate(self):\n \"\"\"Activates a number of streams\"\"\"\n self.distribution_ = 1. / self.n_streams * np.ones(self.n_streams)\n self.valid_streams_ = np.ones(self.n_streams, dtype=bool)\n\n self.streams_ = [None] * self.k\n\n self.stream_weights_ = np.zeros(self.k)\n self.stream_counts_ = np.zeros(self.k, dtype=int)\n # Array of pointers into `self.streamers`\n self.stream_idxs_ = np.zeros(self.k, dtype=int)\n\n for idx in range(self.k):\n\n if not (self.distribution_ > 0).any():\n break\n\n self.stream_idxs_[idx] = self.rng.choice(\n self.n_streams, p=self.distribution_)\n self.streams_[idx], self.stream_weights_[idx] = (\n self._new_stream(self.stream_idxs_[idx]))\n\n self.weight_norm_ = np.sum(self.stream_weights_)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _new_stream(self, idx):\n '''Randomly select and create a stream.\n\n Parameters\n ----------\n idx : int, [0:n_streams - 1]\n The stream index to replace\n '''\n # instantiate\n if self.rate is not None:\n n_stream = 1 + self.rng.poisson(lam=self.rate)\n else:\n n_stream = None\n\n # If we're sampling without replacement, zero this one out\n # This effectively disables this stream as soon as it is chosen,\n # preventing it from being chosen again (unless it is revived)\n if not self.with_replacement:\n self.distribution_[idx] = 0.0\n\n # Correct the distribution\n if (self.distribution_ > 0).any():\n self.distribution_[:] /= np.sum(self.distribution_)\n\n return (self.streamers[idx].iterate(max_iter=n_stream),\n self.weights[idx])", "response": "Randomly select and create a new stream."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the next available sample stream.", "response": "def _next_sample_index(self):\n \"\"\"StochasticMux chooses its next sample stream randomly\"\"\"\n return self.rng.choice(self.n_active,\n p=(self.stream_weights_ /\n self.weight_norm_))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _activate_stream(self, idx):\n '''Randomly select and create a stream.\n\n StochasticMux adds mode handling to _activate_stream, making it so that\n if we're not sampling \"with_replacement\", the distribution for this\n chosen streamer is set to 0, causing the streamer not to be available\n until it is exhausted.\n\n Parameters\n ----------\n idx : int, [0:n_streams - 1]\n The stream index to replace\n '''\n # Get the number of samples for this streamer.\n n_samples_to_stream = None\n if self.rate is not None:\n n_samples_to_stream = 1 + self.rng.poisson(lam=self.rate)\n\n # instantiate a new streamer\n streamer = self.streamers[idx].iterate(max_iter=n_samples_to_stream)\n weight = self.weights[idx]\n\n # If we're sampling without replacement, zero this one out\n # This effectively disables this stream as soon as it is chosen,\n # preventing it from being chosen again (unless it is revived)\n # if not self.with_replacement:\n if self.mode != \"with_replacement\":\n self.distribution_[idx] = 0.0\n\n # Correct the distribution\n if (self.distribution_ > 0).any():\n self.distribution_[:] /= np.sum(self.distribution_)\n\n return streamer, weight", "response": "Randomly select and create a new streamer and weight."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nactivates the current class.", "response": "def _activate(self):\n \"\"\"ShuffledMux's activate is similar to StochasticMux,\n but there is no 'n_active', since all the streams are always available.\n \"\"\"\n self.streams_ = [None] * self.n_streams\n\n # Weights of the active streams.\n # Once a stream is exhausted, it is set to 0.\n # Upon activation, this is just a copy of self.weights.\n self.stream_weights_ = np.array(self.weights, dtype=float)\n # How many samples have been drawn from each (active) stream.\n self.stream_counts_ = np.zeros(self.n_streams, dtype=int)\n\n # Initialize each active stream.\n for idx in range(self.n_streams):\n # Setup a new streamer at this index.\n self._new_stream(idx)\n\n self.weight_norm_ = np.sum(self.stream_weights_)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _next_sample_index(self):\n # Return the next streamer index where the streamer is not None,\n # wrapping around.\n idx = self.active_index_\n self.active_index_ += 1\n\n if self.active_index_ >= len(self.streams_):\n self.active_index_ = 0\n\n # Continue to increment if this streamer is exhausted (None)\n # This should never be infinite looping;\n # the `_streamers_available` check happens immediately\n # before this, so there should always be at least one not-None\n # streamer.\n while self.streams_[idx] is None:\n idx = self.active_index_\n self.active_index_ += 1\n\n if self.active_index_ >= len(self.streams_):\n self.active_index_ = 0\n\n return idx", "response": "Rotates through each active sampler by incrementing the index and returning the next streamer index."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _new_stream(self, idx):\n # Get the stream index from the candidate pool\n stream_index = self.stream_idxs_[idx]\n\n # Activate the Streamer, and get the weights\n self.streams_[idx] = self.streamers[stream_index].iterate()\n\n # Reset the sample count to zero\n self.stream_counts_[idx] = 0", "response": "Activates a new stream given the index into the stream pool."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _replace_stream(self, idx=None):\n self.streams_[idx] = None\n\n # Check if we've now exhausted all the streams.\n if not self._streamers_available():\n if self.mode == 'exhaustive':\n pass\n\n elif self.mode == \"cycle\":\n self._setup_streams(permute=False)\n\n elif self.mode == \"permuted_cycle\":\n self._setup_streams(permute=True)", "response": "Called by BaseMux s iterate method when a stream is exhausted."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _new_stream(self):\n '''Grab the next stream from the input streamers, and start it.\n\n Raises\n ------\n StopIteration\n When the input list or generator of streamers is complete,\n will raise a StopIteration. If `mode == cycle`, it\n will instead restart iterating from the beginning of the sequence.\n '''\n try:\n # Advance the stream_generator_ to get the next available stream.\n # If successful, this will make self.chain_streamer_.active True\n next_stream = six.advance_iterator(self.stream_generator_)\n\n except StopIteration:\n # If running with cycle, restart the chain_streamer_ after\n # exhaustion.\n if self.mode == \"cycle\":\n self.stream_generator_ = self.chain_streamer_.iterate()\n\n # Try again to get the next stream;\n # if it fails this time, just let it raise the StopIteration;\n # this means the streams are probably dead or empty.\n next_stream = six.advance_iterator(self.stream_generator_)\n\n # If running in exhaustive mode\n else:\n # self.chain_streamer_ should no longer be active, so\n # the outer loop should fall out without running.\n next_stream = None\n\n if next_stream is not None:\n # Start that stream, and return it.\n streamer = next_stream.iterate()\n\n # Activate the Streamer\n self.streams_[0] = streamer\n\n # Reset the sample count to zero\n self.stream_counts_[0] = 0", "response": "Grab the next stream from the input streamers and start it."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef split_and_save_datasets(X, Y, paths):\n shuffled_idxs = np.random.permutation(np.arange(len(X)))\n\n for i in range(len(paths)):\n # Take every len(paths) item, starting at i.\n # len(paths) is 3, so this would be [0::3], [1::3], [2::3]\n X_i = X[shuffled_idxs[i::len(paths)]]\n Y_i = Y[shuffled_idxs[i::len(paths)]]\n np.savez(paths[i], X=X_i, Y=Y_i)", "response": "Shuffle X and Y into n datasets and save them to disk at the locations provided in paths."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating data from an npz file.", "response": "def npz_generator(npz_path):\n \"\"\"Generate data from an npz file.\"\"\"\n npz_data = np.load(npz_path)\n X = npz_data['X']\n # Y is a binary maxtrix with shape=(n, k), each y will have shape=(k,)\n y = npz_data['Y']\n\n n = X.shape[0]\n\n while True:\n i = np.random.randint(0, n)\n yield {'X': X[i], 'Y': y[i]}"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncomputes the hypergeometric version of the formula.", "response": "def phyper(k, good, bad, N):\n \"\"\" Current hypergeometric implementation in scipy is broken, so here's the correct version \"\"\"\n pvalues = [phyper_single(x, good, bad, N) for x in range(k + 1, N + 1)]\n return np.sum(pvalues)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write_equalwidth_bedfile(bedfile, width, outfile):\n\n BUFSIZE = 10000\n f = open(bedfile)\n out = open(outfile, \"w\")\n lines = f.readlines(BUFSIZE)\n line_count = 0\n while lines:\n for line in lines:\n line_count += 1\n if not line.startswith(\"#\") and not line.startswith(\"track\") and not line.startswith(\"browser\"):\n vals = line.strip().split(\"\\t\")\n try:\n start, end = int(vals[1]), int(vals[2])\n except ValueError:\n print(\"Error on line %s while reading %s. Is the file in BED or WIG format?\" % (line_count, bedfile))\n sys.exit(1)\n\n start = (start + end) // 2 - (width // 2)\n # This shifts the center, but ensures the width is identical... maybe not ideal\n if start < 0:\n start = 0\n end = start + width\n # Keep all the other information in the bedfile if it's there\n if len(vals) > 3:\n out.write(\"%s\\t%s\\t%s\\t%s\\n\" % (vals[0], start, end, \"\\t\".join(vals[3:])))\n else:\n out.write(\"%s\\t%s\\t%s\\n\" % (vals[0], start, end))\n lines = f.readlines(BUFSIZE)\n \n out.close()\n f.close()", "response": "Read input from bedfile set the width of all entries to width and \n write the result to outfile."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef calc_motif_enrichment(sample, background, mtc=None, len_sample=None, len_back=None):\n \n INF = \"Inf\"\n\n\n if mtc not in [None, \"Bonferroni\", \"Benjamini-Hochberg\", \"None\"]:\n raise RuntimeError(\"Unknown correction: %s\" % mtc)\n\n sig = {}\n p_value = {}\n n_sample = {}\n n_back = {}\n \n if not(len_sample):\n len_sample = sample.seqn()\n if not(len_back):\n len_back = background.seqn()\n\n for motif in sample.motifs.keys():\n p = \"NA\"\n s = \"NA\"\n q = len(sample.motifs[motif])\n m = 0\n if(background.motifs.get(motif)):\n m = len(background.motifs[motif])\n n = len_back - m\n k = len_sample\n p = phyper(q - 1, m, n, k) \n if p != 0:\n s = -(log(p)/log(10))\n else:\n s = INF\n else:\n s = INF\n p = 0.0\n\n sig[motif] = s\n p_value[motif] = p\n n_sample[motif] = q\n n_back[motif] = m\n \n if mtc == \"Bonferroni\":\n for motif in p_value.keys():\n if p_value[motif] != \"NA\":\n p_value[motif] = p_value[motif] * len(p_value.keys())\n if p_value[motif] > 1:\n p_value[motif] = 1\n elif mtc == \"Benjamini-Hochberg\":\n motifs = sorted(p_value.keys(), key=lambda x: -p_value[x])\n l = len(p_value)\n c = l\n for m in motifs:\n if p_value[m] != \"NA\":\n p_value[m] = p_value[m] * l / c \n c -= 1\n\n return (sig, p_value, n_sample, n_back)", "response": "Calculate enrichment based on hypergeometric distribution"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing a cutoff file and return a dictionary with motif id as key and cutoff as value", "response": "def parse_cutoff(motifs, cutoff, default=0.9):\n \"\"\" Provide either a file with one cutoff per motif or a single cutoff\n returns a hash with motif id as key and cutoff as value\n \"\"\"\n \n cutoffs = {}\n if os.path.isfile(str(cutoff)):\n for i,line in enumerate(open(cutoff)):\n if line != \"Motif\\tScore\\tCutoff\\n\":\n try:\n motif,_,c = line.strip().split(\"\\t\")\n c = float(c)\n cutoffs[motif] = c\n except Exception as e:\n sys.stderr.write(\"Error parsing cutoff file, line {0}: {1}\\n\".format(e, i + 1))\n sys.exit(1)\n else:\n for motif in motifs:\n cutoffs[motif.id] = float(cutoff)\n \n for motif in motifs:\n if not motif.id in cutoffs:\n sys.stderr.write(\"No cutoff found for {0}, using default {1}\\n\".format(motif.id, default))\n cutoffs[motif.id] = default\n return cutoffs"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef determine_file_type(fname):\n if not (isinstance(fname, str) or isinstance(fname, unicode)):\n raise ValueError(\"{} is not a file name!\", fname)\n\n if not os.path.isfile(fname):\n raise ValueError(\"{} is not a file!\", fname)\n\n ext = os.path.splitext(fname)[1].lower()\n if ext in [\"bed\"]:\n return \"bed\"\n elif ext in [\"fa\", \"fasta\"]:\n return \"fasta\"\n elif ext in [\"narrowpeak\"]:\n return \"narrowpeak\"\n\n try:\n Fasta(fname)\n return \"fasta\"\n except:\n pass\n # Read first line that is not a comment or an UCSC-specific line\n p = re.compile(r'^(#|track|browser)') \n with open(fname) as f:\n for line in f.readlines():\n line = line.strip()\n if not p.search(line):\n break\n region_p = re.compile(r'^(.+):(\\d+)-(\\d+)$')\n if region_p.search(line):\n return \"region\"\n else:\n vals = line.split(\"\\t\")\n if len(vals) >= 3:\n try:\n _, _ = int(vals[1]), int(vals[2])\n except ValueError:\n return \"unknown\"\n \n if len(vals) == 10:\n try: \n _, _ = int(vals[4]), int(vals[9])\n return \"narrowpeak\"\n except ValueError:\n # As far as I know there is no 10-column BED format\n return \"unknown\"\n pass\n return \"bed\"\n \n # Catch-all\n return \"unknown\"", "response": "Determine the file type of a UCSC file."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalculating md5 checksum of file.", "response": "def file_checksum(fname):\n \"\"\"Return md5 checksum of file.\n\n Note: only works for files < 4GB.\n\n Parameters\n ----------\n filename : str\n File used to calculate checksum.\n\n Returns\n -------\n checkum : str\n \"\"\"\n size = os.path.getsize(fname)\n with open(fname, \"r+\") as f:\n checksum = hashlib.md5(mmap.mmap(f.fileno(), size)).hexdigest()\n return checksum"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef download_annotation(genomebuild, gene_file): \n pred_bin = \"genePredToBed\"\n pred = find_executable(pred_bin)\n if not pred:\n sys.stderr.write(\"{} not found in path!\\n\".format(pred_bin))\n sys.exit(1)\n\n tmp = NamedTemporaryFile(delete=False, suffix=\".gz\")\n\n anno = []\n f = urlopen(UCSC_GENE_URL.format(genomebuild))\n p = re.compile(r'\\w+.Gene.txt.gz')\n for line in f.readlines():\n m = p.search(line.decode())\n if m:\n anno.append(m.group(0))\n\n sys.stderr.write(\"Retrieving gene annotation for {}\\n\".format(genomebuild))\n url = \"\"\n for a in ANNOS:\n if a in anno:\n url = UCSC_GENE_URL.format(genomebuild) + a\n break\n if url:\n sys.stderr.write(\"Using {}\\n\".format(url))\n urlretrieve(\n url,\n tmp.name\n )\n \n with gzip.open(tmp.name) as f:\n cols = f.readline().decode(errors='ignore').split(\"\\t\")\n\n start_col = 1\n for i,col in enumerate(cols):\n if col == \"+\" or col == \"-\":\n start_col = i - 1\n break\n end_col = start_col + 10\n \n cmd = \"zcat {} | cut -f{}-{} | {} /dev/stdin {}\"\n print(cmd.format(tmp.name, start_col, end_col, pred, gene_file))\n sp.call(cmd.format(\n tmp.name, start_col, end_col, pred, gene_file), \n shell=True)\n\n else:\n sys.stderr.write(\"No annotation found!\")", "response": "Download gene annotation from UCSC based on genomebuild."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _check_dir(self, dirname):\n if not os.path.exists(dirname):\n print(\"Directory %s does not exist!\" % dirname)\n sys.exit(1)", "response": "Check if directory exists and die if not"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nindexes a single one - sequence fasta - file", "response": "def _make_index(self, fasta, index):\n \"\"\" Index a single, one-sequence fasta-file\"\"\"\n out = open(index, \"wb\")\n f = open(fasta)\n # Skip first line of fasta-file\n line = f.readline()\n offset = f.tell()\n line = f.readline()\n while line:\n out.write(pack(self.pack_char, offset))\n offset = f.tell()\n line = f.readline()\n f.close()\n out.close()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_index(self,fasta_dir=None, index_dir=None):\n \n # Use default directories if they are not supplied\n if not fasta_dir:\n fasta_dir = self.fasta_dir\n\n if not index_dir:\n index_dir = self.index_dir\n\n # Can't continue if we still don't have an index_dir or fasta_dir\n if not fasta_dir:\n print(\"fasta_dir not defined!\")\n sys.exit(1)\n \n if not index_dir:\n print(\"index_dir not defined!\")\n sys.exit(1)\n \n index_dir = os.path.abspath(index_dir)\n fasta_dir = os.path.abspath(fasta_dir)\n\n self.index_dir = index_dir\n\n # Prepare index directory\n if not os.path.exists(index_dir):\n try:\n os.mkdir(index_dir)\n except OSError as e:\n if e.args[0] == 13:\n sys.stderr.write(\"No permission to create index directory. Superuser access needed?\\n\")\n sys.exit()\n else:\n sys.stderr.write(e)\n\n # Directories need to exist\n self._check_dir(fasta_dir)\n self._check_dir(index_dir)\n\n # Get all fasta-files \n\n fastafiles = find_by_ext(fasta_dir, FASTA_EXT)\n if not(fastafiles):\n msg = \"No fastafiles found in {} with extension in {}\".format(\n fasta_dir, \",\".join(FASTA_EXT))\n raise IOError(msg)\n\n # param_file will hold all the information about the location of the fasta-files, indeces and \n # length of the sequences\n param_file = os.path.join(index_dir, self.param_file)\n size_file = os.path.join(index_dir, self.size_file)\n \n try:\n out = open(param_file, \"w\")\n except IOError as e:\n if e.args[0] == 13:\n sys.stderr.write(\"No permission to create files in index directory. Superuser access needed?\\n\")\n sys.exit()\n else:\n sys.stderr.write(e)\n s_out = open(size_file, \"w\")\n\n for fasta_file in fastafiles:\n #sys.stderr.write(\"Indexing %s\\n\" % fasta_file)\n f = open(fasta_file)\n line = f.readline()\n if not line.startswith(\">\"):\n sys.stderr.write(\"%s is not a valid FASTA file, expected > at first line\\n\" % fasta_file)\n sys.exit()\n \n seqname = line.strip().replace(\">\", \"\")\n line = f.readline()\n line_size = len(line.strip())\n\n total_size = 0 \n while line:\n line = line.strip()\n if line.startswith(\">\"):\n sys.stderr.write(\"Sorry, can only index genomes with \"\n \"one sequence per FASTA file\\n%s contains multiple \"\n \"sequences\\n\" % fasta_file)\n sys.exit()\n \n total_size += len(line)\n line = f.readline()\n\n index_file = os.path.join(index_dir, \"%s.index\" % seqname)\n\n out.write(\"{}\\t{}\\t{}\\t{}\\t{}\\n\".format(\n seqname, fasta_file, index_file, line_size, total_size))\n s_out.write(\"{}\\t{}\\n\".format(seqname, total_size))\n \n self._make_index(fasta_file, index_file)\n f.close()\n out.close()\n s_out.close()\n\n # Read the index we just made so we can immediately use it\n self._read_index_file()", "response": "Create the index for the current object and store the results in index_dir"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _read_index_file(self):\n param_file = os.path.join(self.index_dir, self.param_file)\n with open(param_file) as f:\n for line in f.readlines():\n (name, fasta_file, index_file, line_size, total_size) = line.strip().split(\"\\t\")\n self.size[name] = int(total_size)\n self.fasta_file[name] = fasta_file\n self.index_file[name] = index_file\n self.line_size[name] = int(line_size)", "response": "read the index file"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _read_seq_from_fasta(self, fasta, offset, nr_lines):\n fasta.seek(offset)\n lines = [fasta.readline().strip() for _ in range(nr_lines)]\n return \"\".join(lines)", "response": "retrieve a number of lines from a fasta file - object starting at offset"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_sequences(self, chr, coords):\n # Check if we have an index_dir\n if not self.index_dir:\n print(\"Index dir is not defined!\")\n sys.exit()\n\n # retrieve all information for this specific sequence\n fasta_file = self.fasta_file[chr]\n index_file = self.index_file[chr]\n line_size = self.line_size[chr]\n total_size = self.size[chr]\n index = open(index_file, \"rb\")\n fasta = open(fasta_file)\n \n seqs = []\n for coordset in coords:\n seq = \"\"\n for (start,end) in coordset: \n if start > total_size:\n raise ValueError(\"%s: %s, invalid start, greater than sequence length!\" % (chr,start))\n \n if start < 0:\n raise ValueError(\"Invalid start, < 0!\")\n \n if end > total_size:\n raise ValueError(\"Invalid end, greater than sequence length!\")\n\n\n seq += self._read(index, fasta, start, end, line_size)\n seqs.append(seq)\n index.close()\n fasta.close()\n\n return seqs", "response": "Retrieve multiple sequences from same chr"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_sequence(self, chrom, start, end, strand=None):\n # Check if we have an index_dir\n if not self.index_dir:\n print(\"Index dir is not defined!\")\n sys.exit()\n\n # retrieve all information for this specific sequence\n fasta_file = self.fasta_file[chrom]\n index_file = self.index_file[chrom]\n line_size = self.line_size[chrom]\n total_size = self.size[chrom]\n\n #print fasta_file, index_file, line_size, total_size\n if start > total_size:\n raise ValueError(\n \"Invalid start {0}, greater than sequence length {1} of {2}!\".format(start, total_size, chrom))\n \n if start < 0:\n raise ValueError(\"Invalid start, < 0!\")\n \n if end > total_size:\n raise ValueError(\n \"Invalid end {0}, greater than sequence length {1} of {2}!\".format(end, total_size, chrom))\n\n\n index = open(index_file, \"rb\")\n fasta = open(fasta_file)\n seq = self._read(index, fasta, start, end, line_size)\n index.close()\n fasta.close()\n\n if strand and strand == \"-\":\n seq = rc(seq)\n return seq", "response": "Retrieve a sequence from the index file"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the sizes of all sequences in the index or the size of chrom if chrom is not specified.", "response": "def get_size(self, chrom=None):\n \"\"\" Return the sizes of all sequences in the index, or the size of chrom if specified\n as an optional argument \"\"\"\n if len(self.size) == 0:\n raise LookupError(\"no chromosomes in index, is the index correct?\")\n\n if chrom:\n if chrom in self.size:\n return self.size[chrom]\n else: \n raise KeyError(\"chromosome {} not in index\".format(chrom))\n total = 0\n for size in self.size.values():\n total += size\n\n return total"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns an instance of a specific tool.", "response": "def get_tool(name): \n \"\"\"\n Returns an instance of a specific tool.\n\n Parameters\n ----------\n name : str\n Name of the tool (case-insensitive).\n\n Returns\n -------\n tool : MotifProgram instance\n \"\"\"\n tool = name.lower()\n if tool not in __tools__:\n raise ValueError(\"Tool {0} not found!\\n\".format(name))\n\n t = __tools__[tool]()\n\n if not t.is_installed():\n sys.stderr.write(\"Tool {0} not installed!\\n\".format(tool))\n\n if not t.is_configured():\n sys.stderr.write(\"Tool {0} not configured!\\n\".format(tool))\n\n return t"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlocating a tool in the current directory.", "response": "def locate_tool(name, verbose=True): \n \"\"\"\n Returns the binary of a tool.\n\n Parameters\n ----------\n name : str\n Name of the tool (case-insensitive).\n\n Returns\n -------\n tool_bin : str\n Binary of tool.\n \"\"\"\n m = get_tool(name) \n tool_bin = which(m.cmd) \n if tool_bin:\n if verbose:\n print(\"Found {} in {}\".format(m.name, tool_bin)) \n return tool_bin \n else: \n print(\"Couldn't find {}\".format(m.name))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the command used to run the tool.", "response": "def bin(self):\n \"\"\"\n Get the command used to run the tool.\n\n Returns\n -------\n command : str\n The tool system command.\n \"\"\"\n if self.local_bin:\n return self.local_bin\n else:\n return self.config.bin(self.name)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef is_installed(self):\n return self.is_configured() and os.access(self.bin(), os.X_OK)", "response": "Check if the tool is installed."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run(self, fastafile, params=None, tmp=None):\n if not self.is_configured():\n raise ValueError(\"%s is not configured\" % self.name)\n\n if not self.is_installed():\n raise ValueError(\"%s is not installed or not correctly configured\" % self.name)\n \n self.tmpdir = mkdtemp(prefix=\"{0}.\".format(self.name), dir=tmp)\n fastafile = os.path.abspath(fastafile)\n \n try:\n return self._run_program(self.bin(), fastafile, params)\n except KeyboardInterrupt:\n return ([], \"Killed\", \"Killed\")", "response": "Run the tool and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing parameters. Combine default and user-defined parameters.", "response": "def _parse_params(self, params=None):\n \"\"\"\n Parse parameters.\n\n Combine default and user-defined parameters.\n \"\"\"\n prm = self.default_params.copy()\n if params is not None: \n prm.update(params)\n \n if prm[\"background\"]:\n # Absolute path, just to be sure\n prm[\"background\"] = os.path.abspath(prm[\"background\"])\n prm[\"background\"] = \" --negSet {0} \".format(\n prm[\"background\"])\n \n prm[\"strand\"] = \"\"\n if not prm[\"single\"]:\n prm[\"strand\"] = \" --revcomp \"\n\n return prm"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _run_program(self, bin, fastafile, params=None):\n params = self._parse_params(params) \n \n outfile = os.path.join(\n self.tmpdir, \n os.path.basename(fastafile.replace(\".fa\", \".pwm\")))\n \n stdout = \"\"\n stderr = \"\"\n \n cmd = \"%s %s %s --localization --batch %s %s\" % (\n bin,\n self.tmpdir, \n fastafile,\n params[\"background\"],\n params[\"strand\"],\n )\n \n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n out,err = p.communicate()\n stdout += out.decode()\n stderr += err.decode()\n \n motifs = []\n \n if os.path.exists(outfile):\n motifs = read_motifs(outfile, fmt=\"xxmotif\")\n for m in motifs:\n m.id = \"{0}_{1}\".format(self.name, m.id)\n else:\n stdout += \"\\nMotif file {0} not found!\\n\".format(outfile)\n stderr += \"\\nMotif file {0} not found!\\n\".format(outfile)\n \n return motifs, stdout, stderr", "response": "Run XXmotif and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing parameters. Combine default and user-defined parameters.", "response": "def _parse_params(self, params=None):\n \"\"\"\n Parse parameters.\n\n Combine default and user-defined parameters.\n \"\"\"\n prm = self.default_params.copy()\n if params is not None: \n prm.update(params)\n \n # Background file is essential!\n if not prm[\"background\"]:\n print(\"Background file needed!\")\n sys.exit()\n \n prm[\"background\"] = os.path.abspath(prm[\"background\"])\n \n prm[\"strand\"] = \"\"\n if prm[\"single\"]:\n prm[\"strand\"] = \" -strand + \"\n \n return prm"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _run_program(self, bin, fastafile, params=None):\n params = self._parse_params(params) \n \n outfile = NamedTemporaryFile(\n mode=\"w\",\n dir=self.tmpdir, \n prefix= \"homer_w{}.\".format(params[\"width\"])\n ).name\n \n cmd = \"%s denovo -i %s -b %s -len %s -S %s %s -o %s -p 8\" % (\n bin,\n fastafile,\n params[\"background\"],\n params[\"width\"],\n params[\"number\"],\n params[\"strand\"],\n outfile)\n\n stderr = \"\"\n stdout = \"Running command:\\n{}\\n\".format(cmd)\n \n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.tmpdir) \n out,err = p.communicate()\n stdout += out.decode()\n stderr += err.decode()\n \n motifs = []\n \n if os.path.exists(outfile):\n motifs = read_motifs(outfile, fmt=\"pwm\")\n for i, m in enumerate(motifs):\n m.id = \"{}_{}_{}\".format(self.name, params[\"width\"], i + 1)\n \n return motifs, stdout, stderr", "response": "Run Homer and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts BioProspector output to motifs", "response": "def parse(self, fo):\n \"\"\"\n Convert BioProspector output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing BioProspector output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n \n p = re.compile(r'^\\d+\\s+(\\d+\\.\\d+)\\s+(\\d+\\.\\d+)\\s+(\\d+\\.\\d+)\\s+(\\d+\\.\\d+)')\n pwm = []\n motif_id = \"\"\n for line in fo.readlines():\n if line.startswith(\"Motif #\"):\n if pwm:\n m = Motif(pwm)\n m.id = \"BioProspector_w%s_%s\" % (len(m), motif_id)\n motifs.append(m)\n motif_id = line.split(\"#\")[1].split(\":\")[0]\n pwm = []\n else:\n m = p.search(line)\n if m:\n pwm.append([float(m.group(x))/100.0 for x in range(1,5)])\n\n if pwm:\n m = Motif(pwm)\n m.id = \"BioProspector_w%s_%s\" % (len(m), motif_id)\n motifs.append(m)\n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _run_program(self, bin, fastafile, params=None):\n params = self._parse_params(params)\n \n default_params = {\"width\":10}\n if params is not None: \n default_params.update(params)\n \n fgfile, summitfile, outfile = self._prepare_files(fastafile)\n \n current_path = os.getcwd()\n os.chdir(self.tmpdir)\n \n cmd = \"{} -i {} -w {} -dna 4 -iteration 50 -chain 20 -seqprop -0.1 -strand 2 -peaklocation {} -t_dof 3 -dep 2\".format(\n bin, \n fgfile, \n params['width'], \n summitfile)\n\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n stdout,stderr = p.communicate()\n \n os.chdir(current_path)\n \n motifs = []\n if os.path.exists(outfile):\n with open(outfile) as f: \n motifs = self.parse(f)\n for i,m in enumerate(motifs):\n m.id = \"HMS_w{}_{}\".format(params['width'], i + 1)\n \n return motifs, stdout, stderr", "response": "Run HMS and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert HMS output to motifs", "response": "def parse(self, fo):\n \"\"\"\n Convert HMS output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing HMS output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n m = [[float(x) for x in fo.readline().strip().split(\" \")] for i in range(4)]\n matrix = [[m[0][i], m[1][i],m[2][i],m[3][i]] for i in range(len(m[0]))]\n motifs = [Motif(matrix)]\n motifs[-1].id = self.name\n \n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _run_program(self, bin, fastafile, params=None):\n params = self._parse_params(params)\n\n fgfile = os.path.join(self.tmpdir, \"AMD.in.fa\")\n outfile = fgfile + \".Matrix\" \n shutil.copy(fastafile, fgfile)\n \n current_path = os.getcwd()\n os.chdir(self.tmpdir)\n \n stdout = \"\"\n stderr = \"\"\n \n cmd = \"%s -F %s -B %s\" % (\n bin, \n fgfile, \n params[\"background\"],\n )\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n out,err = p.communicate()\n stdout += out.decode()\n stderr += err.decode()\n \n os.chdir(current_path)\n motifs = []\n if os.path.exists(outfile):\n f = open(outfile)\n motifs = self.parse(f)\n f.close()\n \n return motifs, stdout, stderr", "response": "Run AMD and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconverting AMD output to motifs AttributeNames", "response": "def parse(self, fo):\n \"\"\"\n Convert AMD output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing AMD output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n \n #160: 112 CACGTGC 7.25 chr14:32308489-32308689\n p = re.compile(r'\\d+\\s+([\\d.]+)\\s+([\\d.]+)\\s+([\\d.]+)\\s+([\\d.]+)')\n wm = []\n name = \"\"\n for line in fo.readlines():\n if line.startswith(\"Motif\") and line.strip().endswith(\":\"):\n if name:\n motifs.append(Motif(wm))\n motifs[-1].id = name\n name = \"\"\n wm = []\n name = \"%s_%s\" % (self.name, line.split(\":\")[0])\n else:\n m = p.search(line)\n if m:\n wm.append([float(m.group(x)) for x in range(1,5)])\n motifs.append(Motif(wm))\n motifs[-1].id = name\n \n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting Improbizer output to motifs", "response": "def parse(self, fo):\n \"\"\"\n Convert Improbizer output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing Improbizer output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n p = re.compile(r'\\d+\\s+@\\s+\\d+\\.\\d+\\s+sd\\s+\\d+\\.\\d+\\s+(\\w+)$')\n\n line = fo.readline()\n while line and line.find(\"Color\") == -1:\n m = p.search(line)\n if m:\n pwm_data = {}\n for i in range(4):\n vals = [x.strip() for x in fo.readline().strip().split(\" \") if x]\n pwm_data[vals[0].upper()] = vals[1:]\n pwm = []\n for i in range(len(pwm_data[\"A\"])):\n pwm.append([float(pwm_data[x][i]) for x in [\"A\",\"C\",\"G\",\"T\"]])\n motifs.append(Motif(pwm))\n motifs[-1].id = \"%s_%s\" % (self.name, m.group(1))\n line = fo.readline()\n \n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _run_program(self, bin, fastafile, params=None):\n params = self._parse_params(params)\n\n tmp = NamedTemporaryFile(mode=\"w\", dir=self.tmpdir, delete=False)\n shutil.copy(fastafile, tmp.name)\n fastafile = tmp.name\n \n current_path = os.getcwd()\n os.chdir(self.dir())\n \n motifs = []\n stdout = \"\"\n stderr = \"\"\n for wildcard in [0,1,2]:\n cmd = \"%s -sample %s -background %s -directory %s -strand %s -wildcard %s\" % (\n bin, \n fastafile, \n params[\"background\"], \n self.tmpdir, \n params[\"strand\"],\n wildcard,\n )\n \n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n out,err = p.communicate()\n stdout += out.decode()\n stderr += err.decode()\n \n os.chdir(current_path)\n pwmfiles = glob.glob(\"{}/tmp*/result/*pwm\".format(self.tmpdir))\n if len(pwmfiles) > 0:\n out_file = pwmfiles[0]\n stdout += \"\\nOutfile: {}\".format(out_file)\n \n my_motifs = []\n if os.path.exists(out_file):\n my_motifs = read_motifs(out_file, fmt=\"pwm\")\n for m in motifs:\n m.id = \"{}_{}\".format(self.name, m.id)\n stdout += \"\\nTrawler: {} motifs\".format(len(motifs))\n \n # remove temporary files\n if os.path.exists(tmp.name):\n os.unlink(tmp.name)\n \n for motif in my_motifs:\n motif.id = \"{}_{}_{}\".format(self.name, wildcard, motif.id)\n \n motifs += my_motifs\n else:\n stderr += \"\\nNo outfile found\"\n\n return motifs, stdout, stderr", "response": "Run the tool and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns the weeder and predict motifs from a FASTA file.", "response": "def _run_program(self, bin,fastafile, params=None):\n \"\"\"\n Run Weeder and predict motifs from a FASTA file.\n\n Parameters\n ----------\n bin : str\n Command used to run the tool.\n \n fastafile : str\n Name of the FASTA input file.\n\n params : dict, optional\n Optional parameters. For some of the tools required parameters\n are passed using this dictionary.\n\n Returns\n -------\n motifs : list of Motif instances\n The predicted motifs.\n\n stdout : str\n Standard out of the tool.\n \n stderr : str\n Standard error of the tool.\n \"\"\"\n params = self._parse_params(params)\n \n organism = params[\"organism\"]\n weeder_organisms = {\n \"hg18\":\"HS\",\n \"hg19\":\"HS\",\n \"hg38\":\"HS\",\n \"mm9\":\"MM\",\n \"mm10\":\"MM\",\n \"dm3\":\"DM\",\n \"dm5\":\"DM\",\n \"dm6\":\"DM\",\n \"yeast\":\"SC\",\n \"sacCer2\":\"SC\",\n \"sacCer3\":\"SC\",\n \"TAIR10\":\"AT\",\n \"TAIR11\":\"AT\",\n }\n weeder_organism = weeder_organisms.get(organism, \"HS\")\n\n tmp = NamedTemporaryFile(dir=self.tmpdir)\n name = tmp.name\n tmp.close()\n shutil.copy(fastafile, name)\n fastafile = name\n \n cmd = \"{} -f {} -O\".format(\n self.cmd, \n fastafile,\n weeder_organism,\n )\n \n if params[\"single\"]:\n cmd += \" -ss\"\n \n #print cmd\n stdout, stderr = \"\", \"\"\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.tmpdir) \n out,err = p.communicate()\n stdout += out.decode()\n stderr += err.decode()\n\n motifs = []\n if os.path.exists(fastafile + \".matrix.w2\"):\n f = open(fastafile + \".matrix.w2\")\n motifs = self.parse(f)\n f.close()\n \n for m in motifs:\n m.id = \"{}_{}\".format(self.name, m.id.split(\"\\t\")[0])\n \n for ext in [\".w2\", \".matrix.w2\" ]:\n if os.path.exists(fastafile + ext):\n os.unlink(fastafile + ext)\n\n return motifs, stdout, stderr"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse parameters. Combine default and user-defined parameters.", "response": "def _parse_params(self, params=None):\n \"\"\"\n Parse parameters.\n\n Combine default and user-defined parameters.\n \"\"\"\n prm = self.default_params.copy()\n if params is not None: \n prm.update(params)\n \n if prm[\"background_model\"]:\n # Absolute path, just to be sure\n prm[\"background_model\"] = os.path.abspath(prm[\"background_model\"])\n else:\n if prm.get(\"organism\", None):\n prm[\"background_model\"] = os.path.join(\n self.config.get_bg_dir(), \n \"{}.{}.bg\".format(\n prm[\"organism\"], \n \"MotifSampler\"))\n else: \n raise Exception(\"No background specified for {}\".format(self.name))\n \n prm[\"strand\"] = 1\n if prm[\"single\"]:\n prm[\"strand\"] = 0\n \n tmp = NamedTemporaryFile(dir=self.tmpdir)\n prm[\"pwmfile\"] = tmp.name\n\n tmp2 = NamedTemporaryFile(dir=self.tmpdir)\n prm[\"outfile\"] = tmp2.name\n \n return prm"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun MotifSampler and predict motifs from a FASTA file.", "response": "def _run_program(self, bin, fastafile, params=None):\n \"\"\"\n Run MotifSampler and predict motifs from a FASTA file.\n\n Parameters\n ----------\n bin : str\n Command used to run the tool.\n \n fastafile : str\n Name of the FASTA input file.\n\n params : dict, optional\n Optional parameters. For some of the tools required parameters\n are passed using this dictionary.\n\n Returns\n -------\n motifs : list of Motif instances\n The predicted motifs.\n\n stdout : str\n Standard out of the tool.\n \n stderr : str\n Standard error of the tool.\n \"\"\"\n params = self._parse_params(params)\n # TODO: test organism\n #cmd = \"%s -f %s -b %s -m %s -w %s -n %s -o %s -s %s > /dev/null 2>&1\" % (\n cmd = \"%s -f %s -b %s -m %s -w %s -n %s -o %s -s %s\" % (\n bin, \n fastafile, \n params[\"background_model\"], \n params[\"pwmfile\"], \n params[\"width\"], \n params[\"number\"], \n params[\"outfile\"],\n params[\"strand\"],\n )\n #print cmd\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n stdout, stderr = p.communicate()\n \n #stdout,stderr = \"\",\"\"\n #p = Popen(cmd, shell=True)\n #p.wait()\n\n motifs = []\n if os.path.exists(params[\"outfile\"]):\n with open(params[\"outfile\"]) as f:\n motifs = self.parse_out(f)\n \n for motif in motifs:\n motif.id = \"%s_%s\" % (self.name, motif.id)\n \n return motifs, stdout, stderr"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert MotifSampler output to motifs Arc Arc.", "response": "def parse(self, fo):\n \"\"\"\n Convert MotifSampler output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing MotifSampler output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n\n pwm = []\n info = {}\n for line in fo.readlines():\n if line.startswith(\"#\"):\n vals = line.strip()[1:].split(\" = \")\n if len(vals) > 1:\n info[vals[0]] = vals[1]\n elif len(line) > 1:\n pwm.append([float(x) for x in line.strip().split(\"\\t\")])\n else:\n motifs.append(Motif())\n motifs[-1].consensus = info[\"Consensus\"]\n motifs[-1].width = info[\"W\"]\n motifs[-1].id = info[\"ID\"]\n motifs[-1].pwm = pwm[:]\n pwm = []\n \n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse_out(self, fo):\n motifs = []\n nucs = {\"A\":0,\"C\":1,\"G\":2,\"T\":3}\n pseudo = 0.0 # Should be 1/sqrt(# of seqs)\n aligns = {}\n for line in fo.readlines():\n if line.startswith(\"#\"):\n pass\n elif len(line) > 1:\n vals = line.strip().split(\"\\t\")\n m_id, site = [x.strip().split(\" \")[1].replace('\"',\"\") for x in vals[8].split(\";\") if x]\n #if vals[6] == \"+\":\n if site.upper().find(\"N\") == -1:\n aligns.setdefault(m_id, []).append(site)\n #else:\n # print site, rc(site)\n # aligns.setdefault(id, []).append(rc(site))\n \n for m_id, align in aligns.items():\n #print id, len(align)\n\n width = len(align[0])\n pfm = [[0 for x in range(4)] for x in range(width)]\n for row in align:\n for i in range(len(row)):\n pfm[i][nucs[row[i]]] += 1\n total = float(len(align))\n pwm = [[(x + pseudo/4)/total+(pseudo) for x in row] for row in pfm]\n m = Motif()\n m.align = align[:]\n m.pwm = pwm[:]\n m.pfm = pfm[:]\n m.id = m_id\n motifs.append(m) \n return motifs", "response": "Convert MotifSampler output to motifs\n Arc Arc."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _run_program(self, bin, fastafile, params=None):\n default_params = {\"width\":10, \"number\":10}\n if params is not None: \n default_params.update(params)\n \n new_file = os.path.join(self.tmpdir, \"mdmodule_in.fa\")\n shutil.copy(fastafile, new_file)\n \n fastafile = new_file\n pwmfile = fastafile + \".out\"\n \n width = default_params['width']\n number = default_params['number']\n \n current_path = os.getcwd()\n os.chdir(self.tmpdir) \n cmd = \"%s -i %s -a 1 -o %s -w %s -t 100 -r %s\" % (bin, fastafile, pwmfile, width, number)\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n stdout,stderr = p.communicate()\n \n stdout = \"cmd: {}\\n\".format(cmd) + stdout.decode() \n \n motifs = []\n if os.path.exists(pwmfile):\n with open(pwmfile) as f:\n motifs = self.parse(f)\n \n os.chdir(current_path)\n \n for motif in motifs:\n motif.id = \"%s_%s\" % (self.name, motif.id)\n \n return motifs, stdout, stderr", "response": "Run MDmodule and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing MDmodule output to motifs osity.", "response": "def parse(self, fo):\n \"\"\"\n Convert MDmodule output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing MDmodule output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n nucs = {\"A\":0,\"C\":1,\"G\":2,\"T\":3}\n p = re.compile(r'(\\d+)\\s+(\\d+\\.\\d+)\\s+(\\d+\\.\\d+)\\s+(\\d+\\.\\d+)\\s+(\\d+\\.\\d+)')\n pf = re.compile(r'>.+\\s+[bf]\\d+\\s+(\\w+)')\n\n pwm = []\n pfm = []\n align = []\n m_id = \"\"\n for line in fo.readlines():\n if line.startswith(\"Motif\"):\n if m_id:\n motifs.append(Motif())\n motifs[-1].id = m_id\n motifs[-1].pwm = pwm\n motifs[-1].pfm = pfm\n motifs[-1].align = align\n pwm = []\n pfm = []\n align = []\n m_id = line.split(\"\\t\")[0]\n else: \n m = p.search(line)\n if m:\n pwm.append([float(m.group(x))/100 for x in [2,3,4,5]])\n m = pf.search(line)\n if m:\n if not pfm:\n pfm = [[0 for x in range(4)] for x in range(len(m.group(1)))]\n for i in range(len(m.group(1))):\n pfm[i][nucs[m.group(1)[i]]] += 1\n \n align.append(m.group(1))\n \n if pwm:\n motifs.append(Motif())\n motifs[-1].id = m_id\n motifs[-1].pwm = pwm\n motifs[-1].pfm = pfm\n motifs[-1].align = align\n\n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing parameters. Combine default and user-defined parameters.", "response": "def _parse_params(self, params=None):\n \"\"\"\n Parse parameters.\n\n Combine default and user-defined parameters.\n \"\"\"\n prm = self.default_params.copy()\n if params is not None: \n prm.update(params)\n \n return prm"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _run_program(self, bin, fastafile, params=None):\n params = self._parse_params(params)\n basename = \"munk_in.fa\"\n\n new_file = os.path.join(self.tmpdir, basename)\n out = open(new_file, \"w\")\n f = Fasta(fastafile)\n for seq in f.seqs:\n header = len(seq) // 2\n out.write(\">%s\\n\" % header)\n out.write(\"%s\\n\" % seq)\n out.close()\n \n fastafile = new_file\n outfile = fastafile + \".out\"\n\n current_path = os.getcwd()\n os.chdir(self.dir())\n \n motifs = []\n # Max recommended by ChIPMunk userguide\n ncpus = 4\n stdout = \"\"\n stderr = \"\"\n for zoops_factor in [\"oops\", 0.0, 0.5, 1.0]:\n cmd = \"{} {} {} y {} m:{} 100 10 1 {} 1>{}\".format(\n bin, \n params.get(\"width\", 8),\n params.get(\"width\", 20),\n zoops_factor, \n fastafile, \n ncpus, \n outfile\n )\n #print(\"command: \", cmd)\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n std = p.communicate()\n stdout = stdout + std[0].decode()\n stderr = stderr + std[1].decode()\n\n if \"RuntimeException\" in stderr:\n return [], stdout, stderr\n \n if os.path.exists(outfile):\n with open(outfile) as f:\n motifs += self.parse(f)\n \n os.chdir(current_path)\n \n return motifs, stdout, stderr", "response": "Run ChIPMunk and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef parse(self, fo):\n #KDIC|6.124756232026243\n #A|517.9999999999999 42.99999999999999 345.99999999999994 25.999999999999996 602.9999999999999 155.99999999999997 2.9999999999999996 91.99999999999999\n #C|5.999999999999999 4.999999999999999 2.9999999999999996 956.9999999999999 91.99999999999999 17.999999999999996 22.999999999999996 275.99999999999994\n #G|340.99999999999994 943.9999999999999 630.9999999999999 6.999999999999999 16.999999999999996 48.99999999999999 960.9999999999999 14.999999999999998\n #T|134.99999999999997 7.999999999999999 19.999999999999996 9.999999999999998 287.99999999999994 776.9999999999999 12.999999999999998 616.9999999999999\n #N|999.9999999999998\n line = fo.readline()\n if not line:\n return []\n \n while not line.startswith(\"A|\"):\n line = fo.readline() \n matrix = []\n for _ in range(4):\n matrix.append([float(x) for x in line.strip().split(\"|\")[1].split(\" \")])\n line = fo.readline()\n #print matrix\n matrix = [[matrix[x][y] for x in range(4)] for y in range(len(matrix[0]))]\n #print matrix\n m = Motif(matrix)\n m.id = \"ChIPMunk_w%s\" % len(m)\n return [m]", "response": "Parses ChIPMunk output to motifs"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _run_program(self, bin, fastafile, params=None):\n default_params = {}\n if params is not None: \n default_params.update(params)\n \n width = params.get(\"width\", 8)\n basename = \"posmo_in.fa\"\n\n new_file = os.path.join(self.tmpdir, basename)\n shutil.copy(fastafile, new_file)\n \n fastafile = new_file\n #pwmfile = fastafile + \".pwm\"\n \n motifs = []\n current_path = os.getcwd()\n os.chdir(self.tmpdir) \n for n_ones in range(4, min(width, 11), 2):\n x = \"1\" * n_ones\n outfile = \"%s.%s.out\" % (fastafile, x)\n cmd = \"%s 5000 %s %s 1.6 2.5 %s 200\" % (bin, x, fastafile, width)\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n stdout, stderr = p.communicate()\n stdout = stdout.decode()\n stderr = stderr.decode()\n\n context_file = fastafile.replace(basename, \"context.%s.%s.txt\" % (basename, x))\n cmd = \"%s %s %s simi.txt 0.88 10 2 10\" % (bin.replace(\"posmo\",\"clusterwd\"), context_file, outfile)\n p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) \n out, err = p.communicate()\n stdout += out.decode()\n stderr += err.decode()\n \n if os.path.exists(outfile):\n with open(outfile) as f:\n motifs += self.parse(f, width, n_ones)\n \n os.chdir(current_path)\n \n return motifs, stdout, stderr", "response": "Run Posmo and predict motifs from a FASTA file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert Posmo output to motifs AttributeNames", "response": "def parse(self, fo, width, seed=None):\n \"\"\"\n Convert Posmo output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing Posmo output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n\n lines = [fo.readline() for x in range(6)]\n while lines[0]:\n matrix = [[float(x) for x in line.strip().split(\"\\t\")] for line in lines[2:]]\n matrix = [[matrix[x][y] for x in range(4)] for y in range(len(matrix[0]))]\n m = Motif(matrix)\n m.trim(0.1)\n m.id = lines[0].strip().split(\" \")[-1]\n motifs.append(m)\n lines = [fo.readline() for x in range(6)]\n \n for i,motif in enumerate(motifs):\n if seed:\n motif.id = \"%s_w%s.%s_%s\" % (self.name, width, seed, i + 1)\n else:\n motif.id = \"%s_w%s_%s\" % (self.name, width, i + 1)\n motif.trim(0.25)\n \n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert GADEM output to motifs Arcs.", "response": "def parse(self, fo):\n \"\"\"\n Convert GADEM output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing GADEM output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n nucs = {\"A\":0,\"C\":1,\"G\":2,\"T\":3}\n\n lines = fo.readlines()\n for i in range(0, len(lines), 5):\n align = []\n pwm = []\n pfm = []\n m_id = \"\"\n line = lines[i].strip()\n m_id = line[1:]\n number = m_id.split(\"_\")[0][1:]\n if os.path.exists(\"%s.seq\" % number):\n with open(\"%s.seq\" % number) as f:\n for l in f:\n if \"x\" not in l and \"n\" not in l:\n l = l.strip().upper()\n align.append(l)\n if not pfm:\n pfm = [[0 for x in range(4)] for x in range(len(l))]\n for p in range(len(l)):\n pfm[p][nucs[l[p]]] += 1\n \n m = [l.strip().split(\" \")[1].split(\"\\t\") for l in lines[i + 1: i + 5]]\n\n pwm = [[float(m[x][y]) for x in range(4)] for y in range(len(m[0]))]\n\n\n motifs.append(Motif(pwm))\n motifs[-1].id = \"{}_{}\".format(self.name, m_id)\n #motifs[-1].pwm = pwm\n if align:\n motifs[-1].pfm = pfm\n motifs[-1].align = align\n\n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _run_program(self, bin, fastafile, params=None):\n fname = os.path.join(self.config.get_motif_dir(), \"JASPAR2010_vertebrate.pwm\")\n motifs = read_motifs(fname, fmt=\"pwm\")\n\n for motif in motifs:\n motif.id = \"JASPAR_%s\" % motif.id\n return motifs, \"\", \"\"", "response": "This function runs the JASPAR2010_vertebrate tool and returns a list of Motif instances and a string containing the output of the tool."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun MEME and predict motifs from a FASTA file.", "response": "def _run_program(self, bin, fastafile, params=None):\n \"\"\"\n Run MEME and predict motifs from a FASTA file.\n\n Parameters\n ----------\n bin : str\n Command used to run the tool.\n \n fastafile : str\n Name of the FASTA input file.\n\n params : dict, optional\n Optional parameters. For some of the tools required parameters\n are passed using this dictionary.\n\n Returns\n -------\n motifs : list of Motif instances\n The predicted motifs.\n\n stdout : str\n Standard out of the tool.\n \n stderr : str\n Standard error of the tool.\n \"\"\"\n default_params = {\"width\":10, \"single\":False, \"number\":10}\n if params is not None: \n default_params.update(params)\n \n tmp = NamedTemporaryFile(dir=self.tmpdir)\n tmpname = tmp.name\n \n strand = \"-revcomp\"\n width = default_params[\"width\"]\n number = default_params[\"number\"]\n \n cmd = [bin, fastafile, \"-text\",\"-dna\",\"-nostatus\",\"-mod\", \"zoops\",\"-nmotifs\", \"%s\" % number, \"-w\",\"%s\" % width, \"-maxsize\", \"10000000\"]\n if not default_params[\"single\"]:\n cmd.append(strand)\n \n #sys.stderr.write(\" \".join(cmd) + \"\\n\")\n p = Popen(cmd, bufsize=1, stderr=PIPE, stdout=PIPE) \n stdout,stderr = p.communicate()\n\n motifs = []\n motifs = self.parse(io.StringIO(stdout.decode()))\n \n # Delete temporary files\n tmp.close()\n \n return motifs, stdout, stderr"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert MEME output to motifs Arcs.", "response": "def parse(self, fo):\n \"\"\"\n Convert MEME output to motifs\n \n Parameters\n ----------\n fo : file-like\n File object containing MEME output.\n\n Returns\n -------\n motifs : list\n List of Motif instances.\n \"\"\"\n motifs = []\n nucs = {\"A\":0,\"C\":1,\"G\":2,\"T\":3}\n\n p = re.compile('MOTIF.+MEME-(\\d+)\\s*width\\s*=\\s*(\\d+)\\s+sites\\s*=\\s*(\\d+)')\n pa = re.compile('\\)\\s+([A-Z]+)')\n line = fo.readline()\n while line:\n m = p.search(line)\n align = []\n pfm = None \n if m:\n #print(m.group(0))\n id = \"%s_%s_w%s\" % (self.name, m.group(1), m.group(2))\n while not line.startswith(\"//\"):\n ma = pa.search(line)\n if ma:\n #print(ma.group(0))\n l = ma.group(1)\n align.append(l)\n if not pfm:\n pfm = [[0 for x in range(4)] for x in range(len(l))]\n for pos in range(len(l)):\n if l[pos] in nucs:\n pfm[pos][nucs[l[pos]]] += 1\n else:\n for i in range(4):\n pfm[pos][i] += 0.25\n \n line = fo.readline()\n \n motifs.append(Motif(pfm[:]))\n motifs[-1].id = id\n motifs[-1].align = align[:]\n line = fo.readline()\n\n return motifs"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nscan regions in input table with motifs.", "response": "def scan_to_table(input_table, genome, scoring, pwmfile=None, ncpus=None):\n \"\"\"Scan regions in input table with motifs.\n\n Parameters\n ----------\n input_table : str\n Filename of input table. Can be either a text-separated tab file or a\n feather file.\n \n genome : str\n Genome name. Can be either the name of a FASTA-formatted file or a \n genomepy genome name.\n \n scoring : str\n \"count\" or \"score\"\n \n pwmfile : str, optional\n Specify a PFM file for scanning.\n \n ncpus : int, optional\n If defined this specifies the number of cores to use.\n \n Returns\n -------\n table : pandas.DataFrame\n DataFrame with motif ids as column names and regions as index. Values\n are either counts or scores depending on the 'scoring' parameter.s\n \"\"\"\n config = MotifConfig()\n \n if pwmfile is None:\n pwmfile = config.get_default_params().get(\"motif_db\", None)\n if pwmfile is not None:\n pwmfile = os.path.join(config.get_motif_dir(), pwmfile)\n\n if pwmfile is None:\n raise ValueError(\"no pwmfile given and no default database specified\")\n\n logger.info(\"reading table\")\n if input_table.endswith(\"feather\"):\n df = pd.read_feather(input_table)\n idx = df.iloc[:,0].values\n else:\n df = pd.read_table(input_table, index_col=0, comment=\"#\")\n idx = df.index\n \n regions = list(idx)\n s = Scanner(ncpus=ncpus)\n s.set_motifs(pwmfile)\n s.set_genome(genome)\n s.set_background(genome=genome)\n \n nregions = len(regions)\n\n scores = []\n if scoring == \"count\":\n logger.info(\"setting threshold\")\n s.set_threshold(fpr=FPR)\n logger.info(\"creating count table\")\n for row in s.count(regions):\n scores.append(row)\n logger.info(\"done\")\n else:\n s.set_threshold(threshold=0.0)\n logger.info(\"creating score table\")\n for row in s.best_score(regions, normalize=True):\n scores.append(row)\n logger.info(\"done\")\n \n motif_names = [m.id for m in read_motifs(pwmfile)]\n logger.info(\"creating dataframe\")\n return pd.DataFrame(scores, index=idx, columns=motif_names)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrun maelstrom on a single input table.", "response": "def run_maelstrom(infile, genome, outdir, pwmfile=None, plot=True, cluster=False, \n score_table=None, count_table=None, methods=None, ncpus=None):\n \"\"\"Run maelstrom on an input table.\n \n Parameters\n ----------\n infile : str\n Filename of input table. Can be either a text-separated tab file or a\n feather file.\n \n genome : str\n Genome name. Can be either the name of a FASTA-formatted file or a \n genomepy genome name.\n \n outdir : str\n Output directory for all results.\n\n pwmfile : str, optional\n Specify a PFM file for scanning.\n\n plot : bool, optional\n Create heatmaps.\n \n cluster : bool, optional\n If True and if the input table has more than one column, the data is\n clustered and the cluster activity methods are also run. Not \n well-tested.\n \n score_table : str, optional\n Filename of pre-calculated table with motif scores.\n\n count_table : str, optional\n Filename of pre-calculated table with motif counts.\n\n methods : list, optional\n Activity methods to use. By default are all used.\n\n ncpus : int, optional\n If defined this specifies the number of cores to use.\n \"\"\"\n logger.info(\"Starting maelstrom\")\n if infile.endswith(\"feather\"):\n df = pd.read_feather(infile)\n df = df.set_index(df.columns[0])\n else:\n df = pd.read_table(infile, index_col=0, comment=\"#\")\n \n # Check for duplicates\n if df.index.duplicated(keep=False).any():\n raise ValueError(\"Input file contains duplicate regions! \"\n \"Please remove them.\")\n \n if not os.path.exists(outdir):\n os.mkdir(outdir)\n\n if methods is None:\n methods = Moap.list_predictors() \n methods = [m.lower() for m in methods]\n\n shutil.copyfile(infile, os.path.join(outdir, \"input.table.txt\"))\n \n # Copy the motif informatuon\n pwmfile = pwmfile_location(pwmfile) \n if pwmfile:\n shutil.copy2(pwmfile, outdir)\n mapfile = re.sub(\".p[fw]m$\", \".motif2factors.txt\", pwmfile)\n if os.path.exists(mapfile):\n shutil.copy2(mapfile, outdir)\n \n # Create a file with the number of motif matches\n if not count_table:\n count_table = os.path.join(outdir, \"motif.count.txt.gz\")\n if not os.path.exists(count_table):\n logger.info(\"Motif scanning (counts)\")\n counts = scan_to_table(infile, genome, \"count\",\n pwmfile=pwmfile, ncpus=ncpus)\n counts.to_csv(count_table, sep=\"\\t\", compression=\"gzip\")\n else:\n logger.info(\"Counts, using: %s\", count_table)\n\n # Create a file with the score of the best motif match\n if not score_table:\n score_table = os.path.join(outdir, \"motif.score.txt.gz\")\n if not os.path.exists(score_table):\n logger.info(\"Motif scanning (scores)\")\n scores = scan_to_table(infile, genome, \"score\",\n pwmfile=pwmfile, ncpus=ncpus)\n scores.to_csv(score_table, sep=\"\\t\", float_format=\"%.3f\", \n compression=\"gzip\")\n else:\n logger.info(\"Scores, using: %s\", score_table)\n\n if cluster:\n cluster = False\n for method in methods:\n m = Moap.create(method, ncpus=ncpus)\n if m.ptype == \"classification\":\n cluster = True\n break\n if not cluster:\n logger.info(\"Skipping clustering, no classification methods\")\n \n exps = []\n clusterfile = infile\n if df.shape[1] != 1:\n # More than one column\n for method in Moap.list_regression_predictors():\n if method in methods:\n m = Moap.create(method, ncpus=ncpus)\n exps.append([method, m.pref_table, infile])\n logger.debug(\"Adding %s\", method)\n\n if cluster:\n clusterfile = os.path.join(outdir,\n os.path.basename(infile) + \".cluster.txt\")\n \n df[:] = scale(df, axis=0)\n names = df.columns\n df_changed = pd.DataFrame(index=df.index)\n df_changed[\"cluster\"] = np.nan\n for name in names:\n df_changed.loc[(df[name] - df.loc[:,df.columns != name].max(1)) > 0.5, \"cluster\"] = name\n df_changed.dropna().to_csv(clusterfile, sep=\"\\t\")\n if df.shape[1] == 1 or cluster:\n for method in Moap.list_classification_predictors():\n if method in methods:\n m = Moap.create(method, ncpus=ncpus)\n exps.append([method, m.pref_table, clusterfile])\n\n if len(exps) == 0:\n logger.error(\"No method to run.\")\n sys.exit(1)\n\n for method, scoring, fname in exps:\n try:\n if scoring == \"count\" and count_table:\n moap_with_table(fname, count_table, outdir, method, scoring, ncpus=ncpus)\n elif scoring == \"score\" and score_table:\n moap_with_table(fname, score_table, outdir, method, scoring, ncpus=ncpus)\n else:\n moap_with_bg(fname, genome, outdir, method, scoring, pwmfile=pwmfile, ncpus=ncpus)\n \n except Exception as e:\n logger.warn(\"Method %s with scoring %s failed\", method, scoring)\n logger.warn(e)\n logger.warn(\"Skipping\")\n raise \n dfs = {}\n for method, scoring,fname in exps:\n t = \"{}.{}\".format(method,scoring)\n fname = os.path.join(outdir, \"activity.{}.{}.out.txt\".format(\n method, scoring))\n try:\n dfs[t] = pd.read_table(fname, index_col=0, comment=\"#\")\n except:\n logging.warn(\"Activity file for {} not found!\\n\".format(t))\n \n if len(methods) > 1:\n logger.info(\"Rank aggregation\")\n df_p = df_rank_aggregation(df, dfs, exps)\n df_p.to_csv(os.path.join(outdir, \"final.out.csv\"), sep=\"\\t\")\n #df_p = df_p.join(m2f)\n\n # Write motif frequency table\n \n if df.shape[1] == 1:\n mcount = df.join(pd.read_table(count_table, index_col=0, comment=\"#\"))\n m_group = mcount.groupby(df.columns[0])\n freq = m_group.sum() / m_group.count()\n freq.to_csv(os.path.join(outdir, \"motif.freq.txt\"), sep=\"\\t\")\n\n if plot and len(methods) > 1:\n logger.info(\"html report\")\n maelstrom_html_report(\n outdir, \n os.path.join(outdir, \"final.out.csv\"),\n pwmfile\n )\n logger.info(os.path.join(outdir, \"gimme.maelstrom.report.html\"))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_heatmap(self, kind=\"final\", min_freq=0.01, threshold=2, name=True, max_len=50, aspect=1, **kwargs):\n \n filt = np.any(np.abs(self.result) >= threshold, 1) & np.any(np.abs(self.freq.T) >= min_freq, 1)\n \n idx = self.result[filt].index\n \n cmap = \"RdBu_r\" \n if kind == \"final\":\n data = self.result\n elif kind == \"freq\":\n data = self.freq.T\n cmap = \"Reds\"\n elif kind in self.activity:\n data = self.activity[dtype]\n if kind in [\"hypergeom.count\", \"mwu.score\"]:\n cmap = \"Reds\"\n else:\n raise ValueError(\"Unknown dtype\")\n \n #print(data.head())\n #plt.figure(\n m = data.loc[idx]\n if name:\n m[\"factors\"] = [join_max(self.motifs[n].factors, max_len, \",\", suffix=\",(...)\") for n in m.index]\n m = m.set_index(\"factors\")\n h,w = m.shape\n cg = sns.clustermap(m, cmap=cmap, col_cluster=False, \n figsize=(2 + w * 0.5 * aspect, 0.5 * h), linewidths=1,\n **kwargs)\n cg.ax_col_dendrogram.set_visible(False)\n plt.setp(cg.ax_heatmap.yaxis.get_majorticklabels(), rotation=0);\n return cg", "response": "Plot the clustered heatmap of predicted motif activity."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating motif scores boxplot of different clusters.", "response": "def plot_scores(self, motifs, name=True, max_len=50):\n \"\"\"Create motif scores boxplot of different clusters.\n Motifs can be specified as either motif or factor names.\n The motif scores will be scaled and plotted as z-scores.\n \n Parameters\n ----------\n motifs : iterable or str\n List of motif or factor names.\n \n name : bool, optional\n Use factor names instead of motif names for plotting.\n \n max_len : int, optional\n Truncate the list of factors to this maximum length.\n \n Returns\n -------\n \n g : FacetGrid\n Returns the seaborn FacetGrid object with the plot.\n \"\"\"\n if self.input.shape[1] != 1:\n raise ValueError(\"Can't make a categorical plot with real-valued data\")\n \n if type(\"\") == type(motifs):\n motifs = [motifs]\n \n plot_motifs = []\n for motif in motifs:\n if motif in self.motifs:\n plot_motifs.append(motif)\n else:\n for m in self.motifs.values():\n if motif in m.factors:\n plot_motifs.append(m.id)\n \n data = self.scores[plot_motifs]\n data[:] = data.scale(data, axix=0)\n if name:\n data = data.T\n data[\"factors\"] = [join_max(self.motifs[n].factors, max_len, \",\", suffix=\",(...)\") for n in plot_motifs]\n data = data.set_index(\"factors\").T\n \n data = pd.melt(self.input.join(data), id_vars=[\"cluster\"])\n data.columns = [\"cluster\", \"motif\", \"z-score\"]\n g = sns.factorplot(data=data, y=\"motif\", x=\"z-score\", hue=\"cluster\", kind=\"box\", aspect=2)\n return g"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn version of package on pypi. python. org using json. Adapted from https://stackoverflow. com/a/34366589", "response": "def get_version(package, url_pattern=URL_PATTERN):\n \"\"\"Return version of package on pypi.python.org using json. Adapted from https://stackoverflow.com/a/34366589\"\"\"\n req = requests.get(url_pattern.format(package=package))\n version = parse('0')\n if req.status_code == requests.codes.ok:\n # j = json.loads(req.text.encode(req.encoding))\n j = req.json()\n releases = j.get('releases', [])\n for release in releases:\n ver = parse(release)\n if not ver.is_prerelease:\n version = max(version, ver)\n return version"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert arguments extracted from a parser to a dict and will dismiss arguments which default to NotSet.", "response": "def get_args(parser):\n \"\"\"\n Converts arguments extracted from a parser to a dict,\n and will dismiss arguments which default to NOT_SET.\n\n :param parser: an ``argparse.ArgumentParser`` instance.\n :type parser: argparse.ArgumentParser\n :return: Dictionary with the configs found in the parsed CLI arguments.\n :rtype: dict\n \"\"\"\n args = vars(parser.parse_args()).items()\n return {key: val for key, val in args if not isinstance(val, NotSet)}"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef parse_value(val, parsebool=False):\n try:\n return int(val)\n except ValueError:\n pass\n try:\n return float(val)\n except:\n pass\n if parsebool:\n if re.match('yes|on', str(val), re.IGNORECASE):\n return True\n elif re.match('no|off', str(val), re.IGNORECASE):\n return False\n return val", "response": "Parse input string and return int float or str depending on format."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbuffering read from socket. Reads all data available from buffer.", "response": "def socket_read(fp):\n \"\"\"Buffered read from socket. Reads all data available from socket.\n \n @fp: File pointer for socket.\n @return: String of characters read from buffer.\n \n \"\"\"\n response = ''\n oldlen = 0\n newlen = 0\n while True:\n response += fp.read(buffSize)\n newlen = len(response)\n if newlen - oldlen == 0:\n break\n else:\n oldlen = newlen\n return response"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the value of nested items in the archive.", "response": "def set_nested(self, klist, value):\n \"\"\"D.set_nested((k1, k2,k3, ...), v) -> D[k1][k2][k3] ... = v\"\"\"\n keys = list(klist)\n if len(keys) > 0:\n curr_dict = self\n last_key = keys.pop()\n for key in keys:\n if not curr_dict.has_key(key) or not isinstance(curr_dict[key], \n NestedDict):\n curr_dict[key] = type(self)()\n curr_dict = curr_dict[key]\n curr_dict[last_key] = value"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef registerFilter(self, column, patterns, is_regex=False, \n ignore_case=False):\n \"\"\"Register filter on a column of table.\n \n @param column: The column name.\n @param patterns: A single pattern or a list of patterns used for \n matching column values.\n @param is_regex: The patterns will be treated as regex if True, the \n column values will be tested for equality with the\n patterns otherwise.\n @param ignore_case: Case insensitive matching will be used if True.\n \n \"\"\"\n if isinstance(patterns, basestring):\n patt_list = (patterns,)\n elif isinstance(patterns, (tuple, list)):\n patt_list = list(patterns)\n else:\n raise ValueError(\"The patterns parameter must either be as string \"\n \"or a tuple / list of strings.\")\n if is_regex:\n if ignore_case:\n flags = re.IGNORECASE\n else:\n flags = 0\n patt_exprs = [re.compile(pattern, flags) for pattern in patt_list]\n else:\n if ignore_case:\n patt_exprs = [pattern.lower() for pattern in patt_list]\n else:\n patt_exprs = patt_list\n self._filters[column] = (patt_exprs, is_regex, ignore_case)", "response": "Register a filter on a column of table."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef unregisterFilter(self, column):\n if self._filters.has_key(column):\n del self._filters[column]", "response": "Unregister a filter on a column of the table."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef registerFilters(self, **kwargs):\n for (key, patterns) in kwargs.items():\n if key.endswith('_regex'):\n col = key[:-len('_regex')]\n is_regex = True\n else:\n col = key\n is_regex = False\n if col.endswith('_ic'):\n col = col[:-len('_ic')]\n ignore_case = True\n else:\n ignore_case = False\n self.registerFilter(col, patterns, is_regex, ignore_case)", "response": "Register multiple filters at once."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef applyFilters(self, headers, table):\n result = []\n column_idxs = {}\n for column in self._filters.keys():\n try:\n column_idxs[column] = headers.index(column)\n except ValueError:\n raise ValueError('Invalid column name %s in filter.' % column)\n for row in table:\n for (column, (patterns, \n is_regex, \n ignore_case)) in self._filters.items():\n col_idx = column_idxs[column]\n col_val = row[col_idx]\n if is_regex:\n for pattern in patterns:\n if pattern.search(col_val):\n break\n else:\n break\n else:\n if ignore_case:\n col_val = col_val.lower()\n if col_val in patterns:\n pass\n else:\n break\n else:\n result.append(row)\n return result", "response": "Apply filters on ps command result."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef open(self, host=None, port=0, socket_file=None, \n timeout=socket.getdefaulttimeout()):\n \"\"\"Connect to a host.\n\n With a host argument, it connects the instance using TCP; port number \n and timeout are optional, socket_file must be None. The port number\n defaults to the standard telnet port (23).\n \n With a socket_file argument, it connects the instance using\n named socket; timeout is optional and host must be None.\n\n Don't try to reopen an already connected instance.\n \n \"\"\"\n self.socket_file = socket_file\n if host is not None:\n if sys.version_info[:2] >= (2,6):\n telnetlib.Telnet.open(self, host, port, timeout)\n else:\n telnetlib.Telnet.open(self, host, port)\n elif socket_file is not None:\n self.eof = 0\n self.host = host\n self.port = port\n self.timeout = timeout\n self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n self.sock.settimeout(timeout)\n self.sock.connect(socket_file)\n else:\n raise TypeError(\"Either host or socket_file argument is required.\")", "response": "Open an instance of a Telnet instance."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef analyze(egg, subjgroup=None, listgroup=None, subjname='Subject',\n listname='List', analysis=None, position=0, permute=False,\n n_perms=1000, parallel=False, match='exact',\n distance='euclidean', features=None, ts=None):\n \"\"\"\n General analysis function that groups data by subject/list number and performs analysis.\n\n Parameters\n ----------\n egg : Egg data object\n The data to be analyzed\n\n subjgroup : list of strings or ints\n String/int variables indicating how to group over subjects. Must be\n the length of the number of subjects\n\n subjname : string\n Name of the subject grouping variable\n\n listgroup : list of strings or ints\n String/int variables indicating how to group over list. Must be\n the length of the number of lists\n\n listname : string\n Name of the list grouping variable\n\n analysis : string\n This is the analysis you want to run. Can be accuracy, spc, pfr,\n temporal or fingerprint\n\n position : int\n Optional argument for pnr analysis. Defines encoding position of item\n to run pnr. Default is 0, and it is zero indexed\n\n permute : bool\n Optional argument for fingerprint/temporal cluster analyses. Determines\n whether to correct clustering scores by shuffling recall order for each list\n to create a distribution of clustering scores (for each feature). The\n \"corrected\" clustering score is the proportion of clustering scores in\n that random distribution that were lower than the clustering score for\n the observed recall sequence. Default is False.\n\n n_perms : int\n Optional argument for fingerprint/temporal cluster analyses. Number of\n permutations to run for \"corrected\" clustering scores. Default is 1000 (\n per recall list).\n\n parallel : bool\n Option to use multiprocessing (this can help speed up the permutations\n tests in the clustering calculations)\n\n match : str (exact, best or smooth)\n Matching approach to compute recall matrix. If exact, the presented and\n recalled items must be identical (default). If best, the recalled item\n that is most similar to the presented items will be selected. If smooth,\n a weighted average of all presented items will be used, where the\n weights are derived from the similarity between the recalled item and\n each presented item.\n\n distance : str\n The distance function used to compare presented and recalled items.\n Applies only to 'best' and 'smooth' matching approaches. Can be any\n distance function supported by numpy.spatial.distance.cdist.\n\n\n Returns\n ----------\n result : quail.FriedEgg\n Class instance containing the analysis results\n\n \"\"\"\n if analysis is None:\n raise ValueError('You must pass an analysis type.')\n\n if analysis not in analyses.keys():\n raise ValueError('Analysis not recognized. Choose one of the following: '\n 'accuracy, spc, pfr, lag-crp, fingerprint, temporal')\n\n from ..egg import FriedEgg\n\n if hasattr(egg, 'subjgroup'):\n if egg.subjgroup is not None:\n subjgroup = egg.subjgroup\n\n if hasattr(egg, 'subjname'):\n if egg.subjname is not None:\n subjname = egg.subjname\n\n if hasattr(egg, 'listgroup'):\n if egg.listgroup is not None:\n listgroup = egg.listgroup\n\n if hasattr(egg, 'listname'):\n if egg.listname is not None:\n listname = egg.listname\n\n if features is None:\n features = egg.feature_names\n\n opts = {\n 'subjgroup' : subjgroup,\n 'listgroup' : listgroup,\n 'subjname' : subjname,\n 'parallel' : parallel,\n 'match' : match,\n 'distance' : distance,\n 'features' : features,\n 'analysis_type' : analysis,\n 'analysis' : analyses[analysis]\n }\n\n if analysis is 'pfr':\n opts.update({'position' : 0})\n elif analysis is 'pnr':\n opts.update({'position' : position})\n if analysis is 'temporal':\n opts.update({'features' : ['temporal']})\n if analysis in ['temporal', 'fingerprint']:\n opts.update({'permute' : permute, 'n_perms' : n_perms})\n if analysis is 'lagcrp':\n opts.update({'ts' : ts})\n\n return FriedEgg(data=_analyze_chunk(egg, **opts), analysis=analysis,\n list_length=egg.list_length, n_lists=egg.n_lists,\n n_subjects=egg.n_subjects, position=position)", "response": "Analyze the data of a specific object and returns a tree of items that are part of the tree."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _analyze_chunk(data, subjgroup=None, subjname='Subject', listgroup=None,\n listname='List', analysis=None, analysis_type=None,\n pass_features=False, features=None, parallel=False,\n **kwargs):\n \"\"\"\n Private function that groups data by subject/list number and performs\n analysis for a chunk of data.\n\n Parameters\n ----------\n data : Egg data object\n The data to be analyzed\n\n subjgroup : list of strings or ints\n String/int variables indicating how to group over subjects. Must be\n the length of the number of subjects\n\n subjname : string\n Name of the subject grouping variable\n\n listgroup : list of strings or ints\n String/int variables indicating how to group over list. Must be\n the length of the number of lists\n\n listname : string\n Name of the list grouping variable\n\n analysis : function\n This function analyzes data and returns it.\n\n pass_features : bool\n Logical indicating whether the analyses uses the features field of the\n Egg\n\n Returns\n ----------\n analyzed_data : Pandas DataFrame\n DataFrame containing the analysis results\n\n \"\"\"\n\n # perform the analysis\n def _analysis(c):\n subj, lst = c\n subjects = [s for s in subjdict[subj]]\n lists = [l for l in listdict[subj][lst]]\n s = data.crack(lists=lists, subjects=subjects)\n index = pd.MultiIndex.from_arrays([[subj],[lst]], names=[subjname, listname])\n opts = dict()\n if analysis_type is 'fingerprint':\n opts.update({'columns' : features})\n elif analysis_type is 'lagcrp':\n if kwargs['ts']:\n opts.update({'columns' : range(-kwargs['ts'],kwargs['ts']+1)})\n else:\n opts.update({'columns' : range(-data.list_length,data.list_length+1)})\n return pd.DataFrame([analysis(s, features=features, **kwargs)],\n index=index, **opts)\n\n subjgroup = subjgroup if subjgroup else data.pres.index.levels[0].values\n listgroup = listgroup if listgroup else data.pres.index.levels[1].values\n\n subjdict = {subj : data.pres.index.levels[0].values[subj==np.array(subjgroup)] for subj in set(subjgroup)}\n\n if all(isinstance(el, list) for el in listgroup):\n listdict = [{lst : data.pres.index.levels[1].values[lst==np.array(listgrpsub)] for lst in set(listgrpsub)} for listgrpsub in listgroup]\n else:\n listdict = [{lst : data.pres.index.levels[1].values[lst==np.array(listgroup)] for lst in set(listgroup)} for subj in subjdict]\n\n chunks = [(subj, lst) for subj in subjdict for lst in listdict[0]]\n\n if parallel:\n import multiprocessing\n from pathos.multiprocessing import ProcessingPool as Pool\n p = Pool(multiprocessing.cpu_count())\n res = p.map(_analysis, chunks)\n else:\n res = [_analysis(c) for c in chunks]\n\n return pd.concat(res)", "response": "Private function that performs the analysis of a single chunk of data."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n if self.hasGraph('tomcat_memory'):\n stats = self._tomcatInfo.getMemoryStats()\n self.setGraphVal('tomcat_memory', 'used', \n stats['total'] - stats['free'])\n self.setGraphVal('tomcat_memory', 'free', stats['free'])\n self.setGraphVal('tomcat_memory', 'max', stats['max'])\n for (port, stats) in self._tomcatInfo.getConnectorStats().iteritems():\n thrstats = stats['threadInfo']\n reqstats = stats['requestInfo']\n if self.portIncluded(port):\n name = \"tomcat_threads_%d\" % port\n if self.hasGraph(name):\n self.setGraphVal(name, 'busy', \n thrstats['currentThreadsBusy'])\n self.setGraphVal(name, 'idle', \n thrstats['currentThreadCount'] \n - thrstats['currentThreadsBusy'])\n self.setGraphVal(name, 'max', thrstats['maxThreads'])\n name = \"tomcat_access_%d\" % port\n if self.hasGraph(name):\n self.setGraphVal(name, 'reqs', reqstats['requestCount'])\n name = \"tomcat_error_%d\" % port\n if self.hasGraph(name):\n self.setGraphVal(name, 'errors', reqstats['errorCount'])\n name = \"tomcat_traffic_%d\" % port\n if self.hasGraph(name):\n self.setGraphVal(name, 'rx', reqstats['bytesReceived'])\n self.setGraphVal(name, 'tx', reqstats['bytesSent'])"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate the autocorrelation function for a 1D time series.", "response": "def function(data, maxt=None):\n \"\"\"\n Calculate the autocorrelation function for a 1D time series.\n\n Parameters\n ----------\n data : numpy.ndarray (N,)\n The time series.\n\n Returns\n -------\n rho : numpy.ndarray (N,)\n An autocorrelation function.\n\n \"\"\"\n data = np.atleast_1d(data)\n assert len(np.shape(data)) == 1, \\\n \"The autocorrelation function can only by computed \" \\\n + \"on a 1D time series.\"\n if maxt is None:\n maxt = len(data)\n result = np.zeros(maxt, dtype=float)\n _acor.function(np.array(data, dtype=float), result)\n return result / result[0]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef retrieveVals(self):\n nginxInfo = NginxInfo(self._host, self._port,\n self._user, self._password, \n self._statuspath, self._ssl)\n stats = nginxInfo.getServerStats()\n if stats:\n if self.hasGraph('nginx_activeconn'):\n self.setGraphVal('nginx_activeconn', 'proc', stats['writing'])\n self.setGraphVal('nginx_activeconn', 'read', stats['reading'])\n self.setGraphVal('nginx_activeconn', 'wait', stats['waiting'])\n self.setGraphVal('nginx_activeconn', 'total', \n stats['connections'])\n if self.hasGraph('nginx_connections'):\n self.setGraphVal('nginx_connections', 'handled', stats['handled'])\n self.setGraphVal('nginx_connections', 'nothandled', \n stats['accepts'] - stats['handled'])\n if self.hasGraph('nginx_requests'):\n self.setGraphVal('nginx_requests', 'requests', stats['requests'])\n if self.hasGraph('nginx_requestsperconn'):\n curr_stats = (stats['handled'], stats['requests'])\n hist_stats = self.restoreState()\n if hist_stats:\n prev_stats = hist_stats[0]\n else:\n hist_stats = []\n prev_stats = (0,0)\n conns = max(curr_stats[0] - prev_stats[0], 0)\n reqs = max(curr_stats[1] - prev_stats[1], 0)\n if conns > 0:\n self.setGraphVal('nginx_requestsperconn', 'requests',\n float(reqs) / float(conns))\n else:\n self.setGraphVal('nginx_requestsperconn', 'requests', 0)\n hist_stats.append(curr_stats)\n self.saveState(hist_stats[-self._numSamples:])", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nimplement Munin Plugin Auto - Configuration Option.", "response": "def autoconf(self):\n \"\"\"Implements Munin Plugin Auto-Configuration Option.\n \n @return: True if plugin can be auto-configured, False otherwise.\n \n \"\"\"\n nginxInfo = NginxInfo(self._host, self._port,\n self._user, self._password, \n self._statuspath, self._ssl)\n return nginxInfo is not None"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getStats(self):\n url = \"%s://%s:%d/%s\" % (self._proto, self._host, self._port, \n self._monpath)\n response = util.get_url(url, self._user, self._password)\n stats = {}\n for line in response.splitlines():\n mobj = re.match('([\\w\\s]+):\\s+(\\w+)$', line)\n if mobj:\n stats[mobj.group(1)] = util.parse_value(mobj.group(2))\n return stats", "response": "Query and parse Web Server Status Page.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef retrieveVals(self):\n apcinfo = APCinfo(self._host, self._port, self._user, self._password, \n self._monpath, self._ssl, self._extras)\n stats = apcinfo.getAllStats()\n \n if self.hasGraph('php_apc_memory') and stats:\n filecache = stats['cache_sys']['mem_size']\n usercache = stats['cache_user']['mem_size']\n total = stats['memory']['seg_size'] * stats['memory']['num_seg']\n free = stats['memory']['avail_mem']\n other = total - free - filecache - usercache \n self.setGraphVal('php_apc_memory', 'filecache', filecache)\n self.setGraphVal('php_apc_memory', 'usercache', usercache)\n self.setGraphVal('php_apc_memory', 'other', other)\n self.setGraphVal('php_apc_memory', 'free', free)\n if self.hasGraph('php_apc_items') and stats:\n self.setGraphVal('php_apc_items', 'filecache', \n stats['cache_sys']['num_entries'])\n self.setGraphVal('php_apc_items', 'usercache', \n stats['cache_user']['num_entries'])\n if self.hasGraph('php_apc_reqs_filecache') and stats:\n self.setGraphVal('php_apc_reqs_filecache', 'hits', \n stats['cache_sys']['num_hits'])\n self.setGraphVal('php_apc_reqs_filecache', 'misses', \n stats['cache_sys']['num_misses'])\n self.setGraphVal('php_apc_reqs_filecache', 'inserts', \n stats['cache_sys']['num_inserts'])\n if self.hasGraph('php_apc_reqs_usercache') and stats:\n self.setGraphVal('php_apc_reqs_usercache', 'hits', \n stats['cache_user']['num_hits'])\n self.setGraphVal('php_apc_reqs_usercache', 'misses', \n stats['cache_user']['num_misses'])\n self.setGraphVal('php_apc_reqs_usercache', 'inserts', \n stats['cache_user']['num_inserts'])\n if self.hasGraph('php_apc_expunge') and stats:\n self.setGraphVal('php_apc_expunge', 'filecache', \n stats['cache_sys']['expunges'])\n self.setGraphVal('php_apc_expunge', 'usercache', \n stats['cache_user']['expunges'])\n if self.hasGraph('php_apc_mem_util_frag'):\n self.setGraphVal('php_apc_mem_util_frag', 'util', \n stats['memory']['utilization_ratio'] * 100)\n self.setGraphVal('php_apc_mem_util_frag', 'frag', \n stats['memory']['fragmentation_ratio'] * 100)\n if self.hasGraph('php_apc_mem_frag_count'):\n self.setGraphVal('php_apc_mem_frag_count', 'num',\n stats['memory']['fragment_count'])\n if self.hasGraph('php_apc_mem_frag_avgsize'):\n self.setGraphVal('php_apc_mem_frag_avgsize', 'size',\n stats['memory']['fragment_avg_size'])", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nimplement Munin Plugin Auto - Configuration Option.", "response": "def autoconf(self):\n \"\"\"Implements Munin Plugin Auto-Configuration Option.\n \n @return: True if plugin can be auto-configured, False otherwise.\n \n \"\"\"\n apcinfo = APCinfo(self._host, self._port, self._user, self._password, \n self._monpath, self._ssl)\n return apcinfo is not None"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrun varnishstats command to get stats from Varnish Cache.", "response": "def getStats(self):\n \"\"\"Runs varnishstats command to get stats from Varnish Cache.\n \n @return: Dictionary of stats.\n\n \"\"\"\n info_dict = {}\n args = [varnishstatCmd, '-1']\n if self._instance is not None:\n args.extend(['-n', self._instance])\n output = util.exec_command(args)\n if self._descDict is None:\n self._descDict = {}\n for line in output.splitlines():\n mobj = re.match('(\\S+)\\s+(\\d+)\\s+(\\d+\\.\\d+|\\.)\\s+(\\S.*\\S)\\s*$', \n line)\n if mobj:\n fname = mobj.group(1).replace('.', '_')\n info_dict[fname] = util.parse_value(mobj.group(2))\n self._descDict[fname] = mobj.group(4)\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning description for entry.", "response": "def getDesc(self, entry):\n \"\"\"Returns description for stat entry.\n \n @param entry: Entry name.\n @return: Description for entry.\n \n \"\"\"\n if len(self._descDict) == 0:\n self.getStats()\n return self._descDict.get(entry)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef retrieveVals(self):\n opcinfo = OPCinfo(self._host, self._port, self._user, self._password, \n self._monpath, self._ssl)\n stats = opcinfo.getAllStats()\n \n if self.hasGraph('php_opc_memory') and stats:\n mem = stats['memory_usage']\n keys = ('used_memory', 'wasted_memory', 'free_memory')\n map(lambda k:self.setGraphVal('php_opc_memory',k,mem[k]), keys)\n\n if self.hasGraph('php_opc_opcache_statistics') and stats:\n st = stats['opcache_statistics']\n self.setGraphVal('php_opc_opcache_statistics', 'hits', \n st['hits'])\n self.setGraphVal('php_opc_opcache_statistics', 'misses', \n st['misses'])\n\n if self.hasGraph('php_opc_opcache_hitrate') and stats:\n st = stats['opcache_statistics']\n self.setGraphVal('php_opc_opcache_hitrate', 'opcache_hit_rate',\n st['opcache_hit_rate'])\n\n if self.hasGraph('php_opc_key_status') and stats:\n st = stats['opcache_statistics']\n wasted = st['num_cached_keys'] - st['num_cached_scripts']\n free = st['max_cached_keys'] - st['num_cached_keys']\n self.setGraphVal('php_opc_key_status', 'num_cached_scripts', st['num_cached_scripts'])\n self.setGraphVal('php_opc_key_status', 'num_wasted_keys', wasted)\n self.setGraphVal('php_opc_key_status', 'num_free_keys', free)", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef autoconf(self):\n opcinfo = OPCinfo(self._host, self._port, self._user, self._password, \n self._monpath, self._ssl)\n return opcinfo is not None", "response": "Implements Munin Plugin Auto - Configuration Option.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fingerprint_helper(egg, permute=False, n_perms=1000,\n match='exact', distance='euclidean', features=None):\n \"\"\"\n Computes clustering along a set of feature dimensions\n\n Parameters\n ----------\n egg : quail.Egg\n Data to analyze\n\n dist_funcs : dict\n Dictionary of distance functions for feature clustering analyses\n\n Returns\n ----------\n probabilities : Numpy array\n Each number represents clustering along a different feature dimension\n\n \"\"\"\n\n if features is None:\n features = egg.dist_funcs.keys()\n\n inds = egg.pres.index.tolist()\n slices = [egg.crack(subjects=[i], lists=[j]) for i, j in inds]\n\n weights = _get_weights(slices, features, distdict, permute, n_perms, match,\n distance)\n return np.nanmean(weights, axis=0)", "response": "This function is used by the fingerprint_helper function to compute the fingerprint of a set of features."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the clustering scores along a set of feature dimensions and the distance vector of the next word in the encoding list.", "response": "def compute_feature_weights(pres_list, rec_list, feature_list, distances):\n \"\"\"\n Compute clustering scores along a set of feature dimensions\n\n Parameters\n ----------\n pres_list : list\n list of presented words\n\n rec_list : list\n list of recalled words\n\n feature_list : list\n list of feature dicts for presented words\n\n distances : dict\n dict of distance matrices for each feature\n\n Returns\n ----------\n weights : list\n list of clustering scores for each feature dimension\n \"\"\"\n\n # initialize the weights object for just this list\n weights = {}\n for feature in feature_list[0]:\n weights[feature] = []\n\n # return default list if there is not enough data to compute the fingerprint\n if len(rec_list) <= 2:\n print('Not enough recalls to compute fingerprint, returning default'\n 'fingerprint.. (everything is .5)')\n for feature in feature_list[0]:\n weights[feature] = .5\n return [weights[key] for key in weights]\n\n # initialize past word list\n past_words = []\n past_idxs = []\n\n # loop over words\n for i in range(len(rec_list)-1):\n\n # grab current word\n c = rec_list[i]\n\n # grab the next word\n n = rec_list[i + 1]\n\n # if both recalled words are in the encoding list and haven't been recalled before\n if (c in pres_list and n in pres_list) and (c not in past_words and n not in past_words):\n\n # for each feature\n for feature in feature_list[0]:\n\n # get the distance vector for the current word\n dists = distances[feature][pres_list.index(c),:]\n\n # distance between current and next word\n cdist = dists[pres_list.index(n)]\n\n # filter dists removing the words that have already been recalled\n dists_filt = np.array([dist for idx, dist in enumerate(dists) if idx not in past_idxs])\n\n # get indices\n avg_rank = np.mean(np.where(np.sort(dists_filt)[::-1] == cdist)[0]+1)\n\n # compute the weight\n weights[feature].append(avg_rank / len(dists_filt))\n\n # keep track of what has been recalled already\n past_idxs.append(pres_list.index(c))\n past_words.append(c)\n\n # average over the cluster scores for a particular dimension\n for feature in weights:\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n weights[feature] = np.nanmean(weights[feature])\n\n return [weights[key] for key in weights]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute probabilities for each transition distance (probability that a word recalled will be a given distance--in presentation order--from the previous recalled word). Parameters ---------- egg : quail.Egg Data to analyze match : str (exact, best or smooth) Matching approach to compute recall matrix. If exact, the presented and recalled items must be identical (default). If best, the recalled item that is most similar to the presented items will be selected. If smooth, a weighted average of all presented items will be used, where the weights are derived from the similarity between the recalled item and each presented item. distance : str The distance function used to compare presented and recalled items. Applies only to 'best' and 'smooth' matching approaches. Can be any distance function supported by numpy.spatial.distance.cdist. Returns ---------- prec : numpy array each float is the probability of transition distance (distnaces indexed by position, from -(n-1) to (n-1), excluding zero", "response": "def lagcrp_helper(egg, match='exact', distance='euclidean',\n ts=None, features=None):\n \"\"\"\n Computes probabilities for each transition distance (probability that a word\n recalled will be a given distance--in presentation order--from the previous\n recalled word).\n\n Parameters\n ----------\n egg : quail.Egg\n Data to analyze\n\n match : str (exact, best or smooth)\n Matching approach to compute recall matrix. If exact, the presented and\n recalled items must be identical (default). If best, the recalled item\n that is most similar to the presented items will be selected. If smooth,\n a weighted average of all presented items will be used, where the\n weights are derived from the similarity between the recalled item and\n each presented item.\n\n distance : str\n The distance function used to compare presented and recalled items.\n Applies only to 'best' and 'smooth' matching approaches. Can be any\n distance function supported by numpy.spatial.distance.cdist.\n\n Returns\n ----------\n prec : numpy array\n each float is the probability of transition distance (distnaces indexed by\n position, from -(n-1) to (n-1), excluding zero\n\n \"\"\"\n\n def lagcrp(rec, lstlen):\n \"\"\"Computes lag-crp for a given recall list\"\"\"\n\n def check_pair(a, b):\n if (a>0 and b>0) and (a!=b):\n return True\n else:\n return False\n\n def compute_actual(rec, lstlen):\n arr=pd.Series(data=np.zeros((lstlen)*2),\n index=list(range(-lstlen,0))+list(range(1,lstlen+1)))\n recalled=[]\n for trial in range(0,len(rec)-1):\n a=rec[trial]\n b=rec[trial+1]\n if check_pair(a, b) and (a not in recalled) and (b not in recalled):\n arr[b-a]+=1\n recalled.append(a)\n return arr\n\n def compute_possible(rec, lstlen):\n arr=pd.Series(data=np.zeros((lstlen)*2),\n index=list(range(-lstlen,0))+list(range(1,lstlen+1)))\n recalled=[]\n for trial in rec:\n if np.isnan(trial):\n pass\n else:\n lbound=int(1-trial)\n ubound=int(lstlen-trial)\n chances=list(range(lbound,0))+list(range(1,ubound+1))\n for each in recalled:\n if each-trial in chances:\n chances.remove(each-trial)\n arr[chances]+=1\n recalled.append(trial)\n return arr\n\n actual = compute_actual(rec, lstlen)\n possible = compute_possible(rec, lstlen)\n crp = [0.0 if j == 0 else i / j for i, j in zip(actual, possible)]\n crp.insert(int(len(crp) / 2), np.nan)\n return crp\n\n def nlagcrp(distmat, ts=None):\n\n def lagcrp_model(s):\n idx = list(range(0, -s, -1))\n return np.array([list(range(i, i+s)) for i in idx])\n\n # remove nan columns\n distmat = distmat[:,~np.all(np.isnan(distmat), axis=0)].T\n\n model = lagcrp_model(distmat.shape[1])\n lagcrp = np.zeros(ts * 2)\n for rdx in range(len(distmat)-1):\n item = distmat[rdx, :]\n next_item = distmat[rdx+1, :]\n if not np.isnan(item).any() and not np.isnan(next_item).any():\n outer = np.outer(item, next_item)\n lagcrp += np.array(list(map(lambda lag: np.mean(outer[model==lag]), range(-ts, ts))))\n lagcrp /= ts\n lagcrp = list(lagcrp)\n lagcrp.insert(int(len(lagcrp) / 2), np.nan)\n return np.array(lagcrp)\n\n def _format(p, r):\n p = np.matrix([np.array(i) for i in p])\n if p.shape[0]==1:\n p=p.T\n r = map(lambda x: [np.nan]*p.shape[1] if check_nan(x) else x, r)\n r = np.matrix([np.array(i) for i in r])\n if r.shape[0]==1:\n r=r.T\n return p, r\n\n opts = dict(match=match, distance=distance, features=features)\n if match is 'exact':\n opts.update({'features' : 'item'})\n recmat = recall_matrix(egg, **opts)\n if not ts:\n ts = egg.pres.shape[1]\n if match in ['exact', 'best']:\n lagcrp = [lagcrp(lst, egg.list_length) for lst in recmat]\n elif match is 'smooth':\n lagcrp = np.atleast_2d(np.mean([nlagcrp(r, ts=ts) for r in recmat], 0))\n else:\n raise ValueError('Match must be set to exact, best or smooth.')\n return np.nanmean(lagcrp, axis=0)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef retrieveVals(self):\n if self._diskList:\n self._fetchDevAll('disk', self._diskList, \n self._info.getDiskStats)\n if self._mdList:\n self._fetchDevAll('md', self._mdList, \n self._info.getMDstats)\n if self._partList:\n self._fetchDevAll('part', self._partList, \n self._info.getPartitionStats) \n if self._lvList:\n self._fetchDevAll('lv', self._lvList, \n self._info.getLVstats)\n self._fetchDevAll('fs', self._fsList, \n self._info.getFilesystemStats)", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _configDevRequests(self, namestr, titlestr, devlist):\n name = 'diskio_%s_requests' % namestr\n if self.graphEnabled(name):\n graph = MuninGraph('Disk I/O - %s - Requests' % titlestr, self._category,\n info='Disk I/O - %s Throughput, Read / write requests per second.' \n % titlestr,\n args='--base 1000 --lower-limit 0',\n vlabel='reqs/sec read (-) / write (+)', printf='%6.1lf',\n autoFixNames = True)\n for dev in devlist:\n graph.addField(dev + '_read',\n fixLabel(dev, maxLabelLenGraphDual, \n repl = '..', truncend=False,\n delim = self._labelDelim.get(namestr)), \n draw='LINE2', type='DERIVE', min=0, graph=False)\n graph.addField(dev + '_write',\n fixLabel(dev, maxLabelLenGraphDual, \n repl = '..', truncend=False,\n delim = self._labelDelim.get(namestr)),\n draw='LINE2', type='DERIVE', min=0, \n negative=(dev + '_read'),info=dev)\n self.appendGraph(name, graph)", "response": "Generate configuration for disk I / O Request stats."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _configDevActive(self, namestr, titlestr, devlist):\n name = 'diskio_%s_active' % namestr\n if self.graphEnabled(name):\n graph = MuninGraph('Disk I/O - %s - Queue Length' % titlestr, \n self._category,\n info='Disk I/O - Number of I/O Operations in Progress for every %s.'\n % titlestr,\n args='--base 1000 --lower-limit 0', printf='%6.1lf',\n autoFixNames = True)\n for dev in devlist:\n graph.addField(dev, \n fixLabel(dev, maxLabelLenGraphSimple, \n repl = '..', truncend=False,\n delim = self._labelDelim.get(namestr)), \n draw='AREASTACK', type='GAUGE', info=dev)\n self.appendGraph(name, graph)", "response": "Generate configuration for the active disk i / o queue length."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _fetchDevAll(self, namestr, devlist, statsfunc):\n for dev in devlist:\n stats = statsfunc(dev)\n name = 'diskio_%s_requests' % namestr\n if self.hasGraph(name):\n self.setGraphVal(name, dev + '_read', stats['rios'])\n self.setGraphVal(name, dev + '_write', stats['wios'])\n name = 'diskio_%s_bytes' % namestr\n if self.hasGraph(name):\n self.setGraphVal(name, dev + '_read', stats['rbytes'])\n self.setGraphVal(name, dev + '_write', stats['wbytes'])\n name = 'diskio_%s_active' % namestr\n if self.hasGraph(name):\n self.setGraphVal(name, dev, stats['ios_active'])", "response": "Initialize I / O stats for devices."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n for graph_name in self.getGraphList():\n for field_name in self.getGraphFieldList(graph_name):\n self.setGraphVal(graph_name, field_name, self._stats.get(field_name))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nretrieve values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n if self.hasGraph('sys_loadavg'):\n self._loadstats = self._sysinfo.getLoadAvg()\n if self._loadstats:\n self.setGraphVal('sys_loadavg', 'load15min', self._loadstats[2])\n self.setGraphVal('sys_loadavg', 'load5min', self._loadstats[1])\n self.setGraphVal('sys_loadavg', 'load1min', self._loadstats[0])\n if self._cpustats and self.hasGraph('sys_cpu_util'):\n for field in self.getGraphFieldList('sys_cpu_util'):\n self.setGraphVal('sys_cpu_util', \n field, int(self._cpustats[field] * 1000))\n if self._memstats:\n if self.hasGraph('sys_mem_util'):\n for field in self.getGraphFieldList('sys_mem_util'):\n self.setGraphVal('sys_mem_util', \n field, self._memstats[field])\n if self.hasGraph('sys_mem_avail'):\n for field in self.getGraphFieldList('sys_mem_avail'):\n self.setGraphVal('sys_mem_avail', \n field, self._memstats[field])\n if self.hasGraph('sys_mem_huge'):\n for field in ['Rsvd', 'Surp', 'Free']:\n fkey = 'HugePages_' + field\n if self._memstats.has_key(fkey):\n self.setGraphVal('sys_mem_huge', field, \n self._memstats[fkey] * self._memstats['Hugepagesize'])\n if self.hasGraph('sys_processes'):\n if self._procstats is None:\n self._procstats = self._sysinfo.getProcessStats()\n if self._procstats:\n self.setGraphVal('sys_processes', 'running', \n self._procstats['procs_running'])\n self.setGraphVal('sys_processes', 'blocked', \n self._procstats['procs_blocked'])\n if self.hasGraph('sys_forks'):\n if self._procstats is None:\n self._procstats = self._sysinfo.getProcessStats()\n if self._procstats:\n self.setGraphVal('sys_forks', 'forks', \n self._procstats['processes'])\n if self.hasGraph('sys_intr_ctxt'):\n if self._procstats is None:\n self._procstats = self._sysinfo.getProcessStats()\n if self._procstats:\n for field in self.getGraphFieldList('sys_intr_ctxt'):\n self.setGraphVal('sys_intr_ctxt', field, \n self._procstats[field])\n if self.hasGraph('sys_vm_paging'):\n if self._vmstats is None:\n self._vmstats = self._sysinfo.getVMstats()\n if self._vmstats:\n self.setGraphVal('sys_vm_paging', 'in', \n self._vmstats['pgpgin'])\n self.setGraphVal('sys_vm_paging', 'out', \n self._vmstats['pgpgout'])\n if self.hasGraph('sys_vm_swapping'):\n if self._vmstats is None:\n self._vmstats = self._sysinfo.getVMstats()\n if self._vmstats:\n self.setGraphVal('sys_vm_swapping', 'in', \n self._vmstats['pswpin'])\n self.setGraphVal('sys_vm_swapping', 'out', \n self._vmstats['pswpout'])"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsearches os. environ and returns the string value of the given key.", "response": "def get(key, default=None):\n \"\"\"\n Searches os.environ. If a key is found try evaluating its type else;\n return the string.\n\n returns: k->value (type as defined by ast.literal_eval)\n \"\"\"\n try:\n # Attempt to evaluate into python literal\n return ast.literal_eval(os.environ.get(key.upper(), default))\n except (ValueError, SyntaxError):\n return os.environ.get(key.upper(), default)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsave a list of keyword arguments as environment variables to a file.", "response": "def save(filepath=None, **kwargs):\n \"\"\"\n Saves a list of keyword arguments as environment variables to a file.\n If no filepath given will default to the default `.env` file.\n \"\"\"\n if filepath is None:\n filepath = os.path.join('.env')\n\n with open(filepath, 'wb') as file_handle:\n file_handle.writelines(\n '{0}={1}\\n'.format(key.upper(), val)\n for key, val in kwargs.items()\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load(filepath=None):\n if filepath and os.path.exists(filepath):\n pass\n else:\n if not os.path.exists('.env'):\n return False\n filepath = os.path.join('.env')\n\n for key, value in _get_line_(filepath):\n # set the key, value in the python environment vars dictionary\n # does not make modifications system wide.\n os.environ.setdefault(key, str(value))\n return True", "response": "Reads a. env file into the dictionary that contains the current Python environment vars."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting each line from the file and parses it into a list of tuples.", "response": "def _get_line_(filepath):\n \"\"\"\n Gets each line from the file and parse the data.\n Attempt to translate the value into a python type is possible\n (falls back to string).\n \"\"\"\n for line in open(filepath):\n line = line.strip()\n # allows for comments in the file\n if line.startswith('#') or '=' not in line:\n continue\n # split on the first =, allows for subsiquent `=` in strings\n key, value = line.split('=', 1)\n key = key.strip().upper()\n value = value.strip()\n\n if not (key and value):\n continue\n\n try:\n # evaluate the string before adding into environment\n # resolves any hanging (') characters\n value = ast.literal_eval(value)\n except (ValueError, SyntaxError):\n pass\n\n #return line\n yield (key, value)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef initStats(self):\n url = \"%s://%s:%d/%s?auto\" % (self._proto, self._host, self._port, \n self._statuspath)\n response = util.get_url(url, self._user, self._password)\n self._statusDict = {}\n for line in response.splitlines():\n mobj = re.match('(\\S.*\\S)\\s*:\\s*(\\S+)\\s*$', line)\n if mobj:\n self._statusDict[mobj.group(1)] = util.parse_value(mobj.group(2))\n if self._statusDict.has_key('Scoreboard'):\n self._statusDict['MaxWorkers'] = len(self._statusDict['Scoreboard'])", "response": "Query and parse Apache Web Server Status Page and initialize self. _statusDict."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_pres_features(self, features=None):\n if features is None:\n features = self.dist_funcs.keys()\n elif not isinstance(features, list):\n features = [features]\n return self.pres.applymap(lambda x: {k:v for k,v in x.items() if k in features} if x is not None else None)", "response": "Returns a df of features for presented items"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_rec_features(self, features=None):\n if features is None:\n features = self.dist_funcs.keys()\n elif not isinstance(features, list):\n features = [features]\n return self.rec.applymap(lambda x: {k:v for k,v in x.items() if k != 'item'} if x is not None else None)", "response": "Returns a df of features for recalled items"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprint info about the data egg", "response": "def info(self):\n \"\"\"\n Print info about the data egg\n \"\"\"\n print('Number of subjects: ' + str(self.n_subjects))\n print('Number of lists per subject: ' + str(self.n_lists))\n print('Number of words per list: ' + str(self.list_length))\n print('Date created: ' + str(self.date_created))\n print('Meta data: ' + str(self.meta))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsave the object to a file.", "response": "def save(self, fname, compression='blosc'):\n \"\"\"\n Save method for the Egg object\n\n The data will be saved as a 'egg' file, which is a dictionary containing\n the elements of a Egg saved in the hd5 format using\n `deepdish`.\n\n Parameters\n ----------\n\n fname : str\n A name for the file. If the file extension (.egg) is not specified,\n it will be appended.\n\n compression : str\n The kind of compression to use. See the deepdish documentation for\n options: http://deepdish.readthedocs.io/en/latest/api_io.html#deepdish.io.save\n\n \"\"\"\n\n # put egg vars into a dict\n egg = {\n 'pres' : df2list(self.pres),\n 'rec' : df2list(self.rec),\n 'dist_funcs' : self.dist_funcs,\n 'subjgroup' : self.subjgroup,\n 'subjname' : self.subjname,\n 'listgroup' : self.listgroup,\n 'listname' : self.listname,\n 'date_created' : self.date_created,\n 'meta' : self.meta\n }\n\n # if extension wasn't included, add it\n if fname[-4:]!='.egg':\n fname+='.egg'\n\n # save\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n dd.io.save(fname, egg, compression=compression)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsaves the object to a file.", "response": "def save(self, fname, compression='blosc'):\n \"\"\"\n Save method for the FriedEgg object\n\n The data will be saved as a 'fegg' file, which is a dictionary containing\n the elements of a FriedEgg saved in the hd5 format using\n `deepdish`.\n\n Parameters\n ----------\n\n fname : str\n A name for the file. If the file extension (.fegg) is not specified,\n it will be appended.\n\n compression : str\n The kind of compression to use. See the deepdish documentation for\n options: http://deepdish.readthedocs.io/en/latest/api_io.html#deepdish.io.save\n\n \"\"\"\n\n egg = {\n 'data' : self.data,\n 'analysis' : self.analysis,\n 'list_length' : self.list_length,\n 'n_lists' : self.n_lists,\n 'n_subjects' : self.n_subjects,\n 'position' : self.position,\n 'date_created' : self.date_created,\n 'meta' : self.meta\n }\n\n if fname[-4:]!='.fegg':\n fname+='.fegg'\n\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n dd.io.save(fname, egg, compression=compression)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pnr_helper(egg, position, match='exact',\n distance='euclidean', features=None):\n\n \"\"\"\n Computes probability of a word being recalled nth (in the appropriate recall\n list), given its presentation position. Note: zero indexed\n\n Parameters\n ----------\n egg : quail.Egg\n Data to analyze\n\n position : int\n Position of item to be analyzed\n\n match : str (exact, best or smooth)\n Matching approach to compute recall matrix. If exact, the presented and\n recalled items must be identical (default). If best, the recalled item\n that is most similar to the presented items will be selected. If smooth,\n a weighted average of all presented items will be used, where the\n weights are derived from the similarity between the recalled item and\n each presented item.\n\n distance : str\n The distance function used to compare presented and recalled items.\n Applies only to 'best' and 'smooth' matching approaches. Can be any\n distance function supported by numpy.spatial.distance.cdist.\n\n Returns\n ----------\n prob_recalled : numpy array\n each number represents the probability of nth recall for a word presented in given position/index\n\n \"\"\"\n def pnr(lst, position):\n return [1 if pos==lst[position] else 0 for pos in range(1,egg.list_length+1)]\n\n opts = dict(match=match, distance=distance, features=features)\n if match is 'exact':\n opts.update({'features' : 'item'})\n recmat = recall_matrix(egg, **opts)\n\n if match in ['exact', 'best']:\n result = [pnr(lst, position) for lst in recmat]\n elif match is 'smooth':\n result = np.atleast_2d(recmat[:, :, 0])\n else:\n raise ValueError('Match must be set to exact, best or smooth.')\n return np.nanmean(result, axis=0)", "response": "Returns the probability of a word being recalled nth for a given item in the given presentation position."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n apacheInfo = ApacheInfo(self._host, self._port,\n self._user, self._password, \n self._statuspath, self._ssl)\n stats = apacheInfo.getServerStats()\n if self.hasGraph('apache_access'):\n self.setGraphVal('apache_access', 'reqs', stats['Total Accesses'])\n if self.hasGraph('apache_bytes'):\n self.setGraphVal('apache_bytes', 'bytes', \n stats['Total kBytes'] * 1000)\n if self.hasGraph('apache_workers'):\n self.setGraphVal('apache_workers', 'busy', stats['BusyWorkers'])\n self.setGraphVal('apache_workers', 'idle', stats['IdleWorkers'])\n self.setGraphVal('apache_workers', 'max', stats['MaxWorkers'])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nimplement Munin Plugin Auto - Configuration Option.", "response": "def autoconf(self):\n \"\"\"Implements Munin Plugin Auto-Configuration Option.\n \n @return: True if plugin can be auto-configured, False otherwise.\n \n \"\"\"\n apacheInfo = ApacheInfo(self._host, self._port,\n self._user, self._password, \n self._statuspath, self._ssl)\n return apacheInfo is not None"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nretrieves values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n ntpinfo = NTPinfo()\n ntpstats = ntpinfo.getHostOffsets(self._remoteHosts)\n if ntpstats:\n for host in self._remoteHosts:\n hostkey = re.sub('\\.', '_', host)\n hoststats = ntpstats.get(host)\n if hoststats:\n if self.hasGraph('ntp_host_stratums'):\n self.setGraphVal('ntp_host_stratums', hostkey, \n hoststats.get('stratum'))\n if self.hasGraph('ntp_host_offsets'):\n self.setGraphVal('ntp_host_offsets', hostkey, \n hoststats.get('offset'))\n if self.hasGraph('ntp_host_delays'):\n self.setGraphVal('ntp_host_delays', hostkey, \n hoststats.get('delay'))"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nimplementing Munin Plugin Auto - Configuration Option.", "response": "def autoconf(self):\n \"\"\"Implements Munin Plugin Auto-Configuration Option.\n \n @return: True if plugin can be auto-configured, False otherwise.\n \n \"\"\"\n ntpinfo = NTPinfo()\n ntpstats = ntpinfo.getHostOffsets(self._remoteHosts)\n return len(ntpstats) > 0"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef retrieveVals(self):\n lighttpdInfo = LighttpdInfo(self._host, self._port,\n self._user, self._password, \n self._statuspath, self._ssl)\n stats = lighttpdInfo.getServerStats()\n if self.hasGraph('lighttpd_access'):\n self.setGraphVal('lighttpd_access', 'reqs', stats['Total Accesses'])\n if self.hasGraph('lighttpd_bytes'):\n self.setGraphVal('lighttpd_bytes', 'bytes', \n stats['Total kBytes'] * 1000)\n if self.hasGraph('lighttpd_servers'):\n self.setGraphVal('lighttpd_servers', 'busy', stats['BusyServers'])\n self.setGraphVal('lighttpd_servers', 'idle', stats['IdleServers'])\n self.setGraphVal('lighttpd_servers', 'max', stats['MaxServers'])", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef autoconf(self):\n lighttpdInfo = LighttpdInfo(self._host, self._port,\n self._user, self._password, \n self._statuspath, self._ssl)\n return lighttpdInfo is not None", "response": "Implements Munin Plugin Auto - Configuration Option.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef retrieveVals(self):\n name = 'diskspace'\n if self.hasGraph(name):\n for fspath in self._fslist:\n if self._statsSpace.has_key(fspath):\n self.setGraphVal(name, fspath, \n self._statsSpace[fspath]['inuse_pcent'])\n name = 'diskinode'\n if self.hasGraph(name):\n for fspath in self._fslist:\n if self._statsInode.has_key(fspath):\n self.setGraphVal(name, fspath, \n self._statsInode[fspath]['inuse_pcent'])", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef symbolic(self, mtx):\n self.free_symbolic()\n\n indx = self._getIndx(mtx)\n\n if not assumeSortedIndices:\n # row/column indices cannot be assumed to be sorted\n mtx.sort_indices()\n\n if self.isReal:\n status, self._symbolic\\\n = self.funs.symbolic(mtx.shape[0], mtx.shape[1],\n mtx.indptr,\n indx,\n mtx.data,\n self.control, self.info)\n else:\n real, imag = mtx.data.real.copy(), mtx.data.imag.copy()\n status, self._symbolic\\\n = self.funs.symbolic(mtx.shape[0], mtx.shape[1],\n mtx.indptr,\n indx,\n real, imag,\n self.control, self.info)\n\n if status != UMFPACK_OK:\n raise RuntimeError('%s failed with %s' % (self.funs.symbolic,\n umfStatus[status]))\n\n self.mtx = mtx", "response": "Perform a symbolic LU decomposition of a given object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef numeric(self, mtx):\n\n self.free_numeric()\n\n if self._symbolic is None:\n self.symbolic(mtx)\n\n indx = self._getIndx(mtx)\n\n failCount = 0\n while 1:\n if self.isReal:\n status, self._numeric\\\n = self.funs.numeric(mtx.indptr, indx, mtx.data,\n self._symbolic,\n self.control, self.info)\n else:\n real, imag = mtx.data.real.copy(), mtx.data.imag.copy()\n status, self._numeric\\\n = self.funs.numeric(mtx.indptr, indx,\n real, imag,\n self._symbolic,\n self.control, self.info)\n\n if status != UMFPACK_OK:\n if status == UMFPACK_WARNING_singular_matrix:\n warnings.warn('Singular matrix', UmfpackWarning)\n break\n elif status in (UMFPACK_ERROR_different_pattern,\n UMFPACK_ERROR_invalid_Symbolic_object):\n # Try again.\n warnings.warn('Recomputing symbolic', UmfpackWarning)\n self.symbolic(mtx)\n failCount += 1\n else:\n failCount += 100\n else:\n break\n if failCount >= 2:\n raise RuntimeError('%s failed with %s' % (self.funs.numeric,\n umfStatus[status]))", "response": "Perform numeric object decomposition using the symbolic decomposition."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsolves the system of linear equation using the Numeric object.", "response": "def solve(self, sys, mtx, rhs, autoTranspose=False):\n \"\"\"\n Solution of system of linear equation using the Numeric object.\n\n Parameters\n ----------\n sys : constant\n one of UMFPACK system description constants, like\n UMFPACK_A, UMFPACK_At, see umfSys list and UMFPACK docs\n mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix\n Input.\n rhs : ndarray\n Right Hand Side\n autoTranspose : bool\n Automatically changes `sys` to the transposed type, if `mtx` is in CSR,\n since UMFPACK assumes CSC internally\n\n Returns\n -------\n sol : ndarray\n Solution to the equation system.\n\n \"\"\"\n if sys not in umfSys:\n raise ValueError('sys must be in' % umfSys)\n\n if autoTranspose and self.isCSR:\n ##\n # UMFPACK uses CSC internally...\n if self.family in umfRealTypes:\n ii = 0\n else:\n ii = 1\n if sys in umfSys_transposeMap[ii]:\n sys = umfSys_transposeMap[ii][sys]\n else:\n raise RuntimeError('autoTranspose ambiguous, switch it off')\n\n if self._numeric is not None:\n if self.mtx is not mtx:\n raise ValueError('must be called with same matrix as numeric()')\n else:\n raise RuntimeError('numeric() not called')\n\n indx = self._getIndx(mtx)\n\n if self.isReal:\n rhs = rhs.astype(np.float64)\n sol = np.zeros((mtx.shape[1],), dtype=np.float64)\n status = self.funs.solve(sys, mtx.indptr, indx, mtx.data, sol, rhs,\n self._numeric, self.control, self.info)\n else:\n rhs = rhs.astype(np.complex128)\n sol = np.zeros((mtx.shape[1],), dtype=np.complex128)\n mreal, mimag = mtx.data.real.copy(), mtx.data.imag.copy()\n sreal, simag = sol.real.copy(), sol.imag.copy()\n rreal, rimag = rhs.real.copy(), rhs.imag.copy()\n status = self.funs.solve(sys, mtx.indptr, indx,\n mreal, mimag, sreal, simag, rreal, rimag,\n self._numeric, self.control, self.info)\n sol.real, sol.imag = sreal, simag\n\n # self.funs.report_info( self.control, self.info )\n # pause()\n if status != UMFPACK_OK:\n if status == UMFPACK_WARNING_singular_matrix:\n ## Change inf, nan to zeros.\n warnings.warn('Zeroing nan and inf entries...', UmfpackWarning)\n sol[~np.isfinite(sol)] = 0.0\n else:\n raise RuntimeError('%s failed with %s' % (self.funs.solve,\n umfStatus[status]))\n econd = 1.0 / self.info[UMFPACK_RCOND]\n if econd > self.maxCond:\n msg = '(almost) singular matrix! '\\\n + '(estimated cond. number: %.2e)' % econd\n warnings.warn(msg, UmfpackWarning)\n\n return sol"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a matrix of one - shot solution of system of linear equation.", "response": "def linsolve(self, sys, mtx, rhs, autoTranspose=False):\n \"\"\"\n One-shot solution of system of linear equation. Reuses Numeric object\n if possible.\n\n Parameters\n ----------\n sys : constant\n one of UMFPACK system description constants, like\n UMFPACK_A, UMFPACK_At, see umfSys list and UMFPACK docs\n mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix\n Input.\n rhs : ndarray\n Right Hand Side\n autoTranspose : bool\n Automatically changes `sys` to the transposed type, if `mtx` is in CSR,\n since UMFPACK assumes CSC internally\n\n Returns\n -------\n sol : ndarray\n Solution to the equation system.\n \"\"\"\n\n if sys not in umfSys:\n raise ValueError('sys must be in' % umfSys)\n\n if self._numeric is None:\n self.numeric(mtx)\n else:\n if self.mtx is not mtx:\n self.numeric(mtx)\n\n sol = self.solve(sys, mtx, rhs, autoTranspose)\n self.free_numeric()\n\n return sol"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nperform LU decomposition. For a given matrix A, the decomposition satisfies:: LU = PRAQ when do_recip is true LU = P(R^-1)AQ when do_recip is false Parameters ---------- mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix Input. Returns ------- L : csr_matrix Lower triangular m-by-min(m,n) CSR matrix U : csc_matrix Upper triangular min(m,n)-by-n CSC matrix P : ndarray Vector of row permutations Q : ndarray Vector of column permutations R : ndarray Vector of diagonal row scalings do_recip : bool Whether R is R^-1 or R", "response": "def lu(self, mtx):\n \"\"\"\n Perform LU decomposition.\n\n For a given matrix A, the decomposition satisfies::\n\n LU = PRAQ when do_recip is true\n LU = P(R^-1)AQ when do_recip is false\n\n Parameters\n ----------\n mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix\n Input.\n\n Returns\n -------\n L : csr_matrix\n Lower triangular m-by-min(m,n) CSR matrix\n U : csc_matrix\n Upper triangular min(m,n)-by-n CSC matrix\n P : ndarray\n Vector of row permutations\n Q : ndarray\n Vector of column permutations\n R : ndarray\n Vector of diagonal row scalings\n do_recip : bool\n Whether R is R^-1 or R\n\n \"\"\"\n\n # this should probably be changed\n mtx = mtx.tocsc()\n self.numeric(mtx)\n\n # first find out how much space to reserve\n (status, lnz, unz, n_row, n_col, nz_udiag)\\\n = self.funs.get_lunz(self._numeric)\n\n if status != UMFPACK_OK:\n raise RuntimeError('%s failed with %s' % (self.funs.get_lunz,\n umfStatus[status]))\n\n # allocate storage for decomposition data\n i_type = mtx.indptr.dtype\n\n Lp = np.zeros((n_row+1,), dtype=i_type)\n Lj = np.zeros((lnz,), dtype=i_type)\n Lx = np.zeros((lnz,), dtype=np.double)\n\n Up = np.zeros((n_col+1,), dtype=i_type)\n Ui = np.zeros((unz,), dtype=i_type)\n Ux = np.zeros((unz,), dtype=np.double)\n\n P = np.zeros((n_row,), dtype=i_type)\n Q = np.zeros((n_col,), dtype=i_type)\n\n Dx = np.zeros((min(n_row,n_col),), dtype=np.double)\n\n Rs = np.zeros((n_row,), dtype=np.double)\n\n if self.isReal:\n (status,do_recip) = self.funs.get_numeric(Lp,Lj,Lx,Up,Ui,Ux,\n P,Q,Dx,Rs,\n self._numeric)\n\n if status != UMFPACK_OK:\n raise RuntimeError('%s failed with %s'\n % (self.funs.get_numeric, umfStatus[status]))\n\n L = sp.csr_matrix((Lx,Lj,Lp),(n_row,min(n_row,n_col)))\n U = sp.csc_matrix((Ux,Ui,Up),(min(n_row,n_col),n_col))\n R = Rs\n\n return (L,U,P,Q,R,bool(do_recip))\n\n else:\n # allocate additional storage for imaginary parts\n Lz = np.zeros((lnz,), dtype=np.double)\n Uz = np.zeros((unz,), dtype=np.double)\n Dz = np.zeros((min(n_row,n_col),), dtype=np.double)\n\n (status,do_recip) = self.funs.get_numeric(Lp,Lj,Lx,Lz,Up,Ui,Ux,Uz,\n P,Q,Dx,Dz,Rs,\n self._numeric)\n\n if status != UMFPACK_OK:\n raise RuntimeError('%s failed with %s'\n % (self.funs.get_numeric, umfStatus[status]))\n\n Lxz = np.zeros((lnz,), dtype=np.complex128)\n Uxz = np.zeros((unz,), dtype=np.complex128)\n Dxz = np.zeros((min(n_row,n_col),), dtype=np.complex128)\n\n Lxz.real,Lxz.imag = Lx,Lz\n Uxz.real,Uxz.imag = Ux,Uz\n Dxz.real,Dxz.imag = Dx,Dz\n\n L = sp.csr_matrix((Lxz,Lj,Lp),(n_row,min(n_row,n_col)))\n U = sp.csc_matrix((Uxz,Ui,Up),(min(n_row,n_col),n_col))\n R = Rs\n\n return (L,U,P,Q,R,bool(do_recip))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef order_stick(presenter, egg, dist_dict, strategy, fingerprint):\n\n def compute_feature_stick(features, weights, alpha):\n '''create a 'stick' of feature weights'''\n\n feature_stick = []\n for f, w in zip(features, weights):\n feature_stick+=[f]*int(np.power(w,alpha)*100)\n\n return feature_stick\n\n def reorder_list(egg, feature_stick, dist_dict, tau):\n\n def compute_stimulus_stick(s, tau):\n '''create a 'stick' of feature weights'''\n\n feature_stick = [[weights[feature]]*round(weights[feature]**alpha)*100 for feature in w]\n return [item for sublist in feature_stick for item in sublist]\n\n # parse egg\n pres, rec, features, dist_funcs = parse_egg(egg)\n\n # turn pres and features into np arrays\n pres_arr = np.array(pres)\n features_arr = np.array(features)\n\n # starting with a random word\n reordered_list = []\n reordered_features = []\n\n # start with a random choice\n idx = np.random.choice(len(pres), 1)[0]\n\n # original inds\n inds = list(range(len(pres)))\n\n # keep track of the indices\n inds_used = [idx]\n\n # get the word\n current_word = pres[idx]\n\n # get the features dict\n current_features = features[idx]\n\n # append that word to the reordered list\n reordered_list.append(current_word)\n\n # append the features to the reordered list\n reordered_features.append(current_features)\n\n # loop over the word list\n for i in range(len(pres)-1):\n\n # sample from the stick\n feature_sample = feature_stick[np.random.choice(len(feature_stick), 1)[0]]\n\n # indices left\n inds_left = [ind for ind in inds if ind not in inds_used]\n\n # make a copy of the words filtering out the already used ones\n words_left = pres[inds_left]\n\n # get word distances for the word\n dists_left = np.array([dist_dict[current_word][word][feature_sample] for word in words_left])\n\n # features left\n features_left = features[inds_left]\n\n # normalize distances\n dists_left_max = np.max(dists_left)\n if dists_left_max>0:\n dists_left_norm = dists_left/np.max(dists_left)\n else:\n dists_left_norm = dists_left\n\n # get the min\n dists_left_min = np.min(-dists_left_norm)\n\n # invert the word distances to turn distance->similarity\n dists_left_inv = - dists_left_norm - dists_left_min + .01\n\n # create a word stick\n words_stick = []\n for word, dist in zip(words_left, dists_left_inv):\n words_stick+=[word]*int(np.power(dist,tau)*100)\n\n next_word = np.random.choice(words_stick)\n\n next_word_idx = np.where(pres==next_word)[0]\n\n inds_used.append(next_word_idx)\n\n reordered_list.append(next_word)\n reordered_features.append(features[next_word_idx][0])\n\n return Egg(pres=[reordered_list], rec=[reordered_list], features=[[reordered_features]], dist_funcs=dist_funcs)\n\n # parse egg\n pres, rec, features, dist_funcs = parse_egg(egg)\n\n # get params needed for list reordering\n features = presenter.get_params('fingerprint').get_features()\n alpha = presenter.get_params('alpha')\n tau = presenter.get_params('tau')\n weights = fingerprint\n\n # invert the weights if strategy is destabilize\n if strategy=='destabilize':\n weights = 1 - weights\n\n # compute feature stick\n feature_stick = compute_feature_stick(features, weights, alpha)\n\n # reorder list\n return reorder_list(egg, feature_stick, dist_dict, tau)", "response": "Reorders a list according to a given strategy"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef stick_perm(presenter, egg, dist_dict, strategy):\n\n # seed RNG\n np.random.seed()\n\n # unpack egg\n egg_pres, egg_rec, egg_features, egg_dist_funcs = parse_egg(egg)\n\n # reorder\n regg = order_stick(presenter, egg, dist_dict, strategy)\n\n # unpack regg\n regg_pres, regg_rec, regg_features, regg_dist_funcs = parse_egg(regg)\n\n # # get the order\n regg_pres = list(regg_pres)\n egg_pres = list(egg_pres)\n idx = [egg_pres.index(r) for r in regg_pres]\n\n # compute weights\n weights = compute_feature_weights_dict(list(regg_pres), list(regg_pres), list(regg_features), dist_dict)\n\n # save out the order\n orders = idx\n\n return weights, orders", "response": "Computes weights for one reordering using stick - breaking method"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute the distances between two sets of words in the archive.", "response": "def compute_distances_dict(egg):\n \"\"\" Creates a nested dict of distances \"\"\"\n pres, rec, features, dist_funcs = parse_egg(egg)\n pres_list = list(pres)\n features_list = list(features)\n\n # initialize dist dict\n distances = {}\n\n # for each word in the list\n for idx1, item1 in enumerate(pres_list):\n\n distances[item1]={}\n\n # for each word in the list\n for idx2, item2 in enumerate(pres_list):\n\n distances[item1][item2]={}\n\n # for each feature in dist_funcs\n for feature in dist_funcs:\n distances[item1][item2][feature] = builtin_dist_funcs[dist_funcs[feature]](features_list[idx1][feature],features_list[idx2][feature])\n\n return distances"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the clustering scores along a set of feature dimensions and the distance matrix of each feature.", "response": "def compute_feature_weights_dict(pres_list, rec_list, feature_list, dist_dict):\n \"\"\"\n Compute clustering scores along a set of feature dimensions\n\n Parameters\n ----------\n pres_list : list\n list of presented words\n rec_list : list\n list of recalled words\n feature_list : list\n list of feature dicts for presented words\n distances : dict\n dict of distance matrices for each feature\n\n Returns\n ----------\n weights : list\n list of clustering scores for each feature dimension\n \"\"\"\n\n # initialize the weights object for just this list\n weights = {}\n for feature in feature_list[0]:\n weights[feature] = []\n\n # return default list if there is not enough data to compute the fingerprint\n if len(rec_list) < 2:\n print('Not enough recalls to compute fingerprint, returning default fingerprint.. (everything is .5)')\n for feature in feature_list[0]:\n weights[feature] = .5\n return [weights[key] for key in weights]\n\n # initialize past word list\n past_words = []\n past_idxs = []\n\n # loop over words\n for i in range(len(rec_list)-1):\n\n # grab current word\n c = rec_list[i]\n\n # grab the next word\n n = rec_list[i + 1]\n\n # if both recalled words are in the encoding list and haven't been recalled before\n if (c in pres_list and n in pres_list) and (c not in past_words and n not in past_words):\n\n # for each feature\n for feature in feature_list[0]:\n\n # get the distance vector for the current word\n # dists = [dist_dict[c][j][feature] for j in dist_dict[c]]\n # distance between current and next word\n c_dist = dist_dict[c][n][feature]\n\n # filter dists removing the words that have already been recalled\n # dists_filt = np.array([dist for idx, dist in enumerate(dists) if idx not in past_idxs])\n dists_filt = [dist_dict[c][j][feature] for j in dist_dict[c] if j not in past_words]\n\n # get indices\n avg_rank = np.mean(np.where(np.sort(dists_filt)[::-1] == c_dist)[0]+1)\n\n # compute the weight\n weights[feature].append(avg_rank / len(dists_filt))\n\n # keep track of what has been recalled already\n past_idxs.append(pres_list.index(c))\n past_words.append(c)\n\n # average over the cluster scores for a particular dimension\n for feature in weights:\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n weights[feature] = np.nanmean(weights[feature])\n\n return [weights[key] for key in weights]"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreorders a list of stimuli to match a fingerprint Parameters ---------- egg : quail.Egg Data to compute fingerprint method : str Method to re-sort list. Can be 'stick' or 'permute' (default: permute) nperms : int Number of permutations to use. Only used if method='permute'. (default: 2500) strategy : str or None The strategy to use to reorder the list. This can be 'stabilize', 'destabilize', 'random' or None. If None, the self.strategy field will be used. (default: None) distfun : str or function The distance function to reorder the list fingerprint to the target fingerprint. Can be any distance function supported by scipy.spatial.distance.cdist. For more info, see: https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html (default: euclidean) fingerprint : quail.Fingerprint or np.array Fingerprint (or just the state of a fingerprint) to reorder by. If None, the list will be reordered according to the fingerprint attached to the presenter object. Returns ---------- egg : quail.Egg Egg re-sorted to match fingerprint", "response": "def order(self, egg, method='permute', nperms=2500, strategy=None,\n distfun='correlation', fingerprint=None):\n \"\"\"\n Reorders a list of stimuli to match a fingerprint\n\n Parameters\n ----------\n egg : quail.Egg\n Data to compute fingerprint\n\n method : str\n Method to re-sort list. Can be 'stick' or 'permute' (default: permute)\n\n nperms : int\n Number of permutations to use. Only used if method='permute'. (default:\n 2500)\n\n strategy : str or None\n The strategy to use to reorder the list. This can be 'stabilize',\n 'destabilize', 'random' or None. If None, the self.strategy field\n will be used. (default: None)\n\n distfun : str or function\n The distance function to reorder the list fingerprint to the target\n fingerprint. Can be any distance function supported by\n scipy.spatial.distance.cdist. For more info, see:\n https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html\n (default: euclidean)\n\n fingerprint : quail.Fingerprint or np.array\n Fingerprint (or just the state of a fingerprint) to reorder by. If\n None, the list will be reordered according to the fingerprint\n attached to the presenter object.\n\n Returns\n ----------\n egg : quail.Egg\n Egg re-sorted to match fingerprint\n \"\"\"\n\n def order_perm(self, egg, dist_dict, strategy, nperm, distperm,\n fingerprint):\n \"\"\"\n This function re-sorts a list by computing permutations of a given\n list and choosing the one that maximizes/minimizes variance.\n \"\"\"\n\n # parse egg\n pres, rec, features, dist_funcs = parse_egg(egg)\n\n # length of list\n pres_len = len(pres)\n\n weights = []\n orders = []\n for i in range(nperms):\n x = rand_perm(pres, features, dist_dict, dist_funcs)\n weights.append(x[0])\n orders.append(x[1])\n weights = np.array(weights)\n orders = np.array(orders)\n\n # find the closest (or farthest)\n if strategy=='stabilize':\n closest = orders[np.nanargmin(cdist(np.array(fingerprint, ndmin=2), weights, distperm)),:].astype(int).tolist()\n elif strategy=='destabilize':\n closest = orders[np.nanargmax(cdist(np.array(fingerprint, ndmin=2), weights, distperm)),:].astype(int).tolist()\n\n # return a re-sorted egg\n return Egg(pres=[list(pres[closest])], rec=[list(pres[closest])], features=[list(features[closest])])\n\n def order_best_stick(self, egg, dist_dict, strategy, nperms, distfun,\n fingerprint):\n\n # parse egg\n pres, rec, features, dist_funcs = parse_egg(egg)\n\n results = Parallel(n_jobs=multiprocessing.cpu_count())(\n delayed(stick_perm)(self, egg, dist_dict, strategy) for i in range(nperms))\n\n weights = np.array([x[0] for x in results])\n orders = np.array([x[1] for x in results])\n\n # find the closest (or farthest)\n if strategy=='stabilize':\n closest = orders[np.nanargmin(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()\n elif strategy=='destabilize':\n closest = orders[np.nanargmax(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()\n\n # return a re-sorted egg\n return Egg(pres=[list(pres[closest])], rec=[list(pres[closest])], features=[list(features[closest])], dist_funcs=dist_funcs)\n\n def order_best_choice(self, egg, dist_dict, nperms, distfun,\n fingerprint):\n\n # get strategy\n strategy = self.strategy\n\n # parse egg\n pres, rec, features, dist_funcs = parse_egg(egg)\n\n results = Parallel(n_jobs=multiprocessing.cpu_count())(\n delayed(choice_perm)(self, egg, dist_dict) for i in range(nperms))\n\n weights = np.array([x[0] for x in results])\n orders = np.array([x[1] for x in results])\n\n # find the closest (or farthest)\n if strategy=='stabilize':\n closest = orders[np.nanargmin(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()\n elif strategy=='destabilize':\n closest = orders[np.nanargmax(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()\n\n # return a re-sorted egg\n return Egg(pres=[list(pres[closest])], rec=[list(pres[closest])], features=[list(features[closest])], dist_funcs=dist_funcs)\n\n # if strategy is not set explicitly, default to the class strategy\n if strategy is None:\n strategy = self.strategy\n\n dist_dict = compute_distances_dict(egg)\n\n if fingerprint is None:\n fingerprint = self.get_params('fingerprint').state\n elif isinstance(fingerprint, Fingerprint):\n fingerprint = fingerprint.state\n else:\n print('using custom fingerprint')\n\n if (strategy=='random') or (method=='random'):\n return shuffle_egg(egg)\n elif method=='permute':\n return order_perm(self, egg, dist_dict, strategy, nperms, distfun,\n fingerprint) #\n elif method=='stick':\n return order_stick(self, egg, dist_dict, strategy, fingerprint) #\n elif method=='best_stick':\n return order_best_stick(self, egg, dist_dict, strategy, nperms,\n distfun, fingerprint) #\n elif method=='best_choice':\n return order_best_choice(self, egg, dist_dict, nperms,\n fingerprint)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nquery and parse Web Server Status Page.", "response": "def initStats(self, extras=None):\n \"\"\"Query and parse Web Server Status Page.\n \n @param extras: Include extra metrics, which can be computationally more \n expensive.\n \n \"\"\"\n url = \"%s://%s:%d/%s\" % (self._proto, self._host, self._port, self._monpath)\n response = util.get_url(url, self._user, self._password)\n #with open('/tmp/opcinfo.json') as f:\n # response = f.read()\n self._statusDict = json.loads(response)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns major and minor device number for a given block device path.", "response": "def _getDevMajorMinor(self, devpath):\n \"\"\"Return major and minor device number for block device path devpath.\n @param devpath: Full path for block device.\n @return: Tuple (major, minor).\n \n \"\"\"\n fstat = os.stat(devpath)\n if stat.S_ISBLK(fstat.st_mode):\n return(os.major(fstat.st_rdev), os.minor(fstat.st_rdev))\n else:\n raise ValueError(\"The file %s is not a valid block device.\" % devpath)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn unique device for any block device path.", "response": "def _getUniqueDev(self, devpath):\n \"\"\"Return unique device for any block device path.\n \n @param devpath: Full path for block device.\n @return: Unique device string without the /dev prefix.\n \n \"\"\"\n realpath = os.path.realpath(devpath)\n mobj = re.match('\\/dev\\/(.*)$', realpath)\n if mobj:\n dev = mobj.group(1)\n if dev in self._diskStats:\n return dev\n else:\n try:\n (major, minor) = self._getDevMajorMinor(realpath)\n except:\n return None\n return self._mapMajorMinor2dev.get((major, minor))\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _initBlockMajorMap(self):\n self._mapMajorDevclass = {}\n try:\n fp = open(devicesFile, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading device information from file: %s'\n % devicesFile)\n skip = True\n for line in data.splitlines():\n if skip:\n if re.match('block.*:', line, re.IGNORECASE):\n skip = False\n else:\n mobj = re.match('\\s*(\\d+)\\s+([\\w\\-]+)$', line)\n if mobj:\n major = int(mobj.group(1))\n devtype = mobj.group(2)\n self._mapMajorDevclass[major] = devtype\n if devtype == 'device-mapper':\n self._dmMajorNum = major", "response": "Parses the block devices file to initialize the device class - major number map."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _initDMinfo(self):\n self._mapLVtuple2dm = {}\n self._mapLVname2dm = {}\n self._vgTree = {}\n if self._dmMajorNum is None:\n self._initBlockMajorMap()\n for file in os.listdir(devmapperDir):\n mobj = re.match('([a-zA-Z0-9+_.\\-]*[a-zA-Z0-9+_.])-([a-zA-Z0-9+_.][a-zA-Z0-9+_.\\-]*)$', file)\n if mobj:\n path = os.path.join(devmapperDir, file)\n (major, minor) = self._getDevMajorMinor(path)\n if major == self._dmMajorNum:\n vg = mobj.group(1).replace('--', '-')\n lv = mobj.group(2).replace('--', '-')\n dmdev = \"dm-%d\" % minor\n self._mapLVtuple2dm[(vg,lv)] = dmdev\n self._mapLVname2dm[file] = dmdev\n if not vg in self._vgTree:\n self._vgTree[vg] = []\n self._vgTree[vg].append(lv)", "response": "Initialize the internal data structures for the VGs \n and LVs."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _initFilesystemInfo(self):\n self._mapFSpathDev = {}\n fsinfo = FilesystemInfo()\n for fs in fsinfo.getFSlist():\n devpath = fsinfo.getFSdev(fs)\n dev = self._getUniqueDev(devpath)\n if dev is not None:\n self._mapFSpathDev[fs] = dev", "response": "Initialize filesystem to device mappings."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes swap partition to device mappings.", "response": "def _initSwapInfo(self):\n \"\"\"Initialize swap partition to device mappings.\"\"\"\n self._swapList = []\n sysinfo = SystemInfo()\n for (swap,attrs) in sysinfo.getSwapStats().iteritems():\n if attrs['type'] == 'partition':\n dev = self._getUniqueDev(swap)\n if dev is not None:\n self._swapList.append(dev)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _initDiskStats(self):\n self._diskStats = {}\n self._mapMajorMinor2dev = {}\n try:\n fp = open(diskStatsFile, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading disk stats from file: %s'\n % diskStatsFile)\n for line in data.splitlines():\n cols = line.split()\n dev = cols.pop(2)\n if len(cols) == 13:\n self._diskStats[dev] = dict(zip(\n ('major', 'minor',\n 'rios', 'rmerges', 'rsect', 'rticks',\n 'wios', 'wmerges', 'wsect', 'wticks',\n 'ios_active', 'totticks', 'rqticks'),\n [int(x) for x in cols]))\n elif len(cols) == 6:\n self._diskStats[dev] = dict(zip(\n ('major', 'minor',\n 'rios', 'rsect',\n 'wios', 'wsect'),\n [int(x) for x in cols]))\n else:\n continue\n self._diskStats[dev]['rbytes'] = (\n self._diskStats[dev]['rsect'] * sectorSize)\n self._diskStats[dev]['wbytes'] = (\n self._diskStats[dev]['wsect'] * sectorSize)\n self._mapMajorMinor2dev[(int(cols[0]), int(cols[1]))] = dev", "response": "Parse and initialize block device I / O stats in diskstats file."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsort block devices into lists depending on device class and corresponding partition type and initialize device type map and partition map.", "response": "def _initDevClasses(self):\n \"\"\"Sort block devices into lists depending on device class and \n initialize device type map and partition map.\"\"\"\n self._devClassTree = {}\n self._partitionTree = {}\n self._mapDevType = {}\n basedevs = []\n otherdevs = []\n if self._mapMajorDevclass is None:\n self._initBlockMajorMap()\n for dev in self._diskStats:\n stats = self._diskStats[dev]\n devclass = self._mapMajorDevclass.get(stats['major'])\n if devclass is not None:\n devdir = os.path.join(sysfsBlockdevDir, dev)\n if os.path.isdir(devdir):\n if not self._devClassTree.has_key(devclass):\n self._devClassTree[devclass] = []\n self._devClassTree[devclass].append(dev)\n self._mapDevType[dev] = devclass\n basedevs.append(dev)\n else:\n otherdevs.append(dev)\n basedevs.sort(key=len, reverse=True)\n otherdevs.sort(key=len, reverse=True)\n idx = 0\n for partdev in otherdevs:\n while len(basedevs[idx]) > partdev:\n idx += 1 \n for dev in basedevs[idx:]:\n if re.match(\"%s(\\d+|p\\d+)$\" % dev, partdev):\n if not self._partitionTree.has_key(dev):\n self._partitionTree[dev] = []\n self._partitionTree[dev].append(partdev)\n self._mapDevType[partdev] = 'part'"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn type of device dev.", "response": "def getDevType(self, dev):\n \"\"\"Returns type of device dev.\n \n @return: Device type as string.\n \n \"\"\"\n if self._devClassTree is None:\n self._initDevClasses()\n return self._mapDevType.get(dev)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getPartitionList(self):\n if self._partList is None:\n self._partList = []\n for (disk,parts) in self.getPartitionDict().iteritems():\n for part in parts:\n self._partList.append((disk,part))\n return self._partList", "response": "Returns list of partitions."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning I / O stats for the specified device.", "response": "def getDevStats(self, dev, devtype = None):\n \"\"\"Returns I/O stats for block device.\n \n @param dev: Device name\n @param devtype: Device type. (Ignored if None.)\n @return: Dict of stats.\n \n \"\"\"\n if devtype is not None:\n if self._devClassTree is None:\n self._initDevClasses()\n if devtype <> self._mapDevType.get(dev):\n return None\n return self._diskStats.get(dev)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns I / O stats for a given device name.", "response": "def getSwapStats(self, dev):\n \"\"\"Returns I/O stats for swap partition.\n \n @param dev: Device name for swap partition.\n @return: Dict of stats.\n \n \"\"\"\n if self._swapList is None:\n self._initSwapInfo()\n if dev in self._swapList:\n return self.getDevStats(dev)\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getLVstats(self, *args):\n if not len(args) in (1, 2):\n raise TypeError(\"The getLVstats must be called with either \"\n \"one or two arguments.\")\n if self._vgTree is None:\n self._initDMinfo()\n if len(args) == 1:\n dmdev = self._mapLVname2dm.get(args[0])\n else:\n dmdev = self._mapLVtuple2dm.get(args)\n if dmdev is not None:\n return self.getDevStats(dmdev)\n else:\n return None", "response": "Returns I / O stats for LV."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn I / O stats for filesystem.", "response": "def getFilesystemStats(self, fs):\n \"\"\"Returns I/O stats for filesystem.\n \n @param fs: Filesystem path.\n @return: Dict of stats.\n \n \"\"\"\n if self._mapFSpathDev is None:\n self._initFilesystemInfo()\n return self._diskStats.get(self._mapFSpathDev.get(fs))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nretrieve values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n ntpinfo = NTPinfo()\n stats = ntpinfo.getHostOffset(self._remoteHost)\n if stats:\n graph_name = 'ntp_host_stratum_%s' % self._remoteHost\n if self.hasGraph(graph_name):\n self.setGraphVal(graph_name, 'stratum', stats.get('stratum'))\n graph_name = 'ntp_host_stat_%s' % self._remoteHost\n if self.hasGraph(graph_name):\n self.setGraphVal(graph_name, 'offset', stats.get('offset'))\n self.setGraphVal(graph_name, 'delay', stats.get('delay'))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getContainerStats(self, limit=None, marker=None):\n stats = {}\n for row in self._conn.list_containers_info(limit, marker):\n stats[row['name']] = {'count': row['count'], 'size': row['bytes']}\n return stats", "response": "Returns Rackspace Cloud Files usage stats for containers."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef decode_speech(path, keypath=None, save=False, speech_context=None,\n sample_rate=44100, max_alternatives=1, language_code='en-US',\n enable_word_time_offsets=True, return_raw=False):\n \"\"\"\n Decode speech for a file or folder and return results\n\n This function wraps the Google Speech API and ffmpeg to decode speech for\n free recall experiments. Note: in order for this to work, you must have a\n Google Speech account, a google speech credentials file referenced in your\n _bash_profile, and ffmpeg installed on your computer. See our readthedocs\n for more information on how to set this up:\n http://cdl-quail.readthedocs.io/en/latest/.\n\n Parameters\n ----------\n path : str\n Path to a wav file, or a folder of wav files.\n\n keypath : str\n Google Cloud Speech API key filepath. This is a JSON file containing\n credentials that was generated when creating a service account key.\n If None, assumes you have a local key that is set with an environmental\n variable. See the speech decoding tutorial for details.\n\n save : boolean\n False by default, but if set to true, will save a pickle with the results\n object from google speech, and a text file with the decoded words.\n\n speech_context : list of str\n This allows you to give some context to the speech decoding algorithm.\n For example, this could be the words studied on a given list, or all\n words in an experiment.\n\n sample_rate : float\n The sample rate of your audio files (default is 44100).\n\n max_alternatives : int\n You can specify the speech decoding to return multiple guesses to the\n decoding. This will be saved in the results object (default is 1).\n\n language_code : str\n Decoding language code. Default is en-US. See here for more details:\n https://cloud.google.com/speech/docs/languages\n\n enable_word_time_offsets : bool\n Returns timing information s(onsets/offsets) for each word (default is\n True).\n\n return_raw : boolean\n Intead of returning the parsed results objects (i.e. the words), you can\n return the raw reponse object. This has more details about the decoding,\n such as confidence.\n\n Returns\n ----------\n words : list of str, or list of lists of str\n The results of the speech decoding. This will be a list if only one file\n is input, or a list of lists if more than one file is decoded.\n\n raw : google speech object, or list of objects\n You can optionally return the google speech object instead of the parsed\n results by using the return_raw flag.\n\n \"\"\"\n\n # SUBFUNCTIONS\n def decode_file(file_path, client, speech_context, sample_rate,\n max_alternatives, enable_word_time_offsets):\n\n def recognize(chunk, file_path):\n \"\"\"\n Subfunction that loops over audio segments to recognize speech\n \"\"\"\n # export as flac\n chunk.export(file_path + \".flac\", format = \"flac\", bitrate=\"44.1k\")\n\n # open flac file\n with open(file_path + \".flac\", 'rb') as sc:\n speech_content = sc.read()\n\n # initialize speech sample\n sample = types.RecognitionAudio(content=speech_content)\n\n # run speech decoding\n try:\n result = client.recognize(opts, sample)\n except ValueError as e:\n print(e)\n result = None\n\n return result\n\n opts = {}\n opts['encoding']=enums.RecognitionConfig.AudioEncoding.FLAC\n opts['language_code'] = language_code\n opts['sample_rate_hertz'] = sample_rate\n opts['max_alternatives'] = max_alternatives\n opts['enable_word_time_offsets'] = enable_word_time_offsets\n if speech_context:\n opts['speech_contexts']=[types.SpeechContext(phrases=speech_context)]\n\n # read in wav\n audio = AudioSegment.from_wav(file_path)\n\n # segment into 1 minute chunks\n if len(audio)>60000:\n segments = list(range(0,len(audio),60000))\n if segments[-1]1:\n return raw\n else:\n return raw[0]\n else:\n if len(words)>1:\n return words\n else:\n return words[0]", "response": "Decode a file or folder into a single object."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef parse_value(val):\n \n mobj = re.match('(-{0,1}\\d+)\\s*(\\sseconds|/\\s*\\w+)$', val)\n if mobj:\n return int(mobj.group(1))\n mobj = re.match('(-{0,1}\\d*\\.\\d+)\\s*(\\sseconds|/\\s*\\w+)$', val)\n if mobj:\n return float(mobj.group(1))\n re.match('(-{0,1}\\d+)\\s*([GMK])B$', val)\n if mobj:\n return int(mobj.group(1)) * memMultiplier[mobj.group(2)]\n mobj = re.match('(-{0,1}\\d+(\\.\\d+){0,1})\\s*\\%$', val)\n if mobj:\n return float(mobj.group(1)) / 100 \n return val", "response": "Parse input string and return int float or str depending on format.\n "} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _connect(self):\n if sys.version_info[:2] < (2,6):\n self._conn = httplib.HTTPConnection(self._host, self._port)\n else:\n self._conn = httplib.HTTPConnection(self._host, self._port, \n False, defaultTimeout)", "response": "Connect to Squid Proxy Manager interface."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _retrieve(self, map):\n self._conn.request('GET', \"cache_object://%s/%s\" % (self._host, map), \n None, self._httpHeaders)\n rp = self._conn.getresponse()\n if rp.status == 200:\n data = rp.read()\n return data\n else:\n raise Exception(\"Retrieval of stats from Squid Proxy Server\"\n \"on host %s and port %s failed.\\n\"\n \"HTTP - Status: %s Reason: %s\" \n % (self._host, self._port, rp.status, rp.reason))", "response": "Retrieve statistics from Squid Proxy Server Manager Interface Interface."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _parseCounters(self, data):\n info_dict = util.NestedDict()\n for line in data.splitlines():\n mobj = re.match('^\\s*([\\w\\.]+)\\s*=\\s*(\\S.*)$', line)\n if mobj:\n (key, value) = mobj.groups()\n klist = key.split('.')\n info_dict.set_nested(klist, parse_value(value))\n return info_dict", "response": "Parse simple stats list of key value pairs.\nidAdresse"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _parseSections(self, data):\n section_dict = {}\n lines = data.splitlines()\n idx = 0\n numlines = len(lines)\n section = None\n while idx < numlines:\n line = lines[idx]\n idx += 1\n mobj = re.match('^(\\w[\\w\\s\\(\\)]+[\\w\\)])\\s*:$', line)\n if mobj:\n section = mobj.group(1)\n section_dict[section] = []\n else:\n mobj = re.match('(\\t|\\s)\\s*(\\w.*)$', line)\n if mobj:\n section_dict[section].append(mobj.group(2))\n else:\n mobj = re.match('^(\\w[\\w\\s\\(\\)]+[\\w\\)])\\s*:\\s*(\\S.*)$', line)\n if mobj:\n section = None\n if not section_dict.has_key(section):\n section_dict[section] = []\n section_dict[section].append(line)\n else:\n if not section_dict.has_key('PARSEERROR'):\n section_dict['PARSEERROR'] = []\n section_dict['PARSEERROR'].append(line) \n return section_dict", "response": "Parse data and separate sections. Returns dictionary that maps section names to section data."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting manager interface section list from Squid Proxy Server", "response": "def getMenu(self):\n \"\"\"Get manager interface section list from Squid Proxy Server\n \n @return: List of tuples (section, description, type)\n \n \"\"\"\n data = self._retrieve('')\n info_list = []\n for line in data.splitlines():\n mobj = re.match('^\\s*(\\S.*\\S)\\s*\\t\\s*(\\S.*\\S)\\s*\\t\\s*(\\S.*\\S)$', line)\n if mobj:\n info_list.append(mobj.groups())\n return info_list"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns dictionary of Traffic Stats for each Wanpipe Interface.", "response": "def getIfaceStats(self):\n \"\"\"Return dictionary of Traffic Stats for each Wanpipe Interface.\n \n @return: Nested dictionary of statistics for each interface.\n \n \"\"\"\n ifInfo = netiface.NetIfaceInfo()\n ifStats = ifInfo.getIfStats()\n info_dict = {}\n for ifname in ifStats:\n if re.match('^w\\d+g\\d+$', ifname):\n info_dict[ifname] = ifStats[ifname] \n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getPRIstats(self, iface):\n info_dict = {}\n output = util.exec_command([wanpipemonCmd, '-i', iface, '-c', 'Ta'])\n for line in output.splitlines():\n mobj = re.match('^\\s*(Line Code Violation|Far End Block Errors|'\n 'CRC4 Errors|FAS Errors)\\s*:\\s*(\\d+)\\s*$', \n line, re.IGNORECASE)\n if mobj:\n info_dict[mobj.group(1).lower().replace(' ', '')] = int(mobj.group(2))\n continue\n mobj = re.match('^\\s*(Rx Level)\\s*:\\s*>{0,1}\\s*([-\\d\\.]+)db\\s*', \n line, re.IGNORECASE)\n if mobj:\n info_dict[mobj.group(1).lower().replace(' ', '')] = float(mobj.group(2))\n continue\n return info_dict", "response": "Return RDSI Operational Stats for given interface."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconnecting to FreeSWITCH ESL Interface.", "response": "def _connect(self):\n \"\"\"Connect to FreeSWITCH ESL Interface.\"\"\"\n try:\n self._eslconn = ESL.ESLconnection(self._eslhost, \n str(self._eslport), \n self._eslpass)\n except:\n pass\n if not self._eslconn.connected():\n raise Exception(\n \"Connection to FreeSWITCH ESL Interface on host %s and port %d failed.\"\n % (self._eslhost, self._eslport)\n )"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute command and return result body as list of lines.", "response": "def _execCmd(self, cmd, args):\n \"\"\"Execute command and return result body as list of lines.\n \n @param cmd: Command string.\n @param args: Comand arguments string. \n @return: Result dictionary.\n \n \"\"\"\n output = self._eslconn.api(cmd, args)\n if output:\n body = output.getBody()\n if body:\n return body.splitlines()\n return None"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _execShowCmd(self, showcmd):\n result = None\n lines = self._execCmd(\"show\", showcmd)\n if lines and len(lines) >= 2 and lines[0] != '' and lines[0][0] != '-':\n result = {}\n result['keys'] = lines[0].split(',')\n items = []\n for line in lines[1:]:\n if line == '':\n break\n items.append(line.split(','))\n result['items'] = items\n return result", "response": "Execute show command and return result dictionary."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _execShowCountCmd(self, showcmd):\n result = None\n lines = self._execCmd(\"show\", showcmd + \" count\")\n for line in lines:\n mobj = re.match('\\s*(\\d+)\\s+total', line)\n if mobj:\n return int(mobj.group(1))\n return result", "response": "Execute show count command and return result dictionary.\nalidisoiso"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nretrieving values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n if self.hasGraph('asterisk_calls') or self.hasGraph('asterisk_channels'):\n stats = self._ami.getChannelStats(self._chanList)\n if self.hasGraph('asterisk_calls') and stats:\n self.setGraphVal('asterisk_calls', 'active_calls', \n stats.get('active_calls'))\n self.setGraphVal('asterisk_calls', 'calls_per_min', \n stats.get('calls_processed'))\n if self.hasGraph('asterisk_channels') and stats:\n for field in self._chanList:\n self.setGraphVal('asterisk_channels', \n field, stats.get(field))\n if 'dahdi' in self._chanList:\n self.setGraphVal('asterisk_channels', \n 'mix', stats.get('mix'))\n\n if self.hasGraph('asterisk_peers_sip'):\n stats = self._ami.getPeerStats('sip')\n if stats:\n for field in ('online', 'unmonitored', 'unreachable', \n 'lagged', 'unknown'):\n self.setGraphVal('asterisk_peers_sip', \n field, stats.get(field))\n \n if self.hasGraph('asterisk_peers_iax2'):\n stats = self._ami.getPeerStats('iax2')\n if stats:\n for field in ('online', 'unmonitored', 'unreachable', \n 'lagged', 'unknown'):\n self.setGraphVal('asterisk_peers_iax2', \n field, stats.get(field))\n \n if self.hasGraph('asterisk_voip_codecs'):\n sipstats = self._ami.getVoIPchanStats('sip', self._codecList) or {}\n iax2stats = self._ami.getVoIPchanStats('iax2', self._codecList) or {}\n if stats:\n for field in self._codecList:\n self.setGraphVal('asterisk_voip_codecs', field,\n sipstats.get(field,0) \n + iax2stats.get(field, 0))\n self.setGraphVal('asterisk_voip_codecs', 'other',\n sipstats.get('other', 0) \n + iax2stats.get('other', 0))\n \n if self.hasGraph('asterisk_conferences'):\n stats = self._ami.getConferenceStats()\n if stats:\n self.setGraphVal('asterisk_conferences', 'rooms', \n stats.get('active_conferences'))\n self.setGraphVal('asterisk_conferences', 'users', \n stats.get('conference_users'))\n\n if self.hasGraph('asterisk_voicemail'):\n stats = self._ami.getVoicemailStats()\n if stats:\n self.setGraphVal('asterisk_voicemail', 'accounts', \n stats.get('accounts'))\n self.setGraphVal('asterisk_voicemail', 'msg_avg', \n stats.get('avg_messages'))\n self.setGraphVal('asterisk_voicemail', 'msg_max', \n stats.get('max_messages'))\n self.setGraphVal('asterisk_voicemail', 'msg_total', \n stats.get('total_messages'))\n\n if self.hasGraph('asterisk_trunks') and len(self._trunkList) > 0:\n stats = self._ami.getTrunkStats(self._trunkList)\n for trunk in self._trunkList:\n self.setGraphVal('asterisk_trunks', trunk[0], \n stats.get(trunk[0]))\n \n if self._queues is not None:\n total_answer = 0\n total_abandon = 0\n for queue in self._queue_list:\n stats = self._queues[queue]\n if self.hasGraph('asterisk_queue_len'):\n self.setGraphVal('asterisk_queue_len', queue,\n stats.get('queue_len'))\n if self.hasGraph('asterisk_queue_avg_hold'):\n self.setGraphVal('asterisk_queue_avg_hold', queue,\n stats.get('avg_holdtime'))\n if self.hasGraph('asterisk_queue_avg_talk'):\n self.setGraphVal('asterisk_queue_avg_talk', queue,\n stats.get('avg_talktime'))\n if self.hasGraph('asterisk_queue_calls'):\n total_abandon += stats.get('calls_abandoned')\n total_answer += stats.get('calls_completed')\n if self.hasGraph('asterisk_queue_abandon_pcent'):\n prev_stats = self._queues_prev.get(queue)\n if prev_stats is not None:\n abandon = (stats.get('calls_abandoned', 0) -\n prev_stats.get('calls_abandoned', 0))\n answer = (stats.get('calls_completed', 0) -\n prev_stats.get('calls_completed', 0))\n total = abandon + answer \n if total > 0:\n val = 100.0 * float(abandon) / float(total)\n else:\n val = 0\n self.setGraphVal('asterisk_queue_abandon_pcent', \n queue, val)\n if self.hasGraph('asterisk_queue_calls'):\n self.setGraphVal('asterisk_queue_calls', 'abandon', \n total_abandon)\n self.setGraphVal('asterisk_queue_calls', 'answer', \n total_answer)\n \n if self.hasGraph('asterisk_fax_attempts'):\n fax_stats = self._ami.getFaxStatsCounters()\n stats = fax_stats.get('general')\n if stats is not None:\n self.setGraphVal('asterisk_fax_attempts', 'send', \n stats.get('transmit attempts'))\n self.setGraphVal('asterisk_fax_attempts', 'recv', \n stats.get('receive attempts'))\n self.setGraphVal('asterisk_fax_attempts', 'fail', \n stats.get('failed faxes'))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef retrieveVals(self):\n fpminfo = PHPfpmInfo(self._host, self._port, self._user, self._password, \n self._monpath, self._ssl)\n stats = fpminfo.getStats()\n if self.hasGraph('php_fpm_connections') and stats: \n self.setGraphVal('php_fpm_connections', 'conn', \n stats['accepted conn'])\n if self.hasGraph('php_fpm_processes') and stats: \n self.setGraphVal('php_fpm_processes', 'active', \n stats['active processes'])\n self.setGraphVal('php_fpm_processes', 'idle', \n stats['idle processes'])\n self.setGraphVal('php_fpm_processes', 'total', \n stats['total processes'])", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nimplements Munin Plugin Auto - Configuration Option.", "response": "def autoconf(self):\n \"\"\"Implements Munin Plugin Auto-Configuration Option.\n \n @return: True if plugin can be auto-configured, False otherwise.\n \n \"\"\"\n fpminfo = PHPfpmInfo(self._host, self._port, self._user, self._password, \n self._monpath, self._ssl)\n return fpminfo is not None"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\npinging Redis Server and return Round - Trip - Time in seconds as float.", "response": "def ping(self):\n \"\"\"Ping Redis Server and return Round-Trip-Time in seconds.\n \n @return: Round-trip-time in seconds as float.\n \n \"\"\"\n start = time.time()\n self._conn.ping()\n return (time.time() - start)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntake a list of lists of strings and lists of strings and returns a multi - indexed dataframe of subject data", "response": "def list2pd(all_data, subjindex=None, listindex=None):\n \"\"\"\n Makes multi-indexed dataframe of subject data\n\n Parameters\n ----------\n all_data : list of lists of strings\n strings are either all presented or all recalled items, in the order of presentation or recall\n *should also work for presented / recalled ints and floats, if desired\n\n\n Returns\n ----------\n subs_list_of_dfs : multi-indexed dataframe\n dataframe of subject data (presented or recalled words/items), indexed by subject and list number\n cell populated by the term presented or recalled in the position indicated by the column number\n\n \"\"\"\n # set default index if it is not defined\n # max_nlists = max(map(lambda x: len(x), all_data))\n listindex = [[idx for idx in range(len(sub))] for sub in all_data] if not listindex else listindex\n subjindex = [idx for idx,subj in enumerate(all_data)] if not subjindex else subjindex\n\n\n def make_multi_index(listindex, sub_num):\n return pd.MultiIndex.from_tuples([(sub_num,lst) for lst in listindex], names = ['Subject', 'List'])\n\n listindex = list(listindex)\n subjindex = list(subjindex)\n\n subs_list_of_dfs = [pd.DataFrame(sub_data, index=make_multi_index(listindex[sub_num], subjindex[sub_num])) for sub_num,sub_data in enumerate(all_data)]\n\n return pd.concat(subs_list_of_dfs)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning an Egg object from a zero - indexed recall matrix.", "response": "def recmat2egg(recmat, list_length=None):\n \"\"\"\n Creates egg data object from zero-indexed recall matrix\n\n Parameters\n ----------\n recmat : list of lists (subs) of lists (encoding lists) of ints or 2D numpy array\n recall matrix representing serial positions of freely recalled words \\\n e.g. [[[16, 15, 0, 2, 3, None, None...], [16, 4, 5, 6, 1, None, None...]]]\n\n list_length : int\n The length of each list (e.g. 16)\n\n\n Returns\n ----------\n egg : Egg data object\n egg data object computed from the recall matrix\n \"\"\"\n from .egg import Egg as Egg\n\n pres = [[[str(word) for word in list(range(0,list_length))] for reclist in recsub] for recsub in recmat]\n rec = [[[str(word) for word in reclist if word is not None] for reclist in recsub] for recsub in recmat]\n\n return Egg(pres=pres,rec=rec)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef default_dist_funcs(dist_funcs, feature_example):\n\n if dist_funcs is None:\n dist_funcs = dict()\n\n for key in feature_example:\n if key in dist_funcs:\n pass\n if key == 'item':\n pass\n elif isinstance(feature_example[key], (six.string_types, six.binary_type)):\n dist_funcs[key] = 'match'\n elif isinstance(feature_example[key], (int, np.integer, float)) or all([isinstance(i, (int, np.integer, float)) for i in feature_example[key]]):\n dist_funcs[key] = 'euclidean'\n\n return dist_funcs", "response": "Fill in default distance metrics for fingerprint analyses\n "} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef stack_eggs(eggs, meta='concatenate'):\n '''\n Takes a list of eggs, stacks them and reindexes the subject number\n\n Parameters\n ----------\n eggs : list of Egg data objects\n A list of Eggs that you want to combine\n meta : string\n Determines how the meta data of each Egg combines. Default is 'concatenate'\n 'concatenate' concatenates keys in meta data dictionary shared between eggs, and copies non-overlapping keys\n 'separate' keeps the Eggs' meta data dictionaries separate, with each as a list index in the stacked meta data\n\n\n Returns\n ----------\n new_egg : Egg data object\n A mega egg comprised of the input eggs stacked together\n\n '''\n from .egg import Egg\n\n pres = [egg.pres.loc[sub,:].values.tolist() for egg in eggs for sub in egg.pres.index.levels[0].values.tolist()]\n rec = [egg.rec.loc[sub,:].values.tolist() for egg in eggs for sub in egg.rec.index.levels[0].values.tolist()]\n\n if meta is 'concatenate':\n new_meta = {}\n for egg in eggs:\n for key in egg.meta:\n if key in new_meta:\n new_meta[key] = list(new_meta[key])\n new_meta[key].extend(egg.meta.get(key))\n else:\n new_meta[key] = egg.meta.get(key)\n\n elif meta is 'separate':\n new_meta = list(egg.meta for egg in eggs)\n\n return Egg(pres=pres, rec=rec, meta=new_meta)", "response": "Takes a list of eggs stacks them and reindexes the subject number\n "} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef crack_egg(egg, subjects=None, lists=None):\n '''\n Takes an egg and returns a subset of the subjects or lists\n\n Parameters\n ----------\n egg : Egg data object\n Egg that you want to crack\n\n subjects : list\n List of subject idxs\n\n lists : list\n List of lists idxs\n\n Returns\n ----------\n new_egg : Egg data object\n A sliced egg, good on a salad\n\n '''\n from .egg import Egg\n\n if hasattr(egg, 'features'):\n all_have_features = egg.features is not None\n else:\n all_have_features=False\n opts = {}\n\n if subjects is None:\n subjects = egg.pres.index.levels[0].values.tolist()\n elif type(subjects) is not list:\n subjects = [subjects]\n\n if lists is None:\n lists = egg.pres.index.levels[1].values.tolist()\n elif type(lists) is not list:\n lists = [lists]\n\n idx = pd.IndexSlice\n pres = egg.pres.loc[idx[subjects,lists],egg.pres.columns]\n rec = egg.rec.loc[idx[subjects,lists],egg.rec.columns]\n\n pres = [pres.loc[sub,:].values.tolist() for sub in subjects]\n rec = [rec.loc[sub,:].values.tolist() for sub in subjects]\n\n if all_have_features:\n features = egg.features.loc[idx[subjects,lists],egg.features.columns]\n opts['features'] = [features.loc[sub,:].values.tolist() for sub in subjects]\n\n return Egg(pres=pres, rec=rec, **opts)", "response": "Takes an egg and returns a subset of the subjects or lists idxs\n\n Returns a new egg with the same contents"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert a MultiIndex DataFrame to list of lists of lists of values", "response": "def df2list(df):\n \"\"\"\n Convert a MultiIndex df to list\n\n Parameters\n ----------\n\n df : pandas.DataFrame\n A MultiIndex DataFrame where the first level is subjects and the second\n level is lists (e.g. egg.pres)\n\n Returns\n ----------\n\n lst : a list of lists of lists of values\n The input df reformatted as a list\n\n \"\"\"\n subjects = df.index.levels[0].values.tolist()\n lists = df.index.levels[1].values.tolist()\n idx = pd.IndexSlice\n df = df.loc[idx[subjects,lists],df.columns]\n lst = [df.loc[sub,:].values.tolist() for sub in subjects]\n return lst"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfill in missing lists", "response": "def fill_missing(x):\n \"\"\"\n Fills in missing lists (assumes end lists are missing)\n \"\"\"\n\n # find subject with max number of lists\n maxlen = max([len(xi) for xi in x])\n\n subs = []\n\n for sub in x:\n if len(sub)= 3:\n return [float(col) for col in arr[:3]]\n else:\n return None", "response": "Return system Load Average."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getCPUuse(self):\n hz = os.sysconf('SC_CLK_TCK')\n info_dict = {}\n try:\n fp = open(cpustatFile, 'r')\n line = fp.readline()\n fp.close()\n except:\n raise IOError('Failed reading stats from file: %s' % cpustatFile)\n headers = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq', 'steal', 'guest']\n arr = line.split()\n if len(arr) > 1 and arr[0] == 'cpu':\n return dict(zip(headers[0:len(arr)], [(float(t) / hz) for t in arr[1:]]))\n return info_dict", "response": "Return cpu time utilization in seconds."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getProcessStats(self):\n info_dict = {}\n try:\n fp = open(cpustatFile, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading stats from file: %s' % cpustatFile)\n for line in data.splitlines():\n arr = line.split()\n if len(arr) > 1 and arr[0] in ('ctxt', 'intr', 'softirq',\n 'processes', 'procs_running', \n 'procs_blocked'):\n info_dict[arr[0]] = arr[1]\n return info_dict", "response": "Return stats for running processes forks context switches and interrupts."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getMemoryUse(self):\n info_dict = {}\n try:\n fp = open(meminfoFile, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading stats from file: %s' % meminfoFile)\n for line in data.splitlines():\n mobj = re.match('^(.+):\\s*(\\d+)\\s*(\\w+|)\\s*$', line)\n if mobj:\n if mobj.group(3).lower() == 'kb':\n mult = 1024\n else:\n mult = 1\n info_dict[mobj.group(1)] = int(mobj.group(2)) * mult\n return info_dict", "response": "Return stats for memory utilization.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns information on swap partition and files.", "response": "def getSwapStats(self):\n \"\"\"Return information on swap partition and / or files.\n \n @return: Dictionary of stats.\n \n \"\"\"\n info_dict = {}\n try:\n fp = open(swapsFile, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading stats from file: %s' % swapsFile)\n lines = data.splitlines()\n if len(lines) > 1:\n colnames = [name.lower() for name in lines[0].split()]\n for line in lines[1:]:\n cols = line.split()\n info_dict[cols[0]] = dict(zip(colnames[1:], cols[1:]))\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getVMstats(self):\n info_dict = {}\n try:\n fp = open(vmstatFile, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading stats from file: %s' % vmstatFile)\n for line in data.splitlines():\n cols = line.split()\n if len(cols) == 2:\n info_dict[cols[0]] = cols[1]\n return info_dict", "response": "Return stats for Virtual Memory Subsystem.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _sendStatCmd(self, cmd):\n try:\n self._conn.write(\"%s\\r\\n\" % cmd)\n regex = re.compile('^(END|ERROR)\\r\\n', re.MULTILINE)\n (idx, mobj, text) = self._conn.expect([regex,], self._timeout) #@UnusedVariable\n except:\n raise Exception(\"Communication with %s failed\" % self._instanceName)\n if mobj is not None:\n if mobj.group(1) == 'END':\n return text.splitlines()[:-1]\n elif mobj.group(1) == 'ERROR':\n raise Exception(\"Protocol error in communication with %s.\"\n % self._instanceName)\n else:\n raise Exception(\"Connection with %s timed out.\" % self._instanceName)", "response": "Send stat command to Memcached Server and return response lines."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _parseStats(self, lines, parse_slabs = False):\n info_dict = {}\n info_dict['slabs'] = {}\n for line in lines:\n mobj = re.match('^STAT\\s(\\w+)\\s(\\S+)$', line)\n if mobj:\n info_dict[mobj.group(1)] = util.parse_value(mobj.group(2), True)\n continue\n elif parse_slabs:\n mobj = re.match('STAT\\s(\\w+:)?(\\d+):(\\w+)\\s(\\S+)$', line)\n if mobj:\n (slab, key, val) = mobj.groups()[-3:] \n if not info_dict['slabs'].has_key(slab):\n info_dict['slabs'][slab] = {}\n info_dict['slabs'][slab][key] = util.parse_value(val, True)\n return info_dict", "response": "Parse memcached stats output from memcached and return dictionary of stats - nanos."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n if self._stats is None:\n serverInfo = MemcachedInfo(self._host, self._port, self._socket_file)\n stats = serverInfo.getStats()\n else:\n stats = self._stats\n if stats is None:\n raise Exception(\"Undetermined error accesing stats.\") \n stats['set_hits'] = stats.get('total_items')\n if stats.has_key('cmd_set') and stats.has_key('total_items'): \n stats['set_misses'] = stats['cmd_set'] - stats['total_items']\n self.saveState(stats)\n if self.hasGraph('memcached_connections'):\n self.setGraphVal('memcached_connections', 'conn', \n stats.get('curr_connections'))\n if self.hasGraph('memcached_items'):\n self.setGraphVal('memcached_items', 'items', \n stats.get('curr_items'))\n if self.hasGraph('memcached_memory'):\n self.setGraphVal('memcached_memory', 'bytes', \n stats.get('bytes'))\n if self.hasGraph('memcached_connrate'):\n self.setGraphVal('memcached_connrate', 'conn', \n stats.get('total_connections'))\n if self.hasGraph('memcached_traffic'):\n self.setGraphVal('memcached_traffic', 'rxbytes', \n stats.get('bytes_read'))\n self.setGraphVal('memcached_traffic', 'txbytes', \n stats.get('bytes_written'))\n if self.hasGraph('memcached_reqrate'):\n self.setGraphVal('memcached_reqrate', 'set', \n stats.get('cmd_set'))\n self.setGraphVal('memcached_reqrate', 'get', \n stats.get('cmd_get'))\n if self.graphHasField('memcached_reqrate', 'del'):\n self.setGraphVal('memcached_reqrate', 'del',\n safe_sum([stats.get('delete_hits'), \n stats.get('delete_misses')]))\n if self.graphHasField('memcached_reqrate', 'cas'):\n self.setGraphVal('memcached_reqrate', 'cas',\n safe_sum([stats.get('cas_hits'), \n stats.get('cas_misses'), \n stats.get('cas_badval')]))\n if self.graphHasField('memcached_reqrate', 'incr'):\n self.setGraphVal('memcached_reqrate', 'incr',\n safe_sum([stats.get('incr_hits'), \n stats.get('incr_misses')]))\n if self.graphHasField('memcached_reqrate', 'decr'):\n self.setGraphVal('memcached_reqrate', 'decr',\n safe_sum([stats.get('decr_hits'), \n stats.get('decr_misses')]))\n if self.hasGraph('memcached_statget'):\n self.setGraphVal('memcached_statget', 'hit', \n stats.get('get_hits'))\n self.setGraphVal('memcached_statget', 'miss', \n stats.get('get_misses'))\n self.setGraphVal('memcached_statget', 'total', \n safe_sum([stats.get('get_hits'),\n stats.get('get_misses')]))\n if self.hasGraph('memcached_statset'):\n self.setGraphVal('memcached_statset', 'hit', \n stats.get('set_hits'))\n self.setGraphVal('memcached_statset', 'miss', \n stats.get('set_misses'))\n self.setGraphVal('memcached_statset', 'total', \n safe_sum([stats.get('set_hits'),\n stats.get('set_misses')]))\n if self.hasGraph('memcached_statdel'):\n self.setGraphVal('memcached_statdel', 'hit', \n stats.get('delete_hits'))\n self.setGraphVal('memcached_statdel', 'miss', \n stats.get('delete_misses'))\n self.setGraphVal('memcached_statdel', 'total', \n safe_sum([stats.get('delete_hits'),\n stats.get('delete_misses')]))\n if self.hasGraph('memcached_statcas'):\n self.setGraphVal('memcached_statcas', 'hit', \n stats.get('cas_hits'))\n self.setGraphVal('memcached_statcas', 'miss', \n stats.get('cas_misses'))\n self.setGraphVal('memcached_statcas', 'badval', \n stats.get('cas_badval'))\n self.setGraphVal('memcached_statcas', 'total', \n safe_sum([stats.get('cas_hits'),\n stats.get('cas_misses'),\n stats.get('cas_badval')]))\n if self.hasGraph('memcached_statincrdecr'):\n self.setGraphVal('memcached_statincrdecr', 'incr_hit', \n stats.get('incr_hits'))\n self.setGraphVal('memcached_statincrdecr', 'decr_hit', \n stats.get('decr_hits'))\n self.setGraphVal('memcached_statincrdecr', 'incr_miss', \n stats.get('incr_misses'))\n self.setGraphVal('memcached_statincrdecr', 'decr_miss', \n stats.get('decr_misses'))\n self.setGraphVal('memcached_statincrdecr', 'total', \n safe_sum([stats.get('incr_hits'),\n stats.get('decr_hits'),\n stats.get('incr_misses'),\n stats.get('decr_misses')]))\n if self.hasGraph('memcached_statevict'):\n self.setGraphVal('memcached_statevict', 'evict', \n stats.get('evictions'))\n if self.graphHasField('memcached_statevict', 'reclaim'):\n self.setGraphVal('memcached_statevict', 'reclaim', \n stats.get('reclaimed'))\n if self.hasGraph('memcached_statauth'):\n self.setGraphVal('memcached_statauth', 'reqs', \n stats.get('auth_cmds'))\n self.setGraphVal('memcached_statauth', 'errors', \n stats.get('auth_errors'))\n if self.hasGraph('memcached_hitpct'):\n prev_stats = self._prev_stats\n for (field_name, field_hits, field_misses) in (\n ('set', 'set_hits', 'set_misses'),\n ('get', 'get_hits', 'get_misses'), \n ('del', 'delete_hits', 'delete_misses'), \n ('cas', 'cas_hits', 'cas_misses'), \n ('incr', 'incr_hits', 'incr_misses'), \n ('decr', 'decr_hits', 'decr_misses')\n ):\n if prev_stats:\n if (stats.has_key(field_hits) \n and prev_stats.has_key(field_hits)\n and stats.has_key(field_misses) \n and prev_stats.has_key(field_misses)):\n hits = stats[field_hits] - prev_stats[field_hits]\n misses = stats[field_misses] - prev_stats[field_misses]\n total = hits + misses\n if total > 0:\n val = 100.0 * hits / total\n else:\n val = 0\n self.setGraphVal('memcached_hitpct', field_name, \n round(val, 2))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nimplement Munin Plugin Auto - Configuration Option.", "response": "def autoconf(self):\n \"\"\"Implements Munin Plugin Auto-Configuration Option.\n \n @return: True if plugin can be auto-configured, False otherwise.\n \n \"\"\"\n serverInfo = MemcachedInfo(self._host, self._port, self._socket_file)\n return (serverInfo is not None)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns correlation distance between a and b", "response": "def correlation(a, b):\n \"Returns correlation distance between a and b\"\n if isinstance(a, list):\n a = np.array(a)\n if isinstance(b, list):\n b = np.array(b)\n a = a.reshape(1, -1)\n b = b.reshape(1, -1)\n return cdist(a, b, 'correlation')"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef euclidean(a, b):\n \"Returns euclidean distance between a and b\"\n return np.linalg.norm(np.subtract(a, b))", "response": "Returns euclidean distance between a and b"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute ps command with positional params args and return result as list of lines.", "response": "def execProcCmd(self, *args):\n \"\"\"Execute ps command with positional params args and return result as \n list of lines.\n \n @param *args: Positional params for ps command.\n @return: List of output lines\n \n \"\"\"\n out = util.exec_command([psCmd,] + list(args))\n return out.splitlines()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nexecute ps command with custom output format with columns from fields and return result as a nested list of rows and columns.", "response": "def parseProcCmd(self, fields=('pid', 'user', 'cmd',), threads=False):\n \"\"\"Execute ps command with custom output format with columns from \n fields and return result as a nested list.\n \n The Standard Format Specifiers from ps man page must be used for the\n fields parameter.\n \n @param fields: List of fields included in the output.\n Default: pid, user, cmd\n @param threads: If True, include threads in output. \n @return: List of headers and list of rows and columns.\n \n \"\"\"\n args = []\n headers = [f.lower() for f in fields]\n args.append('--no-headers')\n args.append('-e')\n if threads:\n args.append('-T')\n field_ranges = []\n fmt_strs = []\n start = 0\n for header in headers:\n field_width = psFieldWidth.get(header, psDefaultFieldWidth)\n fmt_strs.append('%s:%d' % (header, field_width))\n end = start + field_width + 1\n field_ranges.append((start,end))\n start = end\n args.append('-o')\n args.append(','.join(fmt_strs))\n lines = self.execProcCmd(*args)\n if len(lines) > 0:\n stats = []\n for line in lines:\n cols = []\n for (start, end) in field_ranges:\n cols.append(line[start:end].strip())\n stats.append(cols)\n return {'headers': headers, 'stats': stats}\n else:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getProcList(self, fields=('pid', 'user', 'cmd',), threads=False,\n **kwargs):\n \"\"\"Execute ps command with custom output format with columns columns \n from fields, select lines using the filters defined by kwargs and return \n result as a nested list.\n \n The Standard Format Specifiers from ps man page must be used for the\n fields parameter.\n \n @param fields: Fields included in the output.\n Default: pid, user, cmd\n @param threads: If True, include threads in output.\n @param **kwargs: Keyword variables are used for filtering the results\n depending on the values of the columns. Each keyword \n must correspond to a field name with an optional \n suffix:\n field: Field equal to value or in list of \n values.\n field_ic: Field equal to value or in list of \n values, using case insensitive \n comparison.\n field_regex: Field matches regex value or matches\n with any regex in list of values.\n field_ic_regex: Field matches regex value or matches\n with any regex in list of values \n using case insensitive match. \n @return: List of headers and list of rows and columns.\n \n \"\"\"\n field_list = list(fields)\n for key in kwargs:\n col = re.sub('(_ic)?(_regex)?$', '', key)\n if not col in field_list:\n field_list.append(col)\n pinfo = self.parseProcCmd(field_list, threads)\n if pinfo:\n if len(kwargs) > 0:\n pfilter = util.TableFilter()\n pfilter.registerFilters(**kwargs)\n stats = pfilter.applyFilters(pinfo['headers'], pinfo['stats'])\n return {'headers': pinfo['headers'], 'stats': stats}\n else:\n return pinfo\n else:\n return None", "response": "Execute ps command with custom output format with columns \n from fields and return a nested list of rows and columns."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getProcDict(self, fields=('user', 'cmd',), threads=False, **kwargs):\n stats = {}\n field_list = list(fields)\n num_cols = len(field_list)\n if threads:\n key = 'spid'\n else:\n key = 'pid'\n try:\n key_idx = field_list.index(key)\n except ValueError:\n field_list.append(key)\n key_idx = len(field_list) - 1\n result = self.getProcList(field_list, threads, **kwargs)\n if result is not None:\n headers = result['headers'][:num_cols]\n lines = result['stats']\n if len(lines) > 1:\n for cols in lines:\n stats[cols[key_idx]] = dict(zip(headers, cols[:num_cols]))\n return stats\n else:\n return None", "response": "Execute ps command with custom output format with columns from fields and return result as a nested dictionary with the keys PID and SPID."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a dictionary of process counts per status and priority.", "response": "def getProcStatStatus(self, threads=False, **kwargs):\n \"\"\"Return process counts per status and priority.\n \n @param **kwargs: Keyword variables are used for filtering the results\n depending on the values of the columns. Each keyword \n must correspond to a field name with an optional \n suffix:\n field: Field equal to value or in list of \n values.\n field_ic: Field equal to value or in list of \n values, using case insensitive \n comparison.\n field_regex: Field matches regex value or matches\n with any regex in list of values.\n field_ic_regex: Field matches regex value or matches\n with any regex in list of values \n using case insensitive match.\n @return: Dictionary of process counters.\n \n \"\"\"\n procs = self.getProcList(['stat',], threads=threads, **kwargs)\n status = dict(zip(procStatusNames.values(), \n [0,] * len(procStatusNames)))\n prio = {'high': 0, 'low': 0, 'norm': 0, 'locked_in_mem': 0}\n total = 0\n locked_in_mem = 0\n if procs is not None:\n for cols in procs['stats']:\n col_stat = cols[0]\n status[procStatusNames[col_stat[0]]] += 1\n if '<' in col_stat[1:]:\n prio['high'] += 1\n elif 'N' in col_stat[1:]:\n prio['low'] += 1\n else:\n prio['norm'] += 1\n if 'L' in col_stat[1:]:\n locked_in_mem += 1\n total += 1\n return {'status': status, \n 'prio': prio, \n 'locked_in_mem': locked_in_mem, \n 'total': total}"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef retrieveVals(self):\n ntpinfo = NTPinfo()\n stats = ntpinfo.getPeerStats()\n if stats:\n if self.hasGraph('ntp_peer_stratum'):\n self.setGraphVal('ntp_peer_stratum', 'stratum', \n stats.get('stratum'))\n if self.hasGraph('ntp_peer_stats'):\n self.setGraphVal('ntp_peer_stats', 'offset', \n stats.get('offset'))\n self.setGraphVal('ntp_peer_stats', 'delay', \n stats.get('delay'))\n self.setGraphVal('ntp_peer_stats', 'jitter', \n stats.get('jitter'))", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef fixLabel(label, maxlen, delim=None, repl='', truncend=True):\n if len(label) <= maxlen:\n return label\n else:\n maxlen -= len(repl)\n if delim is not None: \n if truncend:\n end = label.rfind(delim, 0, maxlen)\n if end > 0:\n return label[:end+1] + repl\n else:\n start = label.find(delim, len(label) - maxlen)\n if start > 0:\n return repl + label[start:]\n if truncend:\n return label[:maxlen] + repl\n else:\n return repl + label[-maxlen:]", "response": "Truncate long graph and field labels."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _getGraph(self, graph_name, fail_noexist=False):\n graph = self._graphDict.get(graph_name)\n if fail_noexist and graph is None:\n raise AttributeError(\"Invalid graph name: %s\" % graph_name)\n else:\n return graph", "response": "Private method for returning graph object with name graph_name. \n "} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _getMultigraphID(self, graph_name, subgraph_name=None):\n if self.isMultiInstance and self._instanceName is not None:\n if subgraph_name is None:\n return \"%s_%s\" % (graph_name, self._instanceName)\n else:\n return \"%s_%s.%s_%s\" % (graph_name, self._instanceName, \n subgraph_name, self._instanceName)\n else:\n if subgraph_name is None:\n return graph_name\n else:\n return \"%s.%s\" % (graph_name, subgraph_name)", "response": "Private method for generating Multigraph ID from graph name and subgraph name."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _formatConfig(self, conf_dict):\n confs = []\n graph_dict = conf_dict['graph']\n field_list = conf_dict['fields']\n \n # Order and format Graph Attributes\n title = graph_dict.get('title')\n if title is not None:\n if self.isMultiInstance and self._instanceLabel is not None:\n if self._instanceLabelType == 'suffix':\n confs.append(\"graph_%s %s - %s\" % ('title', \n title, \n self._instanceLabel,))\n elif self._instanceLabelType == 'prefix':\n confs.append(\"graph_%s %s - %s\" % ('title', \n self._instanceLabel,\n title,))\n else:\n confs.append(\"graph_%s %s\" % ('title', title))\n for key in ('category', 'vlabel', 'info', 'args', 'period', \n 'scale', 'total', 'order', 'printf', 'width', 'height'):\n val = graph_dict.get(key)\n if val is not None:\n if isinstance(val, bool):\n if val:\n val = \"yes\"\n else:\n val = \"no\"\n confs.append(\"graph_%s %s\" % (key, val))\n\n # Order and Format Field Attributes\n for (field_name, field_attrs) in field_list:\n for key in ('label', 'type', 'draw', 'info', 'extinfo', 'colour',\n 'negative', 'graph', 'min', 'max', 'cdef', \n 'line', 'warning', 'critical'):\n val = field_attrs.get(key)\n if val is not None:\n if isinstance(val, bool):\n if val:\n val = \"yes\"\n else:\n val = \"no\"\n confs.append(\"%s.%s %s\" % (field_name, key, val))\n return \"\\n\".join(confs)", "response": "Formats the configuration dictionary into a list of multi - line text entries."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn value for environment variable or None. unityName", "response": "def envGet(self, name, default=None, conv=None):\n \"\"\"Return value for environment variable or None. \n \n @param name: Name of environment variable.\n @param default: Default value if variable is undefined.\n @param conv: Function for converting value to desired type.\n @return: Value of environment variable.\n \n \"\"\"\n if self._env.has_key(name):\n if conv is not None:\n return conv(self._env.get(name))\n else:\n return self._env.get(name)\n else:\n return default"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef envGetList(self, name, attr_regex = '^\\w+$', conv=None):\n key = \"list_%s\" % name\n item_list = []\n if self._env.has_key(key):\n if attr_regex:\n recomp = re.compile(attr_regex)\n else:\n recomp = None\n for attr in self._env[key].split(','):\n attr = attr.strip()\n if recomp is None or recomp.search(attr):\n if conv is not None:\n item_list.append(conv(attr))\n else:\n item_list.append(attr)\n \n return item_list", "response": "Parse the environment variables to return list of items."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef envRegisterFilter(self, name, attr_regex = '^\\w+$', default = True):\n attrs = {}\n for prefix in ('include', 'exclude'):\n key = \"%s_%s\" % (prefix, name)\n val = self._env.get(key)\n if val:\n attrs[prefix] = [attr.strip() for attr in val.split(',')]\n else:\n attrs[prefix] = []\n self._filters[name] = MuninAttrFilter(attrs['include'], attrs['exclude'], \n attr_regex, default)", "response": "Register filter for including excluding attributes in graphs through \n the use of include_ environment variables."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck if a specific graph attribute is enabled or disabled through the use of a filter based on include_ and exclude_ environment variables.", "response": "def envCheckFilter(self, name, attr):\n \"\"\"Check if a specific graph attribute is enabled or disabled through \n the use of a filter based on include_ and exclude_ \n environment variables.\n \n @param name: Name of the Filter.\n @param attr: Name of the Attribute.\n @return: Return True if the attribute is enabled.\n \n \"\"\"\n flt = self._filters.get(name)\n if flt:\n return flt.check(attr) \n else:\n raise AttributeError(\"Undefined filter: %s\" % name)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef envCheckFlag(self, name, default = False):\n if self._flags.has_key(name):\n return self._flags[name]\n else:\n val = self._env.get(name)\n if val is None:\n return default\n elif val.lower() in ['yes', 'on']:\n self._flags[name] = True\n return True\n elif val.lower() in ['no', 'off']:\n self._flags[name] = False\n return False\n else:\n raise AttributeError(\"Value for flag %s, must be yes, no, on or off\" \n % name)", "response": "Checks if the environment variable name is in the flag list."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef saveState(self, stateObj):\n try:\n fp = open(self._stateFile, 'w')\n pickle.dump(stateObj, fp)\n except:\n raise IOError(\"Failure in storing plugin state in file: %s\" \n % self._stateFile)\n return True", "response": "Utility methos to save plugin state stored in stateObj to persistent \n storage"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef restoreState(self):\n if os.path.exists(self._stateFile):\n try:\n fp = open(self._stateFile, 'r')\n stateObj = pickle.load(fp)\n except:\n raise IOError(\"Failure in reading plugin state from file: %s\" \n % self._stateFile)\n return stateObj\n return None", "response": "Utility method to restore plugin state from persistent storage to \n permit access to previous plugin state."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef appendSubgraph(self, parent_name, graph_name, graph):\n if not self.isMultigraph:\n raise AttributeError(\"Simple Munin Plugins cannot have subgraphs.\")\n if self._graphDict.has_key(parent_name):\n if not self._subgraphDict.has_key(parent_name):\n self._subgraphDict[parent_name] = {}\n self._subgraphNames[parent_name] = []\n self._subgraphDict[parent_name][graph_name] = graph\n self._subgraphNames[parent_name].append(graph_name)\n else:\n raise AttributeError(\"Invalid parent graph name %s used for subgraph %s.\"\n % (parent_name, graph_name))", "response": "This method adds a subgraph to the internal dictionary."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setGraphVal(self, graph_name, field_name, val):\n graph = self._getGraph(graph_name, True)\n if graph.hasField(field_name):\n graph.setVal(field_name, val)\n else:\n raise AttributeError(\"Invalid field name %s for graph %s.\" \n % (field_name, graph_name))", "response": "Private method to set Value for Field in Graph.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef setSubgraphVal(self, parent_name, graph_name, field_name, val):\n subgraph = self._getSubGraph(parent_name, graph_name, True)\n if subgraph.hasField(field_name):\n subgraph.setVal(field_name, val)\n else:\n raise AttributeError(\"Invalid field name %s for subgraph %s \"\n \"of parent graph %s.\" \n % (field_name, graph_name, parent_name))", "response": "Set the value of a field in a subgraph."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn list of names of subgraphs for parent_name.", "response": "def getSubgraphList(self, parent_name):\n \"\"\"Returns list of names of subgraphs for Root Graph with name parent_name.\n \n @param parent_name: Name of Root Graph.\n @return: List of subgraph names.\n \n \"\"\"\n if not self.isMultigraph:\n raise AttributeError(\"Simple Munin Plugins cannot have subgraphs.\")\n if self._graphDict.has_key(parent_name):\n return self._subgraphNames[parent_name] or []\n else:\n raise AttributeError(\"Invalid parent graph name %s.\"\n % (parent_name,))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn true if graph with name graph_name has field with name field_name.", "response": "def graphHasField(self, graph_name, field_name):\n \"\"\"Return true if graph with name graph_name has field with \n name field_name.\n \n @param graph_name: Graph Name\n @param field_name: Field Name.\n @return: Boolean\n \n \"\"\"\n graph = self._graphDict.get(graph_name, True)\n return graph.hasField(field_name)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning true if subgraph with name graph_name with parent graph with name parent_name has field with name field_name.", "response": "def subGraphHasField(self, parent_name, graph_name, field_name):\n \"\"\"Return true if subgraph with name graph_name with parent graph with\n name parent_name has field with name field_name.\n \n @param parent_name: Root Graph Name\n @param graph_name: Subgraph Name\n @param field_name: Field Name.\n @return: Boolean\n \n \"\"\"\n subgraph = self._getSubGraph(parent_name, graph_name, True)\n return subgraph.hasField(field_name)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns list of names of fields for graph with name graph_name.", "response": "def getGraphFieldList(self, graph_name):\n \"\"\"Returns list of names of fields for graph with name graph_name.\n \n @param graph_name: Graph Name\n @return: List of field names for graph.\n \n \"\"\"\n graph = self._getGraph(graph_name, True)\n return graph.getFieldList()"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the number of fields for a given graph with name graph_name.", "response": "def getGraphFieldCount(self, graph_name):\n \"\"\"Returns number of fields for graph with name graph_name.\n \n @param graph_name: Graph Name\n @return: Number of fields for graph.\n \n \"\"\"\n graph = self._getGraph(graph_name, True)\n return graph.getFieldCount()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getSubgraphFieldList(self, parent_name, graph_name):\n graph = self._getSubGraph(parent_name, graph_name, True)\n return graph.getFieldList()", "response": "Returns list of names of fields for subgraph with name graph_name\n and parent graph with name parent_name."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getSubgraphFieldCount(self, parent_name, graph_name):\n graph = self._getSubGraph(parent_name, graph_name, True)\n return graph.getFieldCount()", "response": "Returns the number of fields for subgraph with name graph_name and parent \n graph with name parent_name."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nimplements Munin Plugin Graph Configuration.", "response": "def config(self):\n \"\"\"Implements Munin Plugin Graph Configuration.\n \n Prints out configuration for graphs.\n\n Use as is. Not required to be overwritten in child classes. The plugin\n will work correctly as long as the Munin Graph objects have been \n populated.\n\n \"\"\"\n for parent_name in self._graphNames:\n graph = self._graphDict[parent_name]\n if self.isMultigraph:\n print \"multigraph %s\" % self._getMultigraphID(parent_name)\n print self._formatConfig(graph.getConfig())\n print\n if (self.isMultigraph and self._nestedGraphs \n and self._subgraphDict and self._subgraphNames):\n for (parent_name, subgraph_names) in self._subgraphNames.iteritems():\n for graph_name in subgraph_names:\n graph = self._subgraphDict[parent_name][graph_name]\n print \"multigraph %s\" % self.getMultigraphID(parent_name, \n graph_name)\n print self._formatConfig(graph.getConfig())\n print\n return True"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nimplementing Munin Plugin Fetch Option.", "response": "def fetch(self):\n \"\"\"Implements Munin Plugin Fetch Option.\n\n Prints out measured values.\n\n \"\"\"\n self.retrieveVals()\n for parent_name in self._graphNames:\n graph = self._graphDict[parent_name]\n if self.isMultigraph:\n print \"multigraph %s\" % self._getMultigraphID(parent_name)\n print self._formatVals(graph.getVals())\n print\n if (self.isMultigraph and self._nestedGraphs \n and self._subgraphDict and self._subgraphNames):\n for (parent_name, subgraph_names) in self._subgraphNames.iteritems():\n for graph_name in subgraph_names:\n graph = self._subgraphDict[parent_name][graph_name]\n print \"multigraph %s\" % self.getMultigraphID(parent_name, \n graph_name)\n print self._formatVals(graph.getVals())\n print\n return True"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nimplementing main entry point for plugin execution.", "response": "def run(self):\n \"\"\"Implements main entry point for plugin execution.\"\"\"\n if len(self._argv) > 1 and len(self._argv[1]) > 0:\n oper = self._argv[1]\n else:\n oper = 'fetch'\n if oper == 'fetch':\n ret = self.fetch()\n elif oper == 'config':\n ret = self.config()\n if ret and self._dirtyConfig:\n ret = self.fetch()\n elif oper == 'autoconf':\n ret = self.autoconf()\n if ret:\n print \"yes\"\n else:\n print \"no\"\n ret = True\n elif oper == 'suggest':\n ret = self.suggest()\n else:\n raise AttributeError(\"Invalid command argument: %s\" % oper)\n return ret"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a new field to the Munin Graph.", "response": "def addField(self, name, label, type=None, draw=None, info=None, #@ReservedAssignment\n extinfo=None, colour=None, negative=None, graph=None, \n min=None, max=None, cdef=None, line=None, #@ReservedAssignment\n warning=None, critical=None):\n \"\"\"Add field to Munin Graph\n \n @param name: Field Name\n @param label: Field Label\n @param type: Stat Type:\n 'COUNTER' / 'ABSOLUTE' / 'DERIVE' / 'GAUGE'\n @param draw: Graph Type:\n 'AREA' / 'LINE{1,2,3}' / \n 'STACK' / 'LINESTACK{1,2,3}' / 'AREASTACK'\n @param info: Detailed Field Info\n @param extinfo: Extended Field Info\n @param colour: Field Colour\n @param negative: Mirror Value\n @param graph: Draw on Graph - True / False (Default: True)\n @param min: Minimum Valid Value\n @param max: Maximum Valid Value\n @param cdef: CDEF\n @param line: Adds horizontal line at value defined for field. \n @param warning: Warning Value\n @param critical: Critical Value\n \n \"\"\"\n if self._autoFixNames:\n name = self._fixName(name)\n if negative is not None:\n negative = self._fixName(negative)\n self._fieldAttrDict[name] = dict(((k,v) for (k,v) in locals().iteritems()\n if (v is not None\n and k not in ('self',))))\n self._fieldNameList.append(name)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef hasField(self, name):\n if self._autoFixNames:\n name = self._fixName(name)\n return self._fieldAttrDict.has_key(name)", "response": "Returns true if field with field_name exists."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getConfig(self):\n return {'graph': self._graphAttrDict,\n 'fields': [(field_name, self._fieldAttrDict.get(field_name))\n for field_name in self._fieldNameList]}", "response": "Returns dictionary of config entries for Munin Graph.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset value for the given name.", "response": "def setVal(self, name, val):\n \"\"\"Set value for field in graph.\n \n @param name : Graph Name\n @param value : Value for field. \n \n \"\"\"\n if self._autoFixNames:\n name = self._fixName(name)\n self._fieldValDict[name] = val"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getVals(self):\n return [(name, self._fieldValDict.get(name)) \n for name in self._fieldNameList]", "response": "Returns value list for Munin Graph\n \n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nquerying and parse Web Server Status Page.", "response": "def initStats(self, extras=None):\n \"\"\"Query and parse Web Server Status Page.\n \n @param extras: Include extra metrics, which can be computationally more \n expensive.\n \n \"\"\"\n if extras is not None:\n self._extras = extras\n if self._extras:\n detail = 1\n else:\n detail = 0\n url = \"%s://%s:%d/%s?detail=%s\" % (self._proto, self._host, self._port, \n self._monpath, detail)\n response = util.get_url(url, self._user, self._password)\n self._statusDict = {}\n for line in response.splitlines():\n cols = line.split(':')\n if not self._statusDict.has_key(cols[0]):\n self._statusDict[cols[0]] = {}\n self._statusDict[cols[0]][cols[1]] = util.parse_value(cols[2])"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef initStats(self):\n url = \"%s://%s:%d/%s\" % (self._proto, self._host, self._port, \n self._statuspath)\n response = util.get_url(url, self._user, self._password)\n self._statusDict = {}\n for line in response.splitlines():\n mobj = re.match('\\s*(\\d+)\\s+(\\d+)\\s+(\\d+)\\s*$', line)\n if mobj:\n idx = 0\n for key in ('accepts','handled','requests'):\n idx += 1\n self._statusDict[key] = util.parse_value(mobj.group(idx))\n else:\n for (key,val) in re.findall('(\\w+):\\s*(\\d+)', line):\n self._statusDict[key.lower()] = util.parse_value(val)", "response": "Query and parse Nginx Web Server Status Page."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef read_file(filename):\n path = os.path.abspath(os.path.dirname(__file__))\n filepath = os.path.join(path, filename)\n try:\n return open(filepath).read()\n except IOError:\n return ''", "response": "Read a file into a string"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nretrieving values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n proc_info = ProcessInfo()\n stats = {}\n for (prefix, is_thread) in (('proc', False), \n ('thread', True)):\n graph_name = '%s_status' % prefix\n if self.hasGraph(graph_name):\n if not stats.has_key(prefix):\n stats[prefix] = proc_info.getProcStatStatus(is_thread)\n for (fname, stat_key) in (\n ('unint_sleep', 'uninterruptable_sleep'),\n ('stopped', 'stopped'),\n ('defunct', 'defunct'),\n ('running', 'running'),\n ('sleep', 'sleep')):\n self.setGraphVal(graph_name, fname, \n stats[prefix]['status'].get(stat_key))\n graph_name = '%s_prio' % prefix\n if self.hasGraph(graph_name):\n if not stats.has_key(prefix):\n stats[prefix] = proc_info.getProcStatStatus(is_thread)\n for (fname, stat_key) in (\n ('high', 'high'),\n ('low', 'low'),\n ('norm', 'norm'),\n ('locked', 'locked_in_mem')):\n self.setGraphVal(graph_name, fname, \n stats[prefix]['prio'].get(stat_key))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n file_stats = self._fileInfo.getContainerStats()\n for contname in self._fileContList:\n stats = file_stats.get(contname)\n if stats is not None:\n if self.hasGraph('rackspace_cloudfiles_container_size'):\n self.setGraphVal('rackspace_cloudfiles_container_size', contname,\n stats.get('size'))\n if self.hasGraph('rackspace_cloudfiles_container_count'):\n self.setGraphVal('rackspace_cloudfiles_container_count', contname,\n stats.get('count'))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning nested dictionary of Traffic Stats for each network interface.", "response": "def getIfStats(self):\n \"\"\"Return dictionary of Traffic Stats for Network Interfaces.\n \n @return: Nested dictionary of statistics for each interface.\n \n \"\"\"\n info_dict = {}\n try:\n fp = open(ifaceStatsFile, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading interface stats from file: %s'\n % ifaceStatsFile)\n for line in data.splitlines():\n mobj = re.match('^\\s*([\\w\\d:]+):\\s*(.*\\S)\\s*$', line)\n if mobj:\n iface = mobj.group(1)\n statline = mobj.group(2)\n info_dict[iface] = dict(zip(\n ('rxbytes', 'rxpackets', 'rxerrs', 'rxdrop', 'rxfifo',\n 'rxframe', 'rxcompressed', 'rxmulticast',\n 'txbytes', 'txpackets', 'txerrs', 'txdrop', 'txfifo',\n 'txcolls', 'txcarrier', 'txcompressed'),\n [int(x) for x in statline.split()]))\n \n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getIfConfig(self):\n conf = {}\n try:\n out = subprocess.Popen([ipCmd, \"addr\", \"show\"], \n stdout=subprocess.PIPE).communicate()[0]\n except:\n raise Exception('Execution of command %s failed.' % ipCmd)\n for line in out.splitlines():\n mobj = re.match('^\\d+: (\\S+):\\s+<(\\S*)>\\s+(\\S.*\\S)\\s*$', line)\n if mobj:\n iface = mobj.group(1)\n conf[iface] = {}\n continue\n mobj = re.match('^\\s{4}link\\/(.*\\S)\\s*$', line)\n if mobj:\n arr = mobj.group(1).split()\n if len(arr) > 0:\n conf[iface]['type'] = arr[0]\n if len(arr) > 1:\n conf[iface]['hwaddr'] = arr[1]\n continue\n mobj = re.match('^\\s+(inet|inet6)\\s+([\\d\\.\\:A-Za-z]+)\\/(\\d+)($|\\s+.*\\S)\\s*$', line)\n if mobj:\n proto = mobj.group(1)\n if not conf[iface].has_key(proto):\n conf[iface][proto] = []\n addrinfo = {}\n addrinfo['addr'] = mobj.group(2).lower()\n addrinfo['mask'] = int(mobj.group(3))\n arr = mobj.group(4).split()\n if len(arr) > 0 and arr[0] == 'brd':\n addrinfo['brd'] = arr[1]\n conf[iface][proto].append(addrinfo)\n continue\n return conf", "response": "Return dictionary of Interface Configuration keyed by if name."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget routing table. @return: List of routes.", "response": "def getRoutes(self):\n \"\"\"Get routing table.\n \n @return: List of routes.\n \n \"\"\"\n routes = []\n try:\n out = subprocess.Popen([routeCmd, \"-n\"], \n stdout=subprocess.PIPE).communicate()[0]\n except:\n raise Exception('Execution of command %s failed.' % ipCmd)\n lines = out.splitlines()\n if len(lines) > 1:\n headers = [col.lower() for col in lines[1].split()]\n for line in lines[2:]:\n routes.append(dict(zip(headers, line.split())))\n return routes"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef execNetstatCmd(self, *args):\n out = util.exec_command([netstatCmd,] + list(args))\n return out.splitlines()", "response": "Execute netstat command with positional params args and return result as \n list of lines."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef parseNetstatCmd(self, tcp=True, udp=True, ipv4=True, ipv6=True, \n include_listen=True, only_listen=False,\n show_users=False, show_procs=False,\n resolve_hosts=False, resolve_ports=False, \n resolve_users=True):\n \"\"\"Execute netstat command and return result as a nested dictionary.\n \n @param tcp: Include TCP ports in ouput if True.\n @param udp: Include UDP ports in ouput if True.\n @param ipv4: Include IPv4 ports in output if True.\n @param ipv6: Include IPv6 ports in output if True.\n @param include_listen: Include listening ports in output if True.\n @param only_listen: Include only listening ports in output if True.\n @param show_users: Show info on owning users for ports if True.\n @param show_procs: Show info on PID and Program Name attached to\n ports if True.\n @param resolve_hosts: Resolve IP addresses into names if True.\n @param resolve_ports: Resolve numeric ports to names if True.\n @param resolve_users: Resolve numeric user IDs to user names if True.\n @return: List of headers and list of rows and columns.\n \n \"\"\"\n headers = ['proto', 'ipversion', 'recvq', 'sendq', \n 'localaddr', 'localport','foreignaddr', 'foreignport', \n 'state']\n args = []\n proto = []\n if ipv4:\n proto.append('inet')\n if ipv6:\n proto.append('inet6')\n if len(proto) > 0:\n args.append('-A')\n args.append(','.join(proto))\n if tcp:\n args.append('-t')\n if udp:\n args.append('-u')\n if only_listen:\n args.append('-l')\n elif include_listen:\n args.append('-a')\n regexp_str = ('(tcp|udp)(\\d*)\\s+(\\d+)\\s+(\\d+)\\s+'\n '(\\S+):(\\w+)\\s+(\\S+):(\\w+|\\*)\\s+(\\w*)')\n if show_users:\n args.append('-e')\n regexp_str += '\\s+(\\w+)\\s+(\\d+)'\n headers.extend(['user', 'inode'])\n if show_procs:\n args.append('-p')\n regexp_str += '\\s+(\\S+)'\n headers.extend(['pid', 'prog'])\n if not resolve_hosts:\n args.append('--numeric-hosts')\n if not resolve_ports:\n args.append('--numeric-ports')\n if not resolve_users:\n args.append('--numeric-users')\n lines = self.execNetstatCmd(*args)\n stats = []\n regexp = re.compile(regexp_str)\n for line in lines[2:]:\n mobj = regexp.match(line)\n if mobj is not None:\n stat = list(mobj.groups())\n if stat[1] == '0':\n stat[1] = '4'\n if stat[8] == '':\n stat[8] = None\n if show_procs:\n proc = stat.pop().split('/')\n if len(proc) == 2:\n stat.extend(proc)\n else:\n stat.extend([None, None])\n stats.append(stat)\n return {'headers': headers, 'stats': stats}", "response": "Execute netstat command and return result as a nested dictionary."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getStats(self, tcp=True, udp=True, ipv4=True, ipv6=True, \n include_listen=True, only_listen=False,\n show_users=False, show_procs=False, \n resolve_hosts=False, resolve_ports=False, resolve_users=True, \n **kwargs):\n \"\"\"Execute netstat command and return result as a nested dictionary.\n \n @param tcp: Include TCP ports in ouput if True.\n @param udp: Include UDP ports in ouput if True.\n @param ipv4: Include IPv4 ports in output if True.\n @param ipv6: Include IPv6 ports in output if True.\n @param include_listen: Include listening ports in output if True.\n @param only_listen: Include only listening ports in output if True.\n @param show_users: Show info on owning users for ports if True.\n @param show_procs: Show info on PID and Program Name attached to\n ports if True.\n @param resolve_hosts: Resolve IP addresses into names if True.\n @param resolve_ports: Resolve numeric ports to names if True.\n @param resolve_users: Resolve numeric user IDs to user names if True.\n @param **kwargs: Keyword variables are used for filtering the \n results depending on the values of the columns. \n Each keyword must correspond to a field name with \n an optional suffix:\n field: Field equal to value or in list \n of values.\n field_ic: Field equal to value or in list of \n values, using case insensitive \n comparison.\n field_regex: Field matches regex value or \n matches with any regex in list of \n values.\n field_ic_regex: Field matches regex value or \n matches with any regex in list of \n values using case insensitive \n match.\n @return: List of headers and list of rows and columns.\n \n \"\"\"\n pinfo = self.parseNetstatCmd(tcp, udp, ipv4, ipv6, \n include_listen, only_listen,\n show_users, show_procs, \n resolve_hosts, resolve_ports, resolve_users)\n if pinfo:\n if len(kwargs) > 0:\n pfilter = util.TableFilter()\n pfilter.registerFilters(**kwargs)\n stats = pfilter.applyFilters(pinfo['headers'], pinfo['stats'])\n return {'headers': pinfo['headers'], 'stats': stats}\n else:\n return pinfo\n else:\n return None", "response": "Execute netstat command and return result as a nested dictionary."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the number of TCP endpoints discriminated by status.", "response": "def getTCPportConnStatus(self, ipv4=True, ipv6=True, include_listen=False,\n **kwargs):\n \"\"\"Returns the number of TCP endpoints discriminated by status.\n \n @param ipv4: Include IPv4 ports in output if True.\n @param ipv6: Include IPv6 ports in output if True.\n @param include_listen: Include listening ports in output if True.\n @param **kwargs: Keyword variables are used for filtering the \n results depending on the values of the columns. \n Each keyword must correspond to a field name with \n an optional suffix:\n field: Field equal to value or in list \n of values.\n field_ic: Field equal to value or in list of \n values, using case insensitive \n comparison.\n field_regex: Field matches regex value or \n matches with any regex in list of \n values.\n field_ic_regex: Field matches regex value or \n matches with any regex in list of \n values using case insensitive \n match.\n @return: Dictionary mapping connection status to the\n number of endpoints.\n \n \"\"\"\n status_dict = {}\n result = self.getStats(tcp=True, udp=False, \n include_listen=include_listen, \n ipv4=ipv4, ipv6=ipv6, \n **kwargs)\n stats = result['stats']\n for stat in stats:\n if stat is not None:\n status = stat[8].lower()\n status_dict[status] = status_dict.get(status, 0) + 1\n return status_dict"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getTCPportConnCount(self, ipv4=True, ipv6=True, resolve_ports=False,\n **kwargs):\n \"\"\"Returns TCP connection counts for each local port.\n \n @param ipv4: Include IPv4 ports in output if True.\n @param ipv6: Include IPv6 ports in output if True.\n @param resolve_ports: Resolve numeric ports to names if True.\n @param **kwargs: Keyword variables are used for filtering the \n results depending on the values of the columns. \n Each keyword must correspond to a field name with \n an optional suffix:\n field: Field equal to value or in list \n of values.\n field_ic: Field equal to value or in list of \n values, using case insensitive \n comparison.\n field_regex: Field matches regex value or \n matches with any regex in list of \n values.\n field_ic_regex: Field matches regex value or \n matches with any regex in list of \n values using case insensitive \n match.\n @return: Dictionary mapping port number or name to the\n number of established connections.\n \n \"\"\"\n port_dict = {}\n result = self.getStats(tcp=True, udp=False, \n include_listen=False, ipv4=ipv4, \n ipv6=ipv6, resolve_ports=resolve_ports,\n **kwargs)\n stats = result['stats']\n for stat in stats:\n if stat[8] == 'ESTABLISHED':\n port_dict[stat[5]] = port_dict.get(5, 0) + 1\n return port_dict", "response": "Returns a dictionary mapping local port number or name to the number of established connections."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompute the accuracy of the recalled item and item in the specified egg.", "response": "def accuracy_helper(egg, match='exact', distance='euclidean',\n features=None):\n \"\"\"\n Computes proportion of words recalled\n\n Parameters\n ----------\n egg : quail.Egg\n Data to analyze\n\n match : str (exact, best or smooth)\n Matching approach to compute recall matrix. If exact, the presented and\n recalled items must be identical (default). If best, the recalled item\n that is most similar to the presented items will be selected. If smooth,\n a weighted average of all presented items will be used, where the\n weights are derived from the similarity between the recalled item and\n each presented item.\n\n distance : str\n The distance function used to compare presented and recalled items.\n Applies only to 'best' and 'smooth' matching approaches. Can be any\n distance function supported by numpy.spatial.distance.cdist.\n\n Returns\n ----------\n prop_recalled : numpy array\n proportion of words recalled\n\n \"\"\"\n\n def acc(lst):\n return len([i for i in np.unique(lst) if i>=0])/(egg.list_length)\n\n opts = dict(match=match, distance=distance, features=features)\n if match is 'exact':\n opts.update({'features' : 'item'})\n recmat = recall_matrix(egg, **opts)\n\n if match in ['exact', 'best']:\n result = [acc(lst) for lst in recmat]\n elif match is 'smooth':\n result = np.mean(recmat, axis=1)\n else:\n raise ValueError('Match must be set to exact, best or smooth.')\n\n return np.nanmean(result, axis=0)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nestablish connection to PostgreSQL Database.", "response": "def _connect(self):\n \"\"\"Establish connection to PostgreSQL Database.\"\"\"\n if self._connParams:\n self._conn = psycopg2.connect(**self._connParams)\n else:\n self._conn = psycopg2.connect('')\n try:\n ver_str = self._conn.get_parameter_status('server_version')\n except AttributeError:\n ver_str = self.getParam('server_version')\n self._version = util.SoftwareVersion(ver_str)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nexecuting a simple query which returns a single column.", "response": "def _simpleQuery(self, query):\n \"\"\"Executes simple query which returns a single column.\n \n @param query: Query string.\n @return: Query result string.\n \n \"\"\"\n cur = self._conn.cursor()\n cur.execute(query)\n row = cur.fetchone()\n return util.parse_value(row[0])"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn value of Run - time Database Parameter key.", "response": "def getParam(self, key):\n \"\"\"Returns value of Run-time Database Parameter 'key'.\n \n @param key: Run-time parameter name.\n @return: Run-time parameter value.\n \n \"\"\"\n cur = self._conn.cursor()\n cur.execute(\"SHOW %s\" % key)\n row = cur.fetchone()\n return util.parse_value(row[0])"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getConnectionStats(self):\n cur = self._conn.cursor()\n cur.execute(\"\"\"SELECT datname,numbackends FROM pg_stat_database;\"\"\")\n rows = cur.fetchall()\n if rows:\n return dict(rows)\n else:\n return {}", "response": "Returns dictionary with number of connections for each database."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn database block read transaction and tuple stats for each database.", "response": "def getDatabaseStats(self):\n \"\"\"Returns database block read, transaction and tuple stats for each \n database.\n \n @return: Nested dictionary of stats.\n \n \"\"\"\n headers = ('datname', 'numbackends', 'xact_commit', 'xact_rollback', \n 'blks_read', 'blks_hit', 'tup_returned', 'tup_fetched', \n 'tup_inserted', 'tup_updated', 'tup_deleted', 'disk_size')\n cur = self._conn.cursor()\n cur.execute(\"SELECT %s, pg_database_size(datname) FROM pg_stat_database;\" \n % \",\".join(headers[:-1]))\n rows = cur.fetchall()\n dbstats = self._createStatsDict(headers, rows)\n totals = self._createTotalsDict(headers, rows)\n return {'databases': dbstats, 'totals': totals}"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the number of active lock discriminated by lock mode.", "response": "def getLockStatsMode(self):\n \"\"\"Returns the number of active lock discriminated by lock mode.\n \n @return: : Dictionary of stats.\n \n \"\"\"\n info_dict = {'all': dict(zip(self.lockModes, (0,) * len(self.lockModes))),\n 'wait': dict(zip(self.lockModes, (0,) * len(self.lockModes)))}\n cur = self._conn.cursor()\n cur.execute(\"SELECT TRIM(mode, 'Lock'), granted, COUNT(*) FROM pg_locks \"\n \"GROUP BY TRIM(mode, 'Lock'), granted;\")\n rows = cur.fetchall()\n for (mode, granted, cnt) in rows:\n info_dict['all'][mode] += cnt\n if not granted:\n info_dict['wait'][mode] += cnt\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getLockStatsDB(self):\n info_dict = {'all': {},\n 'wait': {}}\n cur = self._conn.cursor()\n cur.execute(\"SELECT d.datname, l.granted, COUNT(*) FROM pg_database d \"\n \"JOIN pg_locks l ON d.oid=l.database \"\n \"GROUP BY d.datname, l.granted;\")\n rows = cur.fetchall()\n for (db, granted, cnt) in rows:\n info_dict['all'][db] = info_dict['all'].get(db, 0) + cnt\n if not granted:\n info_dict['wait'][db] = info_dict['wait'].get(db, 0) + cnt\n return info_dict", "response": "Returns the number of active lock discriminated by database."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning Global Background Writer and Checkpoint Activity stats.", "response": "def getBgWriterStats(self):\n \"\"\"Returns Global Background Writer and Checkpoint Activity stats.\n \n @return: Nested dictionary of stats.\n \n \"\"\"\n info_dict = {}\n if self.checkVersion('8.3'):\n cur = self._conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)\n cur.execute(\"SELECT * FROM pg_stat_bgwriter\")\n info_dict = cur.fetchone()\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a dictionary of status items.", "response": "def getXlogStatus(self):\n \"\"\"Returns Transaction Logging or Recovery Status.\n \n @return: Dictionary of status items.\n \n \"\"\"\n inRecovery = None\n if self.checkVersion('9.0'):\n inRecovery = self._simpleQuery(\"SELECT pg_is_in_recovery();\")\n cur = self._conn.cursor()\n if inRecovery:\n cols = ['pg_last_xlog_receive_location()', \n 'pg_last_xlog_replay_location()',]\n headers = ['xlog_receive_location',\n 'xlog_replay_location',]\n if self.checkVersion('9.1'):\n cols.extend(['pg_last_xact_replay_timestamp()',\n 'pg_is_xlog_replay_paused()',])\n headers.extend(['xact_replay_timestamp', \n 'xlog_replay_paused',])\n cur.execute(\"\"\"SELECT %s;\"\"\" % ','.join(cols))\n headers = ('xlog_receive_location', 'xlog_replay_location')\n else:\n cur.execute(\"\"\"SELECT\n pg_current_xlog_location(), \n pg_xlogfile_name(pg_current_xlog_location());\"\"\")\n headers = ('xlog_location', 'xlog_filename')\n row = cur.fetchone()\n info_dict = dict(zip(headers, row))\n if inRecovery is not None:\n info_dict['in_recovery'] = inRecovery\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getSlaveStatus(self):\n info_dict = {}\n if self.checkVersion('9.1'):\n cols = ['procpid', 'usename', 'application_name', \n 'client_addr', 'client_port', 'backend_start', 'state', \n 'sent_location', 'write_location', 'flush_location', \n 'replay_location', 'sync_priority', 'sync_state',]\n cur = self._conn.cursor()\n cur.execute(\"\"\"SELECT %s FROM pg_stat_replication;\"\"\" \n % ','.join(cols))\n rows = cur.fetchall()\n for row in rows:\n info_dict[row[0]] = dict(zip(cols[1:], row[1:]))\n else:\n return None\n return info_dict", "response": "Returns status of replication slaves."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _connect(self):\n if self._connParams:\n self._conn = MySQLdb.connect(**self._connParams)\n else:\n self._conn = MySQLdb.connect('')", "response": "Establish connection to MySQL Database."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn list of supported storage engines.", "response": "def getStorageEngines(self):\n \"\"\"Returns list of supported storage engines.\n \n @return: List of storage engine names.\n \n \"\"\"\n cur = self._conn.cursor()\n cur.execute(\"\"\"SHOW STORAGE ENGINES;\"\"\")\n rows = cur.fetchall()\n if rows:\n return [row[0].lower() for row in rows if row[1] in ['YES', 'DEFAULT']]\n else:\n return []"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getParam(self, key):\n cur = self._conn.cursor()\n cur.execute(\"SHOW GLOBAL VARIABLES LIKE %s\", key)\n row = cur.fetchone()\n return int(row[1])", "response": "Returns value of Run - time Database Parameter key."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getParams(self):\n cur = self._conn.cursor()\n cur.execute(\"SHOW GLOBAL VARIABLES\")\n rows = cur.fetchall()\n info_dict = {}\n for row in rows:\n key = row[0]\n val = util.parse_value(row[1])\n info_dict[key] = val\n return info_dict", "response": "Returns dictionary of all run - time parameters."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getProcessStatus(self):\n info_dict = {}\n cur = self._conn.cursor()\n cur.execute(\"\"\"SHOW FULL PROCESSLIST;\"\"\")\n rows = cur.fetchall()\n if rows:\n for row in rows:\n if row[6] == '':\n state = 'idle'\n elif row[6] is None:\n state = 'other'\n else:\n state = str(row[6]).replace(' ', '_').lower()\n info_dict[state] = info_dict.get(state, 0) + 1\n return info_dict", "response": "Returns the number of processes discriminated by state.\nAttributeNames - The process names of the processes in the process list."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns number of processes discriminated by database name.", "response": "def getProcessDatabase(self):\n \"\"\"Returns number of processes discriminated by database name.\n \n @return: Dictionary mapping database name to number of processes.\n \n \"\"\"\n info_dict = {}\n cur = self._conn.cursor()\n cur.execute(\"\"\"SHOW FULL PROCESSLIST;\"\"\")\n rows = cur.fetchall()\n if rows:\n for row in rows:\n db = row[3]\n info_dict[db] = info_dict.get(db, 0) + 1\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getDatabases(self):\n cur = self._conn.cursor()\n cur.execute(\"\"\"SHOW DATABASES;\"\"\")\n rows = cur.fetchall()\n if rows:\n return [row[0] for row in rows]\n else:\n return []", "response": "Returns list of databases."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef spc_helper(egg, match='exact', distance='euclidean',\n features=None):\n \"\"\"\n Computes probability of a word being recalled (in the appropriate recall list), given its presentation position\n\n Parameters\n ----------\n egg : quail.Egg\n Data to analyze\n\n match : str (exact, best or smooth)\n Matching approach to compute recall matrix. If exact, the presented and\n recalled items must be identical (default). If best, the recalled item\n that is most similar to the presented items will be selected. If smooth,\n a weighted average of all presented items will be used, where the\n weights are derived from the similarity between the recalled item and\n each presented item.\n\n distance : str\n The distance function used to compare presented and recalled items.\n Applies only to 'best' and 'smooth' matching approaches. Can be any\n distance function supported by numpy.spatial.distance.cdist.\n\n Returns\n ----------\n prec : numpy array\n each number represents the probability of recall for a word presented in given position/index\n\n \"\"\"\n\n def spc(lst):\n d = np.zeros_like(egg.pres.values[0])\n inds = np.array(lst[~np.isnan(lst)]).astype(int)\n d[inds-1]=1\n return d\n\n opts = dict(match=match, distance=distance, features=features)\n if match is 'exact':\n opts.update({'features' : 'item'})\n recmat = recall_matrix(egg, **opts)\n\n if match in ['exact', 'best']:\n result = [spc(lst) for lst in recmat]\n elif match == 'smooth':\n result = np.nanmean(recmat, 2)\n else:\n raise ValueError('Match must be set to exact, best or smooth.')\n return np.mean(result, 0)", "response": "Returns a sequence of probability of a word being recalled given its presentation position and the number of items presented in the item."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _retrieve(self):\n url = \"%s://%s:%d/manager/status\" % (self._proto, self._host, self._port)\n params = {}\n params['XML'] = 'true'\n response = util.get_url(url, self._user, self._password, params)\n tree = ElementTree.XML(response)\n return tree", "response": "Query Apache Tomcat Server Status Page in XML format and return \n the result as an ElementTree object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn JVM Memory Stats for Apache Tomcat Server.", "response": "def getMemoryStats(self):\n \"\"\"Return JVM Memory Stats for Apache Tomcat Server.\n \n @return: Dictionary of memory utilization stats.\n \n \"\"\"\n if self._statusxml is None:\n self.initStats()\n node = self._statusxml.find('jvm/memory')\n memstats = {}\n if node is not None:\n for (key,val) in node.items():\n memstats[key] = util.parse_value(val)\n return memstats"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns nested dictionary of Connector Stats for Apache Tomcat Server.", "response": "def getConnectorStats(self):\n \"\"\"Return dictionary of Connector Stats for Apache Tomcat Server.\n \n @return: Nested dictionary of Connector Stats.\n \n \"\"\"\n if self._statusxml is None:\n self.initStats()\n connnodes = self._statusxml.findall('connector')\n connstats = {}\n if connnodes:\n for connnode in connnodes:\n namestr = connnode.get('name')\n if namestr is not None:\n mobj = re.match('(.*)-(\\d+)', namestr)\n if mobj:\n proto = mobj.group(1)\n port = int(mobj.group(2))\n connstats[port] = {'proto': proto}\n for tag in ('threadInfo', 'requestInfo'):\n stats = {}\n node = connnode.find(tag)\n if node is not None:\n for (key,val) in node.items():\n if re.search('Time$', key):\n stats[key] = float(val) / 1000.0\n else:\n stats[key] = util.parse_value(val)\n if stats:\n connstats[port][tag] = stats\n return connstats"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads an egg or fried eggs and example data from disk.", "response": "def load(filepath, update=True):\n \"\"\"\n Loads eggs, fried eggs ands example data\n\n Parameters\n ----------\n filepath : str\n Location of file\n\n update : bool\n If true, updates egg to latest format\n\n Returns\n ----------\n data : quail.Egg or quail.FriedEgg\n Data loaded from disk\n\n \"\"\"\n\n if filepath == 'automatic' or filepath == 'example':\n fpath = os.path.dirname(os.path.abspath(__file__)) + '/data/automatic.egg'\n return load_egg(fpath)\n elif filepath == 'manual':\n fpath = os.path.dirname(os.path.abspath(__file__)) + '/data/manual.egg'\n return load_egg(fpath, update=False)\n elif filepath == 'naturalistic':\n fpath = os.path.dirname(os.path.abspath(__file__)) + '/data/naturalistic.egg'\n elif filepath.split('.')[-1]=='egg':\n return load_egg(filepath, update=update)\n elif filepath.split('.')[-1]=='fegg':\n return load_fegg(filepath, update=False)\n else:\n raise ValueError('Could not load file.')"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_fegg(filepath, update=True):\n try:\n egg = FriedEgg(**dd.io.load(filepath))\n except ValueError as e:\n print(e)\n # if error, try loading old format\n with open(filepath, 'rb') as f:\n egg = pickle.load(f)\n\n if update:\n return egg.crack()\n else:\n return egg", "response": "Loads a pickled version of the object at filepath."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads a pickled egg into a new object", "response": "def load_egg(filepath, update=True):\n \"\"\"\n Loads pickled egg\n\n Parameters\n ----------\n filepath : str\n Location of pickled egg\n\n update : bool\n If true, updates egg to latest format\n\n Returns\n ----------\n egg : Egg data object\n A loaded unpickled egg\n\n \"\"\"\n try:\n egg = Egg(**dd.io.load(filepath))\n except:\n # if error, try loading old format\n with open(filepath, 'rb') as f:\n egg = pickle.load(f)\n\n if update:\n if egg.meta:\n old_meta = egg.meta\n egg.crack()\n egg.meta = old_meta\n return egg\n else:\n return egg.crack()\n else:\n return egg"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef loadEL(dbpath=None, recpath=None, remove_subs=None, wordpool=None, groupby=None, experiments=None,\n filters=None):\n '''\n Function that loads sql files generated by autoFR Experiment\n '''\n\n assert (dbpath is not None), \"You must specify a db file or files.\"\n assert (recpath is not None), \"You must specify a recall folder.\"\n assert (wordpool is not None), \"You must specify a wordpool file.\"\n assert (experiments is not None), \"You must specify a list of experiments\"\n\n ############################################################################\n # subfunctions #############################################################\n\n def db2df(db, filter_func=None):\n '''\n Loads db file and converts to dataframe\n '''\n db_url = \"sqlite:///\" + db\n table_name = 'turkdemo'\n data_column_name = 'datastring'\n\n # boilerplace sqlalchemy setup\n engine = create_engine(db_url)\n metadata = MetaData()\n metadata.bind = engine\n table = Table(table_name, metadata, autoload=True)\n\n # make a query and loop through\n s = table.select()\n rows = s.execute()\n\n data = []\n for row in rows:\n data.append(row[data_column_name])\n\n # parse each participant's datastring as json object\n # and take the 'data' sub-object\n data = [json.loads(part)['data'] for part in data if part is not None]\n\n # remove duplicate subject data for debugXG82XV:debug7XPXQA\n # data[110] = data[110][348:]\n\n # insert uniqueid field into trialdata in case it wasn't added\n # in experiment:\n for part in data:\n for record in part:\n # print(record)\n if type(record['trialdata']) is list:\n\n record['trialdata'] = {record['trialdata'][0]:record['trialdata'][1]}\n record['trialdata']['uniqueid'] = record['uniqueid']\n\n # flatten nested list so we just have a list of the trialdata recorded\n # each time psiturk.recordTrialData(trialdata) was called.\n def isNotNumber(s):\n try:\n float(s)\n return False\n except ValueError:\n return True\n\n data = [record['trialdata'] for part in data for record in part]\n\n # filter out fields that we dont want using isNotNumber function\n filtered_data = [{k:v for (k,v) in list(part.items()) if isNotNumber(k)} for part in data]\n\n # Put all subjects' trial data into a dataframe object from the\n # 'pandas' python library: one option among many for analysis\n data_frame = pd.DataFrame(filtered_data)\n\n data_column_name = 'codeversion'\n\n # boilerplace sqlalchemy setup\n engine = create_engine(db_url)\n metadata = MetaData()\n metadata.bind = engine\n table = Table(table_name, metadata, autoload=True)\n\n # make a query and loop through\n s = table.select()\n rows = s.execute()\n\n versions = []\n version_dict = {}\n for row in rows:\n version_dict[row[0]]=row[data_column_name]\n\n version_col = []\n for idx,sub in enumerate(data_frame['uniqueid'].unique()):\n for i in range(sum(data_frame['uniqueid']==sub)):\n version_col.append(version_dict[sub])\n data_frame['exp_version']=version_col\n\n if filter_func:\n for idx,filt in enumerate(filter_func):\n data_frame = filt(data_frame)\n return data_frame\n\n # custom filter to clean db file\n def experimenter_filter(data_frame):\n data=[]\n indexes=[]\n for line in data_frame.iterrows():\n try:\n if json.loads(line[1]['responses'])['Q1'].lower() in ['kirsten','allison','allison\\nallison','marisol', 'marisol ','marisiol', 'maddy','campbell', 'campbell field', 'kirsten\\nkirsten', 'emily', 'bryan', 'armando', 'armando ortiz',\n 'maddy/lucy','paxton', 'lucy','campbell\\ncampbell','madison','darya','rachael']:\n delete = False\n else:\n delete = True\n except:\n pass\n\n if delete:\n indexes.append(line[0])\n\n return data_frame.drop(indexes)\n\n def adaptive_filter(data_frame):\n data=[]\n indexes=[]\n subjcb={}\n for line in data_frame.iterrows():\n try:\n if 'Q2' in json.loads(line[1]['responses']):\n delete = False\n else:\n delete = False\n except:\n pass\n\n if delete:\n indexes.append(line[0])\n return data_frame.drop(indexes)\n\n def experiments_filter(data_frame):\n indexes=[]\n for line in data_frame.iterrows():\n try:\n if line[1]['exp_version'] in experiments:\n delete = False\n else:\n delete = True\n except:\n pass\n\n if delete:\n indexes.append(line[0])\n return data_frame.drop(indexes)\n\n\n # this function takes the data frame and returns subject specific data based on the subid variable\n def filterData(data_frame,subid):\n filtered_stim_data = data_frame[data_frame['stimulus'].notnull() & data_frame['listNumber'].notnull()]\n filtered_stim_data = filtered_stim_data[filtered_stim_data['trial_type']=='single-stim']\n filtered_stim_data = filtered_stim_data[filtered_stim_data['uniqueid']==subid]\n return filtered_stim_data\n\n def createStimDict(data):\n stimDict = []\n for index, row in data.iterrows():\n try:\n stimDict.append({\n 'text': str(re.findall('>(.+)<',row['stimulus'])[0]),\n 'color' : { 'r' : int(re.findall('rgb\\((.+)\\)',row['stimulus'])[0].split(',')[0]),\n 'g' : int(re.findall('rgb\\((.+)\\)',row['stimulus'])[0].split(',')[1]),\n 'b' : int(re.findall('rgb\\((.+)\\)',row['stimulus'])[0].split(',')[2])\n },\n 'location' : {\n 'top': float(re.findall('top:(.+)\\%;', row['stimulus'])[0]),\n 'left' : float(re.findall('left:(.+)\\%', row['stimulus'])[0])\n },\n 'category' : wordpool['CATEGORY'].iloc[list(wordpool['WORD'].values).index(str(re.findall('>(.+)<',row['stimulus'])[0]))],\n 'size' : wordpool['SIZE'].iloc[list(wordpool['WORD'].values).index(str(re.findall('>(.+)<',row['stimulus'])[0]))],\n 'wordLength' : len(str(re.findall('>(.+)<',row['stimulus'])[0])),\n 'firstLetter' : str(re.findall('>(.+)<',row['stimulus'])[0])[0],\n 'listnum' : row['listNumber']\n })\n except:\n stimDict.append({\n 'text': str(re.findall('>(.+)<',row['stimulus'])[0]),\n 'color' : { 'r' : 0,\n 'g' : 0,\n 'b' : 0\n },\n 'location' : {\n 'top': 50,\n 'left' : 50\n },\n 'category' : wordpool['CATEGORY'].iloc[list(wordpool['WORD'].values).index(str(re.findall('>(.+)<',row['stimulus'])[0]))],\n 'size' : wordpool['SIZE'].iloc[list(wordpool['WORD'].values).index(str(re.findall('>(.+)<',row['stimulus'])[0]))],\n 'wordLength' : len(str(re.findall('>(.+)<',row['stimulus'])[0])),\n 'firstLetter' : str(re.findall('>(.+)<',row['stimulus'])[0])[0],\n 'listnum' : row['listNumber']\n })\n return stimDict\n\n # this function loads in the recall data into an array of arrays, where each array represents a list of words\n def loadRecallData(subid):\n recalledWords = []\n for i in range(0,16):\n try:\n f = open(recpath + subid + '/' + subid + '-' + str(i) + '.wav.txt', 'r')\n spamreader = csv.reader(f, delimiter=',', quotechar='|')\n except (IOError, OSError) as e:\n try:\n f = open(recpath + subid + '-' + str(i) + '.wav.txt', 'r')\n spamreader = csv.reader(f, delimiter=',', quotechar='|')\n except (IOError, OSError) as e:\n print(e)\n try:\n words=[]\n altformat=True\n for row in spamreader:\n if len(row)>1:\n recalledWords.append(row)\n altformat=False\n break\n else:\n try:\n words.append(row[0])\n except:\n pass\n if altformat:\n recalledWords.append(words)\n except:\n print('couldnt process '+ recpath + subid + '/' + subid + '-' + str(i) + '.wav.txt')\n return recalledWords\n\n # this function computes accuracy for a series of lists\n def computeListAcc(stimDict,recalledWords):\n accVec = []\n for i in range(0,16):\n stim = [stim['text'] for stim in stimDict if stim['listnum']==i]\n recalled= recalledWords[i]\n\n acc = 0\n tmpstim = stim[:]\n for word in recalled:\n if word in tmpstim:\n tmpstim.remove(word)\n acc+=1\n accVec.append(acc/len(stim))\n return accVec\n\n def getFeatures(stimDict):\n stimDict_copy = stimDict[:]\n for item in stimDict_copy:\n item['location'] = [item['location']['top'], item['location']['left']]\n item['color'] = [item['color']['r'], item['color']['g'], item['color']['b']]\n item.pop('text', None)\n item.pop('listnum', None)\n stimDict_copy = [stimDict_copy[i:i+16] for i in range(0, len(stimDict_copy), 16)]\n return stimDict_copy\n\n ############################################################################\n # main program #############################################################\n\n # if its not a list, make it one\n if type(dbpath) is not list:\n dbpath = [dbpath]\n\n # read in stimulus library\n wordpool = pd.read_csv(wordpool)\n\n # add custom filters\n if filters:\n filter_func = [adaptive_filter, experimeter_filter, experiments_filter] + filters\n else:\n filter_func = [adaptive_filter, experimenter_filter, experiments_filter]\n\n # load in dbs and convert to df, and filter\n dfs = [db2df(db, filter_func=filter_func) for db in dbpath]\n\n # concatenate the db files\n df = pd.concat(dfs)\n\n # subjects who have completed the exp\n subids = list(df[df['listNumber']==15]['uniqueid'].unique())\n\n # remove problematic subjects\n if remove_subs:\n for sub in remove_subs:\n try:\n subids.remove(sub)\n except:\n print('Could not find subject: ' + sub + ', skipping...')\n\n # set up data structure to load in subjects\n if groupby:\n pres = [[] for i in range(len(groupby['exp_version']))]\n rec = [[] for i in range(len(groupby['exp_version']))]\n features = [[] for i in range(len(groupby['exp_version']))]\n subs = [[] for i in range(len(groupby['exp_version']))]\n\n # make each groupby item a list\n groupby = [exp if type(exp) is list else [exp] for exp in groupby['exp_version']]\n\n else:\n pres = [[]]\n rec = [[]]\n features = [[]]\n subs = [[]]\n\n # for each subject that completed the experiment\n for idx,sub in enumerate(subids):\n\n # get the subjects data\n filteredStimData = filterData(df,sub)\n\n if filteredStimData['exp_version'].values[0] in experiments:\n\n # create stim dict\n stimDict = createStimDict(filteredStimData)\n\n sub_data = pd.DataFrame(stimDict)\n sub_data['subject']=idx\n sub_data['experiment']=filteredStimData['exp_version'].values[0]\n sub_data = sub_data[['experiment','subject','listnum','text','category','color','location','firstLetter','size','wordLength']]\n\n # get features from stim dict\n feats = getFeatures(stimDict)\n\n # load in the recall data\n recalledWords = loadRecallData(sub)\n\n # get experiment version\n exp_version = filteredStimData['exp_version'].values[0]\n\n # find the idx of the experiment for this subjects\n if groupby:\n exp_idx = list(np.where([exp_version in item for item in groupby])[0])\n else:\n exp_idx = [0]\n\n if exp_idx != []:\n pres[exp_idx[0]].append([list(sub_data[sub_data['listnum']==lst]['text'].values) for lst in sub_data['listnum'].unique()])\n rec[exp_idx[0]].append(recalledWords)\n features[exp_idx[0]].append(feats)\n subs[exp_idx[0]].append(sub)\n\n eggs = [Egg(pres=ipres, rec=irec, features=ifeatures, meta={'ids' : isub}) for ipres,irec,ifeatures,isub in zip(pres, rec, features, subs)]\n\n # map feature dictionaries in pres df to rec df\n def checkword(x):\n if x is None:\n return x\n else:\n try:\n return stim_dict[x['item']]\n except:\n return x\n\n # convert utf-8 bytes type to string\n def update_types(egg):\n featlist = list(egg.pres.loc[0].loc[0].values.tolist()[0].keys())\n def update1df(df):\n for sub in range(egg.n_subjects):\n for liszt in range(egg.n_lists):\n for item in range(len(df.loc[sub].loc[liszt].values.tolist())):\n for feat in featlist:\n if feat in df.loc[sub].loc[liszt].values.tolist()[item].keys():\n if isinstance(df.loc[sub].loc[liszt].values.tolist()[item][feat], np.bytes_):\n try:\n df.loc[sub].loc[liszt].values.tolist()[item][feat] = str(df.loc[sub].loc[liszt].values.tolist()[item][feat], 'utf-8')\n except:\n print(\"Subject \" + str(sub) + \", list \" + str(liszt) + \", item \" + str(item) + \", feature \" + str(feat) + \": Could not convert type \" + str(type(egg.rec.loc[sub].loc[liszt].values.tolist()[item][feat])) + \" to string.\")\n update1df(egg.pres)\n update1df(egg.rec)\n\n for egg in eggs:\n update_types(egg)\n old_meta = egg.meta\n temp_eggs = [egg]\n\n for i in range(egg.n_subjects):\n e = egg.crack(subjects=[i])\n stim = e.pres.values.ravel()\n stim_dict = {str(x['item']) : {k:v for k, v in iter(x.items())} for x in stim}\n\n e.rec = e.rec.applymap(lambda x: checkword(x))\n temp_eggs.append(e)\n\n edited_egg = stack_eggs(temp_eggs)\n mapped_egg = edited_egg.crack(subjects=[i for i in range(egg.n_subjects,egg.n_subjects*2)])\n mapped_egg.meta = old_meta\n eggs[eggs.index(egg)] = mapped_egg\n\n if len(eggs)>1:\n return eggs\n else:\n return eggs[0]", "response": "Function that loads an el file and converts it to a dataframe."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef load_example_data(dataset='automatic'):\n\n # can only be auto or manual\n assert dataset in ['automatic', 'manual', 'naturalistic'], \"Dataset can only be automatic, manual, or naturalistic\"\n\n\n if dataset == 'naturalistic':\n # open naturalistic egg\n egg = Egg(**dd.io.load(os.path.dirname(os.path.abspath(__file__)) + '/data/' + dataset + '.egg'))\n else:\n # open pickled egg\n try:\n with open(os.path.dirname(os.path.abspath(__file__)) + '/data/' + dataset + '.egg', 'rb') as handle:\n egg = pickle.load(handle)\n except:\n f = dd.io.load(os.path.dirname(os.path.abspath(__file__)) + '/data/' + dataset + '.egg')\n egg = Egg(pres=f['pres'], rec=f['rec'], dist_funcs=f['dist_funcs'],\n subjgroup=f['subjgroup'], subjname=f['subjname'],\n listgroup=f['listgroup'], listname=f['listname'],\n date_created=f['date_created'])\n return egg.crack()", "response": "Loads example data for a specific dataset."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef retrieveVals(self):\n fs = FSinfo(self._fshost, self._fsport, self._fspass)\n if self.hasGraph('fs_calls'):\n count = fs.getCallCount()\n self.setGraphVal('fs_calls', 'calls', count)\n if self.hasGraph('fs_channels'):\n count = fs.getChannelCount()\n self.setGraphVal('fs_channels', 'channels', count)", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef autoconf(self):\n fs = FSinfo(self._fshost, self._fsport, self._fspass)\n return fs is not None", "response": "Implements Munin Plugin Auto - Configuration Option.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsolving the sparse linear system Ax = b.", "response": "def spsolve(A, b):\n \"\"\"Solve the sparse linear system Ax=b, where b may be a vector or a matrix.\n\n Parameters\n ----------\n A : ndarray or sparse matrix\n The square matrix A will be converted into CSC or CSR form\n b : ndarray or sparse matrix\n The matrix or vector representing the right hand side of the equation.\n\n Returns\n -------\n x : ndarray or sparse matrix\n the solution of the sparse linear equation.\n If b is a vector, then x is a vector of size A.shape[0]\n If b is a matrix, then x is a matrix of size (A.shape[0],)+b.shape[1:]\n\n \"\"\"\n x = UmfpackLU(A).solve(b)\n\n if b.ndim == 2 and b.shape[1] == 1:\n # compatibility with scipy.sparse.spsolve quirk\n return x.ravel()\n else:\n return x"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsolve linear equation A x = b for all the entries in the logarithm A.", "response": "def solve(self, b):\n \"\"\"\n Solve linear equation A x = b for x\n\n Parameters\n ----------\n b : ndarray\n Right-hand side of the matrix equation. Can be vector or a matrix.\n\n Returns\n -------\n x : ndarray\n Solution to the matrix equation\n\n \"\"\"\n if isspmatrix(b):\n b = b.toarray()\n\n if b.shape[0] != self._A.shape[1]:\n raise ValueError(\"Shape of b is not compatible with that of A\")\n\n b_arr = asarray(b, dtype=self._A.dtype).reshape(b.shape[0], -1)\n x = np.zeros((self._A.shape[0], b_arr.shape[1]), dtype=self._A.dtype)\n for j in range(b_arr.shape[1]):\n x[:,j] = self.umf.solve(UMFPACK_A, self._A, b_arr[:,j], autoTranspose=True)\n return x.reshape((self._A.shape[0],) + b.shape[1:])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef recall_matrix(egg, match='exact', distance='euclidean', features=None):\n\n if match in ['best', 'smooth']:\n if not features:\n features = [k for k,v in egg.pres.loc[0][0].values[0].items() if k!='item']\n if not features:\n raise('No features found. Cannot match with best or smooth strategy')\n\n if not isinstance(features, list):\n features = [features]\n\n if match=='exact':\n features=['item']\n return _recmat_exact(egg.pres, egg.rec, features)\n else:\n return _recmat_smooth(egg.pres, egg.rec, features, distance, match)", "response": "Compute the recall matrix given a list of presented and list of recalled words and a distance function."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef retrieveVals(self):\n net_info = NetstatInfo()\n if self.hasGraph('netstat_conn_status'):\n stats = net_info.getTCPportConnStatus(include_listen=True)\n for fname in ('listen', 'established', 'syn_sent', 'syn_recv',\n 'fin_wait1', 'fin_wait2', 'time_wait', \n 'close','close_wait', 'last_ack', 'closing', \n 'unknown',):\n self.setGraphVal('netstat_conn_status', fname, \n stats.get(fname,0))\n if self.hasGraph('netstat_conn_server'):\n stats = net_info.getTCPportConnCount(localport=self._port_list)\n for srv in self._srv_list:\n numconn = 0\n for port in self._srv_dict[srv]:\n numconn += stats.get(port, 0)\n self.setGraphVal('netstat_conn_server', srv, numconn)", "response": "Retrieve values for graphs."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _parseFreePBXconf(self):\n amiuser = None\n amipass = None\n if os.path.isfile(confFileFreePBX):\n try:\n fp = open(confFileFreePBX, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading FreePBX configuration file: %s'\n % confFileFreePBX)\n for (key, val) in re.findall('^(AMPMGR\\w+)\\s*=\\s*(\\S+)\\s*$',\n data, re.MULTILINE):\n if key == 'AMPMGRUSER':\n amiuser = val\n elif key == 'AMPMGRPASS':\n amipass = val\n if amiuser and amipass:\n self._amiuser = amiuser\n self._amipass = amipass\n return True\n return False", "response": "Parses the FreePBX configuration file and sets the attributes self. _amiuser and self. _amipass attributes."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse Asterisk configuration file for a specific Asterisk class. Returns True on success.", "response": "def _parseAsteriskConf(self):\n \"\"\"Parses Asterisk configuration file /etc/asterisk/manager.conf for\n user and password for Manager Interface. Returns True on success.\n \n @return: True if configuration file is found and parsed successfully.\n \n \"\"\"\n if os.path.isfile(confFileAMI):\n try:\n fp = open(confFileAMI, 'r')\n data = fp.read()\n fp.close()\n except:\n raise IOError('Failed reading Asterisk configuration file: %s'\n % confFileAMI)\n mobj = re.search('^\\[(\\w+)\\]\\s*\\r{0,1}\\nsecret\\s*=\\s*(\\S+)\\s*$', \n data, re.MULTILINE)\n if mobj:\n self._amiuser = mobj.group(1)\n self._amipass = mobj.group(2)\n return True\n return False"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _connect(self):\n try:\n if sys.version_info[:2] >= (2,6):\n self._conn = telnetlib.Telnet(self._amihost, self._amiport, \n connTimeout)\n else:\n self._conn = telnetlib.Telnet(self._amihost, self._amiport)\n except:\n raise Exception(\n \"Connection to Asterisk Manager Interface on \"\n \"host %s and port %s failed.\"\n % (self._amihost, self._amiport)\n )", "response": "Connect to Asterisk Manager Interface."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _sendAction(self, action, attrs=None, chan_vars=None):\n self._conn.write(\"Action: %s\\r\\n\" % action)\n if attrs:\n for (key,val) in attrs:\n self._conn.write(\"%s: %s\\r\\n\" % (key, val))\n if chan_vars:\n for (key,val) in chan_vars:\n self._conn.write(\"Variable: %s=%s\\r\\n\" % (key, val))\n self._conn.write(\"\\r\\n\")", "response": "Send an action to Asterisk Manager Interface."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads and parse response from Asterisk Manager Interface.", "response": "def _getResponse(self):\n \"\"\"Read and parse response from Asterisk Manager Interface.\n \n @return: Dictionary with response key-value pairs.\n\n \"\"\"\n resp_dict= dict()\n resp_str = self._conn.read_until(\"\\r\\n\\r\\n\", connTimeout)\n for line in resp_str.split(\"\\r\\n\"):\n mobj = re.match('(\\w+):\\s*(\\S.*)$', line);\n if mobj:\n resp_dict[mobj.group(1)] = mobj.group(2)\n else:\n mobj = re.match('(.*)--END COMMAND--\\s*$', line, flags=re.DOTALL)\n if mobj:\n resp_dict['command_response'] = mobj.group(1)\n return resp_dict"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _getGreeting(self):\n greeting = self._conn.read_until(\"\\r\\n\", connTimeout)\n mobj = re.match('Asterisk Call Manager\\/([\\d\\.]+)\\s*$', greeting)\n if mobj:\n self._ami_version = util.SoftwareVersion(mobj.group(1))\n else:\n raise Exception(\"Asterisk Manager Interface version cannot be determined.\")", "response": "Read and parse Asterisk Manager Interface Greeting to determine and\n set Manager Interface version."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nquerying Asterisk Manager Interface for Asterisk Version to configure with multiple versions of Asterisk Version.", "response": "def _initAsteriskVersion(self):\n \"\"\"Query Asterisk Manager Interface for Asterisk Version to configure\n system for compatibility with multiple versions\n .\n CLI Command - core show version\n\n \"\"\"\n if self._ami_version > util.SoftwareVersion('1.0'):\n cmd = \"core show version\"\n else:\n cmd = \"show version\"\n cmdresp = self.executeCommand(cmd)\n mobj = re.match('Asterisk\\s*(SVN-branch-|\\s)(\\d+(\\.\\d+)*)', cmdresp)\n if mobj:\n self._asterisk_version = util.SoftwareVersion(mobj.group(2))\n else:\n raise Exception('Asterisk version cannot be determined.')"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _login(self):\n self._sendAction(\"login\", (\n (\"Username\", self._amiuser),\n (\"Secret\", self._amipass),\n (\"Events\", \"off\"),\n ))\n resp = self._getResponse()\n if resp.get(\"Response\") == \"Success\":\n return True\n else:\n raise Exception(\"Authentication to Asterisk Manager Interface Failed.\")", "response": "Login to Asterisk Manager Interface."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsends Command to Asterisk Manager Interface to execute CLI Command.", "response": "def executeCommand(self, command):\n \"\"\"Send Action to Asterisk Manager Interface to execute CLI Command.\n \n @param command: CLI command to execute.\n @return: Command response string.\n\n \"\"\"\n self._sendAction(\"Command\", (\n (\"Command\", command),\n ))\n resp = self._getResponse()\n result = resp.get(\"Response\")\n if result == \"Follows\":\n return resp.get(\"command_response\")\n elif result == \"Error\":\n raise Exception(\"Execution of Asterisk Manager Interface Command \"\n \"(%s) failed with error message: %s\" % \n (command, str(resp.get(\"Message\"))))\n else:\n raise Exception(\"Execution of Asterisk Manager Interface Command \"\n \"failed: %s\" % command)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _initModuleList(self):\n if self.checkVersion('1.4'):\n cmd = \"module show\"\n else:\n cmd = \"show modules\"\n cmdresp = self.executeCommand(cmd)\n self._modules = set()\n for line in cmdresp.splitlines()[1:-1]:\n mobj = re.match('\\s*(\\S+)\\s', line)\n if mobj:\n self._modules.add(mobj.group(1).lower())", "response": "Query Asterisk Manager Interface to initialize internal list of \n loaded modules."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nquerying Asterisk Manager Interface to initialize internal list of available applications.", "response": "def _initApplicationList(self):\n \"\"\"Query Asterisk Manager Interface to initialize internal list of \n available applications.\n \n CLI Command - core show applications\n \n \"\"\"\n if self.checkVersion('1.4'):\n cmd = \"core show applications\"\n else:\n cmd = \"show applications\"\n cmdresp = self.executeCommand(cmd)\n self._applications = set()\n for line in cmdresp.splitlines()[1:-1]:\n mobj = re.match('\\s*(\\S+):', line)\n if mobj:\n self._applications.add(mobj.group(1).lower())"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _initChannelTypesList(self):\n if self.checkVersion('1.4'):\n cmd = \"core show channeltypes\"\n else:\n cmd = \"show channeltypes\"\n cmdresp = self.executeCommand(cmd)\n self._chantypes = set()\n for line in cmdresp.splitlines()[2:]:\n mobj = re.match('\\s*(\\S+)\\s+.*\\s+(yes|no)\\s+', line)\n if mobj:\n self._chantypes.add(mobj.group(1).lower())", "response": "Query Asterisk Manager Interface to initialize internal list of \n supported channel types."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning True if mod is among the loaded modules.", "response": "def hasModule(self, mod):\n \"\"\"Returns True if mod is among the loaded modules.\n \n @param mod: Module name.\n @return: Boolean \n \n \"\"\"\n if self._modules is None:\n self._initModuleList()\n return mod in self._modules"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn True if app is among the loaded modules.", "response": "def hasApplication(self, app):\n \"\"\"Returns True if app is among the loaded modules.\n \n @param app: Module name.\n @return: Boolean \n \n \"\"\"\n if self._applications is None:\n self._initApplicationList()\n return app in self._applications"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning True if chan is among the supported channel types.", "response": "def hasChannelType(self, chan):\n \"\"\"Returns True if chan is among the supported channel types.\n \n @param app: Module name.\n @return: Boolean \n \n \"\"\"\n if self._chantypes is None:\n self._initChannelTypesList()\n return chan in self._chantypes"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nqueries Asterisk Manager Interface for defined codecs.", "response": "def getCodecList(self):\n \"\"\"Query Asterisk Manager Interface for defined codecs.\n \n CLI Command - core show codecs\n \n @return: Dictionary - Short Name -> (Type, Long Name)\n \n \"\"\"\n if self.checkVersion('1.4'):\n cmd = \"core show codecs\"\n else:\n cmd = \"show codecs\"\n cmdresp = self.executeCommand(cmd)\n info_dict = {}\n for line in cmdresp.splitlines():\n mobj = re.match('\\s*(\\d+)\\s+\\((.+)\\)\\s+\\((.+)\\)\\s+(\\w+)\\s+(\\w+)\\s+\\((.+)\\)$',\n line)\n if mobj:\n info_dict[mobj.group(5)] = (mobj.group(4), mobj.group(6))\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nquery Asterisk Manager Interface for Channel Stats.", "response": "def getChannelStats(self, chantypes=('dahdi', 'zap', 'sip', 'iax2', 'local')):\n \"\"\"Query Asterisk Manager Interface for Channel Stats.\n \n CLI Command - core show channels\n\n @return: Dictionary of statistics counters for channels.\n Number of active channels for each channel type.\n\n \"\"\"\n if self.checkVersion('1.4'):\n cmd = \"core show channels\"\n else:\n cmd = \"show channels\"\n cmdresp = self.executeCommand(cmd)\n info_dict ={}\n for chanstr in chantypes:\n chan = chanstr.lower()\n if chan in ('zap', 'dahdi'):\n info_dict['dahdi'] = 0\n info_dict['mix'] = 0\n else:\n info_dict[chan] = 0\n for k in ('active_calls', 'active_channels', 'calls_processed'):\n info_dict[k] = 0\n regexstr = ('(%s)\\/(\\w+)' % '|'.join(chantypes)) \n for line in cmdresp.splitlines():\n mobj = re.match(regexstr, \n line, re.IGNORECASE)\n if mobj:\n chan_type = mobj.group(1).lower()\n chan_id = mobj.group(2).lower()\n if chan_type == 'dahdi' or chan_type == 'zap':\n if chan_id == 'pseudo':\n info_dict['mix'] += 1\n else:\n info_dict['dahdi'] += 1\n else:\n info_dict[chan_type] += 1\n continue\n\n mobj = re.match('(\\d+)\\s+(active channel|active call|calls processed)', \n line, re.IGNORECASE)\n if mobj:\n if mobj.group(2) == 'active channel':\n info_dict['active_channels'] = int(mobj.group(1))\n elif mobj.group(2) == 'active call':\n info_dict['active_calls'] = int(mobj.group(1))\n elif mobj.group(2) == 'calls processed':\n info_dict['calls_processed'] = int(mobj.group(1))\n continue\n\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getPeerStats(self, chantype):\n chan = chantype.lower()\n if not self.hasChannelType(chan):\n return None\n if chan == 'iax2':\n cmd = \"iax2 show peers\"\n elif chan == 'sip':\n cmd = \"sip show peers\"\n else:\n raise AttributeError(\"Invalid channel type in query for Peer Stats.\")\n cmdresp = self.executeCommand(cmd)\n \n info_dict = dict(\n online = 0, unreachable = 0, lagged = 0, \n unknown = 0, unmonitored = 0)\n for line in cmdresp.splitlines():\n if re.search('ok\\s+\\(\\d+\\s+ms\\)\\s*$', line, re.IGNORECASE):\n info_dict['online'] += 1\n else:\n mobj = re.search('(unreachable|lagged|unknown|unmonitored)\\s*$', \n line, re.IGNORECASE)\n if mobj:\n info_dict[mobj.group(1).lower()] += 1\n \n return info_dict", "response": "Query Asterisk Manager Interface for SIP or IAX2 Peer Stats."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nqueries Asterisk Manager Interface for Channel Stats.", "response": "def getVoIPchanStats(self, chantype, \n codec_list=('ulaw', 'alaw', 'gsm', 'g729')):\n \"\"\"Query Asterisk Manager Interface for SIP / IAX2 Channel / Codec Stats.\n \n CLI Commands - sip show channels\n iax2 show channnels\n \n @param chantype: Must be 'sip' or 'iax2'.\n @param codec_list: List of codec names to parse.\n (Codecs not in the list are summed up to the other \n count.)\n @return: Dictionary of statistics counters for Active VoIP \n Channels.\n\n \"\"\"\n chan = chantype.lower()\n if not self.hasChannelType(chan):\n return None\n if chan == 'iax2':\n cmd = \"iax2 show channels\"\n elif chan == 'sip':\n cmd = \"sip show channels\"\n else:\n raise AttributeError(\"Invalid channel type in query for Channel Stats.\")\n cmdresp = self.executeCommand(cmd)\n lines = cmdresp.splitlines()\n headers = re.split('\\s\\s+', lines[0])\n try:\n idx = headers.index('Format')\n except ValueError:\n try:\n idx = headers.index('Form')\n except:\n raise Exception(\"Error in parsing header line of %s channel stats.\" \n % chan)\n codec_list = tuple(codec_list) + ('other', 'none')\n info_dict = dict([(k,0) for k in codec_list])\n for line in lines[1:-1]:\n codec = None\n cols = re.split('\\s\\s+', line)\n colcodec = cols[idx]\n mobj = re.match('0x\\w+\\s\\((\\w+)\\)$', colcodec)\n if mobj:\n codec = mobj.group(1).lower()\n elif re.match('\\w+$', colcodec):\n codec = colcodec.lower()\n if codec:\n if codec in info_dict:\n info_dict[codec] += 1\n elif codec == 'nothing' or codec[0:4] == 'unkn':\n info_dict['none'] += 1\n else:\n info_dict['other'] += 1\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getConferenceStats(self):\n if not self.hasConference():\n return None\n if self.checkVersion('1.6'):\n cmd = \"meetme list\"\n else:\n cmd = \"meetme\"\n cmdresp = self.executeCommand(cmd)\n\n info_dict = dict(active_conferences = 0, conference_users = 0)\n for line in cmdresp.splitlines():\n mobj = re.match('\\w+\\s+0(\\d+)\\s', line)\n if mobj:\n info_dict['active_conferences'] += 1\n info_dict['conference_users'] += int(mobj.group(1))\n\n return info_dict", "response": "Query Asterisk Manager Interface for Conference Room Stats. meetme list\n "} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getVoicemailStats(self):\n if not self.hasVoicemail():\n return None\n if self.checkVersion('1.4'):\n cmd = \"voicemail show users\"\n else:\n cmd = \"show voicemail users\"\n cmdresp = self.executeCommand(cmd)\n\n info_dict = dict(accounts = 0, avg_messages = 0, max_messages = 0, \n total_messages = 0)\n for line in cmdresp.splitlines():\n mobj = re.match('\\w+\\s+\\w+\\s+.*\\s+(\\d+)\\s*$', line)\n if mobj:\n msgs = int(mobj.group(1))\n info_dict['accounts'] += 1\n info_dict['total_messages'] += msgs\n if msgs > info_dict['max_messages']:\n info_dict['max_messages'] = msgs\n if info_dict['accounts'] > 0:\n info_dict['avg_messages'] = (float(info_dict['total_messages']) \n / info_dict['accounts'])\n \n return info_dict", "response": "Query Asterisk Manager Interface for Voicemail Stats."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nqueries Asterisk Manager Interface for Trunk Stats.", "response": "def getTrunkStats(self, trunkList):\n \"\"\"Query Asterisk Manager Interface for Trunk Stats.\n \n CLI Command - core show channels\n\n @param trunkList: List of tuples of one of the two following types:\n (Trunk Name, Regular Expression)\n (Trunk Name, Regular Expression, MIN, MAX)\n @return: Dictionary of trunk utilization statistics.\n\n \"\"\"\n re_list = []\n info_dict = {}\n for filt in trunkList:\n info_dict[filt[0]] = 0\n re_list.append(re.compile(filt[1], re.IGNORECASE))\n \n if self.checkVersion('1.4'):\n cmd = \"core show channels\"\n else:\n cmd = \"show channels\"\n cmdresp = self.executeCommand(cmd)\n\n for line in cmdresp.splitlines():\n for idx in range(len(re_list)):\n recomp = re_list[idx]\n trunkid = trunkList[idx][0]\n mobj = recomp.match(line)\n if mobj:\n if len(trunkList[idx]) == 2:\n info_dict[trunkid] += 1\n continue\n elif len(trunkList[idx]) == 4:\n num = mobj.groupdict().get('num')\n if num is not None:\n (vmin,vmax) = trunkList[idx][2:4]\n if int(num) >= int(vmin) and int(num) <= int(vmax):\n info_dict[trunkid] += 1\n continue\n return info_dict"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getQueueStats(self):\n if not self.hasQueue():\n return None\n info_dict = {}\n if self.checkVersion('1.4'):\n cmd = \"queue show\"\n else:\n cmd = \"show queues\"\n cmdresp = self.executeCommand(cmd)\n \n queue = None\n ctxt = None\n member_states = (\"unknown\", \"not in use\", \"in use\", \"busy\", \"invalid\", \n \"unavailable\", \"ringing\", \"ring+inuse\", \"on hold\", \n \"total\")\n member_state_dict = dict([(k.lower().replace(' ', '_'),0) \n for k in member_states]) \n for line in cmdresp.splitlines():\n mobj = re.match(r\"([\\w\\-]+)\\s+has\\s+(\\d+)\\s+calls\\s+\"\n r\"\\(max (\\d+|unlimited)\\)\\s+in\\s+'(\\w+)'\\s+strategy\\s+\"\n r\"\\((.+)\\),\\s+W:(\\d+),\\s+C:(\\d+),\\s+A:(\\d+),\\s+\"\n r\"SL:([\\d\\.]+)%\\s+within\\s+(\\d+)s\", line)\n if mobj:\n ctxt = None\n queue = mobj.group(1)\n info_dict[queue] = {}\n info_dict[queue]['queue_len'] = int(mobj.group(2))\n try:\n info_dict[queue]['queue_maxlen'] = int(mobj.group(3))\n except ValueError:\n info_dict[queue]['queue_maxlen'] = None\n info_dict[queue]['strategy'] = mobj.group(4)\n for tkn in mobj.group(5).split(','):\n mobjx = re.match(r\"\\s*(\\d+)s\\s+(\\w+)\\s*\", tkn)\n if mobjx:\n info_dict[queue]['avg_' + mobjx.group(2)] = int(mobjx.group(1))\n info_dict[queue]['queue_weight'] = int(mobj.group(6))\n info_dict[queue]['calls_completed'] = int(mobj.group(7))\n info_dict[queue]['calls_abandoned'] = int(mobj.group(8))\n info_dict[queue]['sla_pcent'] = float(mobj.group(9))\n info_dict[queue]['sla_cutoff'] = int(mobj.group(10))\n info_dict[queue]['members'] = member_state_dict.copy() \n continue\n mobj = re.match('\\s+(Members|Callers):\\s*$', line)\n if mobj:\n ctxt = mobj.group(1).lower()\n continue\n if ctxt == 'members':\n mobj = re.match(r\"\\s+\\S.*\\s\\((.*)\\)\\s+has\\s+taken.*calls\", line)\n if mobj:\n info_dict[queue]['members']['total'] += 1\n state = mobj.group(1).lower().replace(' ', '_')\n if info_dict[queue]['members'].has_key(state):\n info_dict[queue]['members'][state] += 1\n else:\n raise AttributeError(\"Undefined queue member state %s\"\n % state)\n continue\n return info_dict", "response": "Query Asterisk Manager Interface for Queue Stats."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getFaxStatsCounters(self):\n if not self.hasFax():\n return None\n info_dict = {}\n cmdresp = self.executeCommand('fax show stats')\n ctxt = 'general'\n for section in cmdresp.strip().split('\\n\\n')[1:]:\n i = 0\n for line in section.splitlines():\n mobj = re.match('(\\S.*\\S)\\s*:\\s*(\\d+)\\s*$', line)\n if mobj:\n if not info_dict.has_key(ctxt):\n info_dict[ctxt] = {}\n info_dict[ctxt][mobj.group(1).lower()] = int(mobj.group(2).lower())\n elif i == 0:\n ctxt = line.strip().lower()\n i += 1 \n return info_dict", "response": "Query Asterisk Manager Interface for Fax Stats."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getFaxStatsSessions(self):\n if not self.hasFax():\n return None\n info_dict = {}\n info_dict['total'] = 0\n fax_types = ('g.711', 't.38')\n fax_operations = ('send', 'recv')\n fax_states = ('uninitialized', 'initialized', 'open', \n 'active', 'inactive', 'complete', 'unknown',)\n info_dict['type'] = dict([(k,0) for k in fax_types])\n info_dict['operation'] = dict([(k,0) for k in fax_operations])\n info_dict['state'] = dict([(k,0) for k in fax_states])\n cmdresp = self.executeCommand('fax show sessions')\n sections = cmdresp.strip().split('\\n\\n')\n if len(sections) >= 3:\n for line in sections[1][1:]:\n cols = re.split('\\s\\s+', line)\n if len(cols) == 7:\n info_dict['total'] += 1\n if cols[3].lower() in fax_types:\n info_dict['type'][cols[3].lower()] += 1\n if cols[4] == 'receive':\n info_dict['operation']['recv'] += 1\n elif cols[4] == 'send':\n info_dict['operation']['send'] += 1\n if cols[5].lower() in fax_states:\n info_dict['state'][cols[5].lower()] += 1\n return info_dict", "response": "Query Asterisk Manager Interface for Fax Stats."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nretrieves values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n for iface in self._ifaceList:\n if self._reqIfaceList is None or iface in self._reqIfaceList:\n if (self.graphEnabled('wanpipe_traffic') \n or self.graphEnabled('wanpipe_errors')):\n stats = self._ifaceStats.get(iface)\n if stats:\n graph_name = 'wanpipe_traffic_%s' % iface\n if self.hasGraph(graph_name):\n for field in ('rxpackets', 'txpackets'):\n self.setGraphVal(graph_name, field, \n stats.get(field))\n graph_name = 'wanpipe_errors_%s' % iface\n if self.hasGraph(graph_name):\n for field in ('rxerrs', 'txerrs', 'rxframe', 'txcarrier',\n 'rxdrop', 'txdrop', 'rxfifo', 'txfifo'):\n self.setGraphVal(graph_name, field, \n stats.get(field))\n if (self.graphEnabled('wanpipe_pri_errors') \n or self.graphEnabled('wanpipe_rxlevel')):\n try:\n stats = self._wanpipeInfo.getPRIstats(iface)\n except:\n stats = None\n if stats:\n graph_name = 'wanpipe_pri_errors_%s' % iface\n if self.hasGraph(graph_name):\n for field in ('linecodeviolation', \n 'farendblockerrors',\n 'crc4errors', 'faserrors'):\n self.setGraphVal(graph_name, field, \n stats.get(field))\n if self.hasGraph('wanpipe_rxlevel'):\n self.setGraphVal('wanpipe_pri_rxlevel', \n iface, stats.get('rxlevel'))"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef simulate_list(nwords=16, nrec=10, ncats=4):\n\n # load wordpool\n wp = pd.read_csv('data/cut_wordpool.csv')\n\n # get one list\n wp = wp[wp['GROUP']==np.random.choice(list(range(16)), 1)[0]].sample(16)\n\n wp['COLOR'] = [[int(np.random.rand() * 255) for i in range(3)] for i in range(16)]", "response": "A function to simulate a list"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nretrieving values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\"\n if self._genStats is None:\n self._genStats = self._dbconn.getStats()\n if self._genVars is None:\n self._genVars = self._dbconn.getParams()\n if self.hasGraph('mysql_connections'):\n self.setGraphVal('mysql_connections', 'conn',\n self._genStats.get('Connections'))\n self.setGraphVal('mysql_connections', 'abort_conn',\n self._genStats.get('Aborted_connects'))\n self.setGraphVal('mysql_connections', 'abort_client',\n self._genStats.get('Aborted_clients'))\n if self.hasGraph('mysql_traffic'):\n self.setGraphVal('mysql_traffic', 'rx',\n self._genStats.get('Bytes_received'))\n self.setGraphVal('mysql_traffic', 'tx',\n self._genStats.get('Bytes_sent'))\n if self.graphEnabled('mysql_slowqueries'):\n self.setGraphVal('mysql_slowqueries', 'queries',\n self._genStats.get('Slow_queries'))\n if self.hasGraph('mysql_rowmodifications'):\n self.setGraphVal('mysql_rowmodifications', 'insert',\n self._genStats.get('Handler_write'))\n self.setGraphVal('mysql_rowmodifications', 'update',\n self._genStats.get('Handler_update'))\n self.setGraphVal('mysql_rowmodifications', 'delete',\n self._genStats.get('Handler_delete'))\n if self.hasGraph('mysql_rowreads'):\n for field in self.getGraphFieldList('mysql_rowreads'):\n self.setGraphVal('mysql_rowreads', field, \n self._genStats.get('Handler_read_%s' % field))\n if self.hasGraph('mysql_tablelocks'):\n self.setGraphVal('mysql_tablelocks', 'waited',\n self._genStats.get('Table_locks_waited'))\n self.setGraphVal('mysql_tablelocks', 'immediate',\n self._genStats.get('Table_locks_immediate'))\n if self.hasGraph('mysql_threads'):\n self.setGraphVal('mysql_threads', 'running',\n self._genStats.get('Threads_running'))\n self.setGraphVal('mysql_threads', 'idle',\n self._genStats.get('Threads_connected')\n - self._genStats.get('Threads_running'))\n self.setGraphVal('mysql_threads', 'cached',\n self._genStats.get('Threads_cached'))\n self.setGraphVal('mysql_threads', 'total',\n self._genStats.get('Threads_connected') \n + self._genStats.get('Threads_cached'))\n if self.hasGraph('mysql_commits_rollbacks'):\n self.setGraphVal('mysql_commits_rollbacks', 'commit',\n self._genStats.get('Handler_commit'))\n self.setGraphVal('mysql_commits_rollbacks', 'rollback',\n self._genStats.get('Handler_rollback'))\n if self.hasGraph('mysql_qcache_memory'):\n try:\n total = self._genVars['query_cache_size']\n free = self._genStats['Qcache_free_memory']\n used = total - free\n except KeyError:\n free = None\n used = None\n self.setGraphVal('mysql_qcache_memory', 'used', used)\n self.setGraphVal('mysql_qcache_memory', 'free', free)\n if self.hasGraph('mysql_qcache_hits'):\n try:\n hits = self._genStats['Qcache_hits']\n misses = self._genStats['Com_select'] - hits\n except KeyError:\n hits = None\n misses = None\n self.setGraphVal('mysql_qcache_hits', 'hits', hits)\n self.setGraphVal('mysql_qcache_hits', 'misses', misses)\n \n if self.hasGraph('mysql_qcache_prunes'):\n self.setGraphVal('mysql_qcache_prunes', 'insert', \n self._genStats.get('Qcache_inserts'))\n self.setGraphVal('mysql_qcache_prunes', 'prune',\n self._genStats.get('Qcache_lowmem_prunes'))\n if self.hasGraph('mysql_proc_status'):\n self._procStatus = self._dbconn.getProcessStatus()\n if self._procStatus:\n stats = {}\n for field in self.getGraphFieldList('mysql_proc_status'):\n stats[field] = 0\n for (k, v) in self._procStatus.items():\n if stats.has_key(k):\n stats[k] = v\n else:\n stats['unknown'] += v\n for (k,v) in stats.items():\n self.setGraphVal('mysql_proc_status', k, v)\n if self.hasGraph('mysql_proc_db'):\n self._procDB = self._dbconn.getProcessDatabase()\n for db in self._dbList:\n self.setGraphVal('mysql_proc_db', db, self._procDB.get(db, 0))\n \n if self.engineIncluded('myisam'):\n \n if self.hasGraph('mysql_myisam_key_buffer_util'):\n try:\n bsize = self._genVars['key_cache_block_size']\n total = self._genVars['key_buffer_size']\n free = self._genStats['Key_blocks_unused'] * bsize\n dirty = self._genStats['Key_blocks_not_flushed'] * bsize\n clean = total - free - dirty\n except KeyError:\n total = None\n free = None\n dirty = None\n clean = None\n for (field,val) in (('dirty', dirty), \n ('clean', clean),\n ('free', free),\n ('total', total)):\n self.setGraphVal('mysql_myisam_key_buffer_util', \n field, val)\n if self.hasGraph('mysql_myisam_key_read_reqs'):\n try:\n misses = self._genStats['Key_reads']\n hits = (self._genStats['Key_read_requests']\n - misses)\n except KeyError:\n misses = None\n hits = None\n self.setGraphVal('mysql_myisam_key_read_reqs', 'disk', misses)\n self.setGraphVal('mysql_myisam_key_read_reqs', 'buffer', hits)\n \n if self.engineIncluded('innodb'):\n \n if self.hasGraph('mysql_innodb_buffer_pool_util'):\n self._genStats['Innodb_buffer_pool_pages_clean'] = (\n self._genStats.get('Innodb_buffer_pool_pages_data')\n - self._genStats.get('Innodb_buffer_pool_pages_dirty'))\n page_size = int(self._genStats.get('Innodb_page_size'))\n for field in ('dirty', 'clean', 'misc', 'free', 'total'):\n self.setGraphVal('mysql_innodb_buffer_pool_util', \n field, \n self._genStats.get('Innodb_buffer_pool_pages_%s'\n % field)\n * page_size)\n if self.hasGraph('mysql_innodb_buffer_pool_activity'):\n for field in ('created', 'read', 'written'):\n self.setGraphVal('mysql_innodb_buffer_pool_activity', field, \n self._genStats.get('Innodb_pages_%s' % field))\n if self.hasGraph('mysql_innodb_buffer_pool_read_reqs'):\n try:\n misses = self._genStats['Innodb_buffer_pool_reads']\n hits = (self._genStats['Innodb_buffer_pool_read_requests']\n - misses)\n except KeyError:\n misses = None\n hits = None\n self.setGraphVal('mysql_innodb_buffer_pool_read_reqs', 'disk', \n misses)\n self.setGraphVal('mysql_innodb_buffer_pool_read_reqs', 'buffer', \n hits)\n if self.hasGraph('mysql_innodb_row_ops'):\n for field in ('inserted', 'updated', 'deleted', 'read'):\n self.setGraphVal('mysql_innodb_row_ops', field, \n self._genStats.get('Innodb_rows_%s' % field))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef engineIncluded(self, name):\n if self._engines is None:\n self._engines = self._dbconn.getStorageEngines()\n return self.envCheckFilter('engine', name) and name in self._engines", "response": "Utility method to check if a storage engine is included in graphs.\negerName is a string that represents the name of the storage engine."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting disk space usage.", "response": "def getSpaceUse(self):\n \"\"\"Get disk space usage.\n \n @return: Dictionary of filesystem space utilization stats for filesystems.\n \n \"\"\"\n stats = {}\n try:\n out = subprocess.Popen([dfCmd, \"-Pk\"], \n stdout=subprocess.PIPE).communicate()[0]\n except:\n raise Exception('Execution of command %s failed.' % dfCmd)\n lines = out.splitlines()\n if len(lines) > 1:\n for line in lines[1:]:\n fsstats = {}\n cols = line.split()\n fsstats['device'] = cols[0]\n fsstats['type'] = self._fstypeDict[cols[5]]\n fsstats['total'] = 1024 * int(cols[1])\n fsstats['inuse'] = 1024 * int(cols[2])\n fsstats['avail'] = 1024 * int(cols[3])\n fsstats['inuse_pcent'] = int(cols[4][:-1])\n stats[cols[5]] = fsstats\n return stats"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving values for graphs.", "response": "def retrieveVals(self):\n \"\"\"Retrieve values for graphs.\"\"\" \n stats = self._dbconn.getDatabaseStats()\n databases = stats.get('databases')\n totals = stats.get('totals')\n if self.hasGraph('pg_connections'):\n limit = self._dbconn.getParam('max_connections')\n self.setGraphVal('pg_connections', 'max_conn', limit)\n for (db, dbstats) in databases.iteritems():\n if self.dbIncluded(db):\n self.setGraphVal('pg_connections', db, \n dbstats['numbackends'])\n self.setGraphVal('pg_connections', 'total', totals['numbackends'])\n if self.hasGraph('pg_diskspace'):\n for (db, dbstats) in databases.iteritems():\n if self.dbIncluded(db):\n self.setGraphVal('pg_diskspace', db, dbstats['disk_size'])\n self.setGraphVal('pg_diskspace', 'total', totals['disk_size'])\n if self.hasGraph('pg_blockreads'):\n self.setGraphVal('pg_blockreads', 'blk_hit', totals['blks_hit'])\n self.setGraphVal('pg_blockreads', 'blk_read', totals['blks_read'])\n if self.hasGraph('pg_xact'):\n self.setGraphVal('pg_xact', 'commits', totals['xact_commit'])\n self.setGraphVal('pg_xact', 'rollbacks', totals['xact_rollback'])\n if self.hasGraph('pg_tup_read'):\n self.setGraphVal('pg_tup_read', 'fetch', totals['tup_fetched'])\n self.setGraphVal('pg_tup_read', 'return', totals['tup_returned'])\n if self.hasGraph('pg_tup_write'):\n self.setGraphVal('pg_tup_write', 'delete', totals['tup_deleted'])\n self.setGraphVal('pg_tup_write', 'update', totals['tup_updated'])\n self.setGraphVal('pg_tup_write', 'insert', totals['tup_inserted'])\n lock_stats = None\n for lock_state in ('all', 'wait',):\n graph_name = \"pg_lock_%s\" % lock_state\n if self.hasGraph(graph_name):\n if lock_stats is None:\n lock_stats = self._dbconn.getLockStatsMode()\n mode_iter = iter(PgInfo.lockModes)\n for mode in ('AccessExcl', 'Excl', 'ShrRwExcl', 'Shr', \n 'ShrUpdExcl', 'RwExcl', 'RwShr', 'AccessShr',):\n self.setGraphVal(graph_name, mode, \n lock_stats[lock_state].get(mode_iter.next()))\n \n stats = None \n if self.hasGraph('pg_checkpoints'):\n if stats is None:\n stats = self._dbconn.getBgWriterStats()\n self.setGraphVal('pg_checkpoints', 'req', \n stats.get('checkpoints_req'))\n self.setGraphVal('pg_checkpoints', 'timed', \n stats.get('checkpoints_timed'))\n if self.hasGraph('pg_bgwriter'):\n if stats is None:\n stats = self._dbconn.getBgWriterStats()\n self.setGraphVal('pg_bgwriter', 'backend', \n stats.get('buffers_backend'))\n self.setGraphVal('pg_bgwriter', 'clean', \n stats.get('buffers_clean'))\n self.setGraphVal('pg_bgwriter', 'chkpoint', \n stats.get('buffers_checkpoint'))\n \n if self._detailGraphs:\n for (db, dbstats) in databases.iteritems():\n if self.dbIncluded(db):\n if self.hasGraph('pg_blockread_detail'):\n self.setGraphVal('pg_blockread_detail', db, \n dbstats['blks_hit'] + dbstats['blks_read'])\n for (graph_name, attr_name) in (\n ('pg_xact_commit_detail', 'xact_commit'),\n ('pg_xact_rollback_detail', 'xact_rollback'),\n ('pg_tup_return_detail', 'tup_returned'),\n ('pg_tup_fetch_detail', 'tup_fetched'),\n ('pg_tup_delete_detail', 'tup_deleted'),\n ('pg_tup_update_detail', 'tup_updated'),\n ('pg_tup_insert_detail', 'tup_inserted'),\n ):\n if self.hasGraph(graph_name):\n self.setGraphVal(graph_name, db, dbstats[attr_name])\n lock_stats_db = None\n for lock_state in ('all', 'wait',):\n graph_name = \"pg_lock_%s_detail\" % lock_state\n if self.hasGraph(graph_name):\n if lock_stats_db is None:\n lock_stats_db = self._dbconn.getLockStatsDB()\n self.setGraphVal(graph_name, db, \n lock_stats_db[lock_state].get(db, 0))\n \n if self._replGraphs:\n repl_stats = self._dbconn.getSlaveConflictStats()\n if self.hasGraph('pg_repl_conflicts'): \n for field in self.getGraphFieldList('pg_repl_conflicts'):\n self.setGraphVal('pg_repl_conflicts', field, \n repl_stats['totals'].get(\"confl_%s\" % field))\n if self._detailGraphs and self.hasGraph('pg_repl_conflicts_detail'):\n for (db, dbstats) in repl_stats['databases'].iteritems():\n if self.dbIncluded(db):\n self.setGraphVal('pg_repl_conflicts_detail', db,\n sum(dbstats.values()))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef connect(self, host, port):\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n sock.connect((host, port))\n self._reader = sock.makefile(mode='rb')\n self._writer = sock.makefile(mode='wb')", "response": "Connects via an RS - 485 to Ethernet adapter."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprocessing data ; returns when the reader signals EOF. data_changed_callback is notified when any data changes.", "response": "def process(self, data_changed_callback):\n \"\"\"Process data; returns when the reader signals EOF.\n Callback is notified when any data changes.\"\"\"\n # pylint: disable=too-many-locals,too-many-branches,too-many-statements\n while True:\n byte = self._reader.read(1)\n\n while True:\n # Search for FRAME_DLE + FRAME_STX\n if not byte:\n return\n if byte[0] == self.FRAME_DLE:\n next_byte = self._reader.read(1)\n if not next_byte:\n return\n if next_byte[0] == self.FRAME_STX:\n break\n else:\n continue\n byte = self._reader.read(1)\n\n frame = bytearray()\n byte = self._reader.read(1)\n\n while True:\n if not byte:\n return\n if byte[0] == self.FRAME_DLE:\n # Should be FRAME_ETX or 0 according to\n # the AQ-CO-SERIAL manual\n next_byte = self._reader.read(1)\n if not next_byte:\n return\n if next_byte[0] == self.FRAME_ETX:\n break\n elif next_byte[0] != 0:\n # Error?\n pass\n\n frame.append(byte[0])\n byte = self._reader.read(1)\n\n # Verify CRC\n frame_crc = int.from_bytes(frame[-2:], byteorder='big')\n frame = frame[:-2]\n\n calculated_crc = self.FRAME_DLE + self.FRAME_STX\n for byte in frame:\n calculated_crc += byte\n\n if frame_crc != calculated_crc:\n _LOGGER.warning('Bad CRC')\n continue\n\n frame_type = frame[0:2]\n frame = frame[2:]\n\n if frame_type == self.FRAME_TYPE_KEEP_ALIVE:\n # Keep alive\n # If a frame has been queued for transmit, send it.\n if not self._send_queue.empty():\n data = self._send_queue.get(block=False)\n self._writer.write(data['frame'])\n self._writer.flush()\n _LOGGER.info('Sent: %s', binascii.hexlify(data['frame']))\n\n try:\n if data['desired_states'] is not None:\n # Set a timer to verify the state changes\n # Wait 2 seconds as it can take a while for\n # the state to change.\n Timer(2.0, self._check_state, [data]).start()\n except KeyError:\n pass\n\n continue\n elif frame_type == self.FRAME_TYPE_KEY_EVENT:\n _LOGGER.info('Key: %s', binascii.hexlify(frame))\n elif frame_type == self.FRAME_TYPE_LEDS:\n _LOGGER.debug('LEDs: %s', binascii.hexlify(frame))\n # First 4 bytes are the LEDs that are on;\n # second 4 bytes_ are the LEDs that are flashing\n states = int.from_bytes(frame[0:4], byteorder='little')\n flashing_states = int.from_bytes(frame[4:8],\n byteorder='little')\n states |= flashing_states\n if (states != self._states\n or flashing_states != self._flashing_states):\n self._states = states\n self._flashing_states = flashing_states\n data_changed_callback(self)\n elif frame_type == self.FRAME_TYPE_PUMP_SPEED_REQUEST:\n value = int.from_bytes(frame[0:2], byteorder='big')\n _LOGGER.debug('Pump speed request: %d%%', value)\n if self._pump_speed != value:\n self._pump_speed = value\n data_changed_callback(self)\n elif frame_type == self.FRAME_TYPE_PUMP_STATUS:\n # Pump status messages sent out by Hayward VSP pumps\n self._multi_speed_pump = True\n speed = frame[2]\n # Power is in BCD\n power = ((((frame[3] & 0xf0) >> 4) * 1000)\n + (((frame[3] & 0x0f)) * 100)\n + (((frame[4] & 0xf0) >> 4) * 10)\n + (((frame[4] & 0x0f))))\n _LOGGER.debug('Pump speed: %d%%, power: %d watts',\n speed, power)\n if self._pump_power != power:\n self._pump_power = power\n data_changed_callback(self)\n elif frame_type == self.FRAME_TYPE_DISPLAY_UPDATE:\n parts = frame.decode('latin-1').split()\n _LOGGER.debug('Display update: %s', parts)\n\n try:\n if parts[0] == 'Pool' and parts[1] == 'Temp':\n # Pool Temp \u00b0[C|F]\n value = int(parts[2][:-2])\n if self._pool_temp != value:\n self._pool_temp = value\n self._is_metric = parts[2][-1:] == 'C'\n data_changed_callback(self)\n elif parts[0] == 'Spa' and parts[1] == 'Temp':\n # Spa Temp \u00b0[C|F]\n value = int(parts[2][:-2])\n if self._spa_temp != value:\n self._spa_temp = value\n self._is_metric = parts[2][-1:] == 'C'\n data_changed_callback(self)\n elif parts[0] == 'Air' and parts[1] == 'Temp':\n # Air Temp \u00b0[C|F]\n value = int(parts[2][:-2])\n if self._air_temp != value:\n self._air_temp = value\n self._is_metric = parts[2][-1:] == 'C'\n data_changed_callback(self)\n elif parts[0] == 'Pool' and parts[1] == 'Chlorinator':\n # Pool Chlorinator %\n value = int(parts[2][:-1])\n if self._pool_chlorinator != value:\n self._pool_chlorinator = value\n data_changed_callback(self)\n elif parts[0] == 'Spa' and parts[1] == 'Chlorinator':\n # Spa Chlorinator %\n value = int(parts[2][:-1])\n if self._spa_chlorinator != value:\n self._spa_chlorinator = value\n data_changed_callback(self)\n elif parts[0] == 'Salt' and parts[1] == 'Level':\n # Salt Level [g/L|PPM|\n value = float(parts[2])\n if self._salt_level != value:\n self._salt_level = value\n self._is_metric = parts[3] == 'g/L'\n data_changed_callback(self)\n elif parts[0] == 'Check' and parts[1] == 'System':\n # Check System \n value = ' '.join(parts[2:])\n if self._check_system_msg != value:\n self._check_system_msg = value\n data_changed_callback(self)\n except ValueError:\n pass\n else:\n _LOGGER.info('Unknown frame: %s %s',\n binascii.hexlify(frame_type),\n binascii.hexlify(frame))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsending a key to the send queue.", "response": "def send_key(self, key):\n \"\"\"Sends a key.\"\"\"\n _LOGGER.info('Queueing key %s', key)\n frame = self._get_key_event_frame(key)\n\n # Queue it to send immediately following the reception\n # of a keep-alive packet in an attempt to avoid bus collisions.\n self._send_queue.put({'frame': frame})"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef states(self):\n state_list = []\n for state in States:\n if state.value & self._states != 0:\n state_list.append(state)\n\n if (self._flashing_states & States.FILTER) != 0:\n state_list.append(States.FILTER_LOW_SPEED)\n\n return state_list", "response": "Returns a list containing the enabled states."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_state(self, state):\n # Check to see if we have a change request pending; if we do\n # return the value we expect it to change to.\n for data in list(self._send_queue.queue):\n desired_states = data['desired_states']\n for desired_state in desired_states:\n if desired_state['state'] == state:\n return desired_state['enabled']\n if state == States.FILTER_LOW_SPEED:\n return (States.FILTER.value & self._flashing_states) != 0\n return (state.value & self._states) != 0", "response": "Returns True if the specified state is enabled."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_state(self, state, enable):\n\n is_enabled = self.get_state(state)\n if is_enabled == enable:\n return True\n\n key = None\n desired_states = [{'state': state, 'enabled': not is_enabled}]\n\n if state == States.FILTER_LOW_SPEED:\n if not self._multi_speed_pump:\n return False\n # Send the FILTER key once.\n # If the pump is in high speed, it wil switch to low speed.\n # If the pump is off the retry mechanism will send an additional\n # FILTER key to switch into low speed.\n # If the pump is in low speed then we pretend the pump is off;\n # the retry mechanism will send an additional FILTER key\n # to switch into high speed.\n key = Keys.FILTER\n desired_states.append({'state': States.FILTER, 'enabled': True})\n else:\n # See if this state has a corresponding Key\n try:\n key = Keys[state.name]\n except KeyError:\n # TODO: send the appropriate combination of keys\n # to enable the state\n return False\n\n frame = self._get_key_event_frame(key)\n\n # Queue it to send immediately following the reception\n # of a keep-alive packet in an attempt to avoid bus collisions.\n self._send_queue.put({'frame': frame, 'desired_states': desired_states,\n 'retries': 10})\n\n return True", "response": "Sets the state of the Keyton."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndecorate a function by tracing the begining and the end of the function execution.", "response": "def trace(function, *args, **k) :\n\t\"\"\"Decorates a function by tracing the begining and\n\tend of the function execution, if doTrace global is True\"\"\"\n\n\tif doTrace : print (\"> \"+function.__name__, args, k)\n\tresult = function(*args, **k)\n\tif doTrace : print (\"< \"+function.__name__, args, k, \"->\", result)\n\treturn result"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting a geocode of a location.", "response": "def geocode(self, location) :\n\t\turl = QtCore.QUrl(\"http://maps.googleapis.com/maps/api/geocode/xml\")\n\t\turl.addQueryItem(\"address\", location)\n\t\turl.addQueryItem(\"sensor\", \"false\")\n\t\t\"\"\"\n\t\turl = QtCore.QUrl(\"http://maps.google.com/maps/geo/\")\n\t\turl.addQueryItem(\"q\", location)\n\t\turl.addQueryItem(\"output\", \"csv\")\n\t\turl.addQueryItem(\"sensor\", \"false\")\n\t\t\"\"\"\n\t\trequest = QtNetwork.QNetworkRequest(url)\n\t\treply = self.get(request)\n\t\twhile reply.isRunning() :\n\t\t\tQtGui.QApplication.processEvents()\n\n\t\treply.deleteLater()\n\t\tself.deleteLater()\n\t\treturn self._parseResult(reply)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nforcing a correlation matrix on a set of statistically distributed objects. This function works on objects in-place. Parameters ---------- params : array An array of of uv objects. corrmat : 2d-array The correlation matrix to be imposed", "response": "def correlate(params, corrmat):\r\n \"\"\"\r\n Force a correlation matrix on a set of statistically distributed objects.\r\n This function works on objects in-place.\r\n \r\n Parameters\r\n ----------\r\n params : array\r\n An array of of uv objects.\r\n corrmat : 2d-array\r\n The correlation matrix to be imposed\r\n \r\n \"\"\"\r\n # Make sure all inputs are compatible\r\n assert all(\r\n [isinstance(param, UncertainFunction) for param in params]\r\n ), 'All inputs to \"correlate\" must be of type \"UncertainFunction\"'\r\n\r\n # Put each ufunc's samples in a column-wise matrix\r\n data = np.vstack([param._mcpts for param in params]).T\r\n\r\n # Apply the correlation matrix to the sampled data\r\n new_data = induce_correlations(data, corrmat)\r\n\r\n # Re-set the samples to the respective variables\r\n for i in range(len(params)):\r\n params[i]._mcpts = new_data[:, i]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninduce a set of correlations on a column - wise dataset.", "response": "def induce_correlations(data, corrmat):\r\n \"\"\"\r\n Induce a set of correlations on a column-wise dataset\r\n \r\n Parameters\r\n ----------\r\n data : 2d-array\r\n An m-by-n array where m is the number of samples and n is the\r\n number of independent variables, each column of the array corresponding\r\n to each variable\r\n corrmat : 2d-array\r\n An n-by-n array that defines the desired correlation coefficients\r\n (between -1 and 1). Note: the matrix must be symmetric and\r\n positive-definite in order to induce.\r\n \r\n Returns\r\n -------\r\n new_data : 2d-array\r\n An m-by-n array that has the desired correlations.\r\n \r\n \"\"\"\r\n # Create an rank-matrix\r\n data_rank = np.vstack([rankdata(datai) for datai in data.T]).T\r\n\r\n # Generate van der Waerden scores\r\n data_rank_score = data_rank / (data_rank.shape[0] + 1.0)\r\n data_rank_score = norm(0, 1).ppf(data_rank_score)\r\n\r\n # Calculate the lower triangular matrix of the Cholesky decomposition\r\n # of the desired correlation matrix\r\n p = chol(corrmat)\r\n\r\n # Calculate the current correlations\r\n t = np.corrcoef(data_rank_score, rowvar=0)\r\n\r\n # Calculate the lower triangular matrix of the Cholesky decomposition\r\n # of the current correlation matrix\r\n q = chol(t)\r\n\r\n # Calculate the re-correlation matrix\r\n s = np.dot(p, np.linalg.inv(q))\r\n\r\n # Calculate the re-sampled matrix\r\n new_data = np.dot(data_rank_score, s.T)\r\n\r\n # Create the new rank matrix\r\n new_data_rank = np.vstack([rankdata(datai) for datai in new_data.T]).T\r\n\r\n # Sort the original data according to new_data_rank\r\n for i in range(data.shape[1]):\r\n vals, order = np.unique(\r\n np.hstack((data_rank[:, i], new_data_rank[:, i])), return_inverse=True\r\n )\r\n old_order = order[: new_data_rank.shape[0]]\r\n new_order = order[-new_data_rank.shape[0] :]\r\n tmp = data[np.argsort(old_order), i][new_order]\r\n data[:, i] = tmp[:]\r\n\r\n return data"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plotcorr(X, plotargs=None, full=True, labels=None):\r\n import matplotlib.pyplot as plt\r\n\r\n X = [Xi._mcpts if isinstance(Xi, UncertainFunction) else Xi for Xi in X]\r\n X = np.atleast_2d(X)\r\n numvars, numdata = X.shape\r\n fig, axes = plt.subplots(nrows=numvars, ncols=numvars, figsize=(8, 8))\r\n fig.subplots_adjust(hspace=0.0, wspace=0.0)\r\n\r\n for ax in axes.flat:\r\n # Hide all ticks and labels\r\n ax.xaxis.set_visible(False)\r\n ax.yaxis.set_visible(False)\r\n\r\n # Set up ticks only on one side for the \"edge\" subplots...\r\n if full:\r\n if ax.is_first_col():\r\n ax.yaxis.set_ticks_position(\"left\")\r\n if ax.is_last_col():\r\n ax.yaxis.set_ticks_position(\"right\")\r\n if ax.is_first_row():\r\n ax.xaxis.set_ticks_position(\"top\")\r\n if ax.is_last_row():\r\n ax.xaxis.set_ticks_position(\"bottom\")\r\n else:\r\n if ax.is_first_row():\r\n ax.xaxis.set_ticks_position(\"top\")\r\n if ax.is_last_col():\r\n ax.yaxis.set_ticks_position(\"right\")\r\n\r\n # Label the diagonal subplots...\r\n if not labels:\r\n labels = [\"x\" + str(i) for i in range(numvars)]\r\n\r\n for i, label in enumerate(labels):\r\n axes[i, i].annotate(\r\n label, (0.5, 0.5), xycoords=\"axes fraction\", ha=\"center\", va=\"center\"\r\n )\r\n\r\n # Plot the data\r\n for i, j in zip(*np.triu_indices_from(axes, k=1)):\r\n if full:\r\n idx = [(i, j), (j, i)]\r\n else:\r\n idx = [(i, j)]\r\n for x, y in idx:\r\n # FIX #1: this needed to be changed from ...(data[x], data[y],...)\r\n if plotargs is None:\r\n if len(X[x]) > 100:\r\n plotargs = \",b\" # pixel marker\r\n else:\r\n plotargs = \".b\" # point marker\r\n axes[x, y].plot(X[y], X[x], plotargs)\r\n ylim = min(X[y]), max(X[y])\r\n xlim = min(X[x]), max(X[x])\r\n axes[x, y].set_ylim(\r\n xlim[0] - (xlim[1] - xlim[0]) * 0.1, xlim[1] + (xlim[1] - xlim[0]) * 0.1\r\n )\r\n axes[x, y].set_xlim(\r\n ylim[0] - (ylim[1] - ylim[0]) * 0.1, ylim[1] + (ylim[1] - ylim[0]) * 0.1\r\n )\r\n\r\n # Turn on the proper x or y axes ticks.\r\n if full:\r\n for i, j in zip(list(range(numvars)), itertools.cycle((-1, 0))):\r\n axes[j, i].xaxis.set_visible(True)\r\n axes[i, j].yaxis.set_visible(True)\r\n else:\r\n for i in range(numvars - 1):\r\n axes[0, i + 1].xaxis.set_visible(True)\r\n axes[i, -1].yaxis.set_visible(True)\r\n for i in range(1, numvars):\r\n for j in range(0, i):\r\n fig.delaxes(axes[i, j])\r\n\r\n # FIX #2: if numvars is odd, the bottom right corner plot doesn't have the\r\n # correct axes limits, so we pull them from other axes\r\n if numvars % 2:\r\n xlimits = axes[0, -1].get_xlim()\r\n ylimits = axes[-1, 0].get_ylim()\r\n axes[-1, -1].set_xlim(xlimits)\r\n axes[-1, -1].set_ylim(ylimits)\r\n\r\n return fig", "response": "Returns a scatterplot matrix of the same size as the input array X."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef chol(A):\r\n A = np.array(A)\r\n assert A.shape[0] == A.shape[1], \"Input matrix must be square\"\r\n\r\n L = [[0.0] * len(A) for _ in range(len(A))]\r\n for i in range(len(A)):\r\n for j in range(i + 1):\r\n s = sum(L[i][k] * L[j][k] for k in range(j))\r\n L[i][j] = (\r\n (A[i][i] - s) ** 0.5 if (i == j) else (1.0 / L[j][j] * (A[i][j] - s))\r\n )\r\n\r\n return np.array(L)", "response": "Calculates the lower triangular matrix of the Cholesky decomposition of a symmetric positive - definite matrix."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get(self, uri, params={}):\n '''A generic method to make GET requests to the OpenDNS Investigate API\n on the given URI.\n '''\n return self._session.get(urljoin(Investigate.BASE_URL, uri),\n params=params, headers=self._auth_header, proxies=self.proxies\n )", "response": "A generic method to make GET requests to the OpenDNS Investigate API\n on the given URI."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef post(self, uri, params={}, data={}):\n '''A generic method to make POST requests to the OpenDNS Investigate API\n on the given URI.\n '''\n return self._session.post(\n urljoin(Investigate.BASE_URL, uri),\n params=params, data=data, headers=self._auth_header,\n proxies=self.proxies\n )", "response": "A generic method to make POST requests to the OpenDNS Investigate API on the given URI."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_parse(self, uri, params={}):\n '''Convenience method to call get() on an arbitrary URI and parse the response\n into a JSON object. Raises an error on non-200 response status.\n '''\n return self._request_parse(self.get, uri, params)", "response": "Convenience method to call get on an arbitrary URI and parse the response\n into a JSON object. Raises an error on non - 200 response status."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the domain status and categorization of a list of domains.", "response": "def categorization(self, domains, labels=False):\n '''Get the domain status and categorization of a domain or list of domains.\n 'domains' can be either a single domain, or a list of domains.\n Setting 'labels' to True will give back categorizations in human-readable\n form.\n\n For more detail, see https://investigate.umbrella.com/docs/api#categorization\n '''\n if type(domains) is str:\n return self._get_categorization(domains, labels)\n elif type(domains) is list:\n return self._post_categorization(domains, labels)\n else:\n raise Investigate.DOMAIN_ERR"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cooccurrences(self, domain):\n '''Get the cooccurrences of the given domain.\n\n For details, see https://investigate.umbrella.com/docs/api#co-occurrences\n '''\n uri = self._uris[\"cooccurrences\"].format(domain)\n return self.get_parse(uri)", "response": "Get the cooccurrences of the given domain."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the related domains of the given domain.", "response": "def related(self, domain):\n '''Get the related domains of the given domain.\n\n For details, see https://investigate.umbrella.com/docs/api#relatedDomains\n '''\n uri = self._uris[\"related\"].format(domain)\n return self.get_parse(uri)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the Security Information for the given domain.", "response": "def security(self, domain):\n '''Get the Security Information for the given domain.\n\n For details, see https://investigate.umbrella.com/docs/api#securityInfo\n '''\n uri = self._uris[\"security\"].format(domain)\n return self.get_parse(uri)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget the RR ( Resource Record ) History of the given domain or IP.", "response": "def rr_history(self, query, query_type=\"A\"):\n '''Get the RR (Resource Record) History of the given domain or IP.\n The default query type is for 'A' records, but the following query types\n are supported:\n\n A, NS, MX, TXT, CNAME\n\n For details, see https://investigate.umbrella.com/docs/api#dnsrr_domain\n '''\n if query_type not in Investigate.SUPPORTED_DNS_TYPES:\n raise Investigate.UNSUPPORTED_DNS_QUERY\n\n # if this is an IP address, query the IP\n if Investigate.IP_PATTERN.match(query):\n return self._ip_rr_history(query, query_type)\n\n # otherwise, query the domain\n return self._domain_rr_history(query, query_type)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef domain_whois(self, domain):\n '''Gets whois information for a domain'''\n uri = self._uris[\"whois_domain\"].format(domain)\n resp_json = self.get_parse(uri)\n return resp_json", "response": "Gets whois information for a domain"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget whois history for a domain", "response": "def domain_whois_history(self, domain, limit=None):\n '''Gets whois history for a domain'''\n\n params = dict()\n if limit is not None:\n params['limit'] = limit\n\n uri = self._uris[\"whois_domain_history\"].format(domain)\n resp_json = self.get_parse(uri, params)\n return resp_json"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ns_whois(self, nameservers, limit=DEFAULT_LIMIT, offset=DEFAULT_OFFSET, sort_field=DEFAULT_SORT):\n '''Gets the domains that have been registered with a nameserver or\n nameservers'''\n if not isinstance(nameservers, list):\n uri = self._uris[\"whois_ns\"].format(nameservers)\n params = {'limit': limit, 'offset': offset, 'sortField': sort_field}\n else:\n uri = self._uris[\"whois_ns\"].format('')\n params = {'emailList' : ','.join(nameservers), 'limit': limit, 'offset': offset, 'sortField': sort_field}\n\n resp_json = self.get_parse(uri, params=params)\n return resp_json", "response": "Gets the domains that have been registered with a nameserver or a nameserver list"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef search(self, pattern, start=None, limit=None, include_category=None):\n '''Searches for domains that match a given pattern'''\n\n params = dict()\n\n if start is None:\n start = datetime.timedelta(days=30)\n\n if isinstance(start, datetime.timedelta):\n params['start'] = int(time.mktime((datetime.datetime.utcnow() - start).timetuple()) * 1000)\n elif isinstance(start, datetime.datetime):\n params['start'] = int(time.mktime(start.timetuple()) * 1000)\n else:\n raise Investigate.SEARCH_ERR\n\n if limit is not None and isinstance(limit, int):\n params['limit'] = limit\n if include_category is not None and isinstance(include_category, bool):\n params['includeCategory'] = str(include_category).lower()\n\n uri = self._uris['search'].format(quote_plus(pattern))\n\n return self.get_parse(uri, params)", "response": "Searches for domains that match a given pattern"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns an object representing the samples identified by the input domain IP or URL", "response": "def samples(self, anystring, limit=None, offset=None, sortby=None):\n '''Return an object representing the samples identified by the input domain, IP, or URL'''\n\n uri = self._uris['samples'].format(anystring)\n params = {'limit': limit, 'offset': offset, 'sortby': sortby}\n\n return self.get_parse(uri, params)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sample(self, hash, limit=None, offset=None):\n '''Return an object representing the sample identified by the input hash, or an empty object if that sample is not found'''\n\n uri = self._uris['sample'].format(hash)\n params = {'limit': limit, 'offset': offset}\n\n return self.get_parse(uri, params)", "response": "Return an object representing the sample identified by the input hash or an empty object if that sample is not found"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef as_for_ip(self, ip):\n '''Gets the AS information for a given IP address.'''\n if not Investigate.IP_PATTERN.match(ip):\n raise Investigate.IP_ERR\n\n uri = self._uris[\"as_for_ip\"].format(ip)\n resp_json = self.get_parse(uri)\n\n return resp_json", "response": "Gets the AS information for a given IP address."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the AS information for a given ASN. Return the CIDR and geolocation associated with the AS.", "response": "def prefixes_for_asn(self, asn):\n '''Gets the AS information for a given ASN. Return the CIDR and geolocation associated with the AS.'''\n\n uri = self._uris[\"prefixes_for_asn\"].format(asn)\n resp_json = self.get_parse(uri)\n\n return resp_json"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the domain tagging timeline for a given uri.", "response": "def timeline(self, uri):\n '''Get the domain tagging timeline for a given uri. \n Could be a domain, ip, or url.\n For details, see https://docs.umbrella.com/investigate-api/docs/timeline\n '''\n uri = self._uris[\"timeline\"].format(uri)\n resp_json = self.get_parse(uri)\n\n return resp_json"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef abs(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.abs(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.abs(x)", "response": "Absolute value of a single object"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninversing cosine of a sequence of numbers.", "response": "def acos(x):\n \"\"\"\n Inverse cosine\n \"\"\"\n if isinstance(x, UncertainFunction):\n mcpts = np.arccos(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.arccos(x)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef asin(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.arcsin(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.arcsin(x)", "response": "Inverse sine of numpy. asin"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninverse tangent of a single object", "response": "def atan(x):\n \"\"\"\n Inverse tangent\n \"\"\"\n if isinstance(x, UncertainFunction):\n mcpts = np.arctan(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.arctan(x)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ceil(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.ceil(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.ceil(x)", "response": "ceil - Returns the ceil of x"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cos(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.cos(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.cos(x)", "response": "cos - like function"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef degrees(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.degrees(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.degrees(x)", "response": "Convert radians to degrees"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates exp - 1 of x", "response": "def expm1(x):\n \"\"\"\n Calculate exp(x) - 1\n \"\"\"\n if isinstance(x, UncertainFunction):\n mcpts = np.expm1(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.expm1(x)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef floor(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.floor(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.floor(x)", "response": "Floor function (round towards negative infinity)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef hypot(x, y):\n if isinstance(x, UncertainFunction) or isinstance(x, UncertainFunction):\n ufx = to_uncertain_func(x)\n ufy = to_uncertain_func(y)\n mcpts = np.hypot(ufx._mcpts, ufy._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.hypot(x, y)", "response": "Calculate the hypotenuse given two legs of a right triangle"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef log(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.log(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.log(x)", "response": "Natural logarithm of a single object"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nbase - 10 logarithm of x", "response": "def log10(x):\n \"\"\"\n Base-10 logarithm\n \"\"\"\n if isinstance(x, UncertainFunction):\n mcpts = np.log10(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.log10(x)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert degrees to radians", "response": "def radians(x):\n \"\"\"\n Convert degrees to radians\n \"\"\"\n if isinstance(x, UncertainFunction):\n mcpts = np.radians(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.radians(x)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sqrt(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.sqrt(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.sqrt(x)", "response": "sqrt - returns the square root of x"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef trunc(x):\n if isinstance(x, UncertainFunction):\n mcpts = np.trunc(x._mcpts)\n return UncertainFunction(mcpts)\n else:\n return np.trunc(x)", "response": "Truncates the values to the integer value without rounding"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef lhd(\n dist=None,\n size=None,\n dims=1,\n form=\"randomized\",\n iterations=100,\n showcorrelations=False,\n):\n \"\"\"\n Create a Latin-Hypercube sample design based on distributions defined in the\n `scipy.stats` module\n \n Parameters\n ----------\n dist: array_like\n frozen scipy.stats.rv_continuous or rv_discrete distribution objects \n that are defined previous to calling LHD\n\n size: int\n integer value for the number of samples to generate for each \n distribution object\n \n dims: int, optional\n if dist is a single distribution object, and dims > 1, the one\n distribution will be used to generate a size-by-dims sampled design\n \n form: str, optional (non-functional at the moment)\n determines how the sampling is to occur, with the following optional \n values:\n - 'randomized' - completely randomized sampling\n - 'spacefilling' - space-filling sampling (generally gives a more \n accurate sampling of the design when the number of sample points \n is small)\n - 'orthogonal' - balanced space-filling sampling (experimental)\n \n The 'spacefilling' and 'orthogonal' forms require some iterations to \n determine the optimal sampling pattern. \n \n iterations: int, optional (non-functional at the moment)\n used to control the number of allowable search iterations for generating\n 'spacefilling' and 'orthogonal' designs\n \n Returns\n -------\n out: 2d-array, \n A 2d-array where each column corresponds to each input distribution and \n each row is a sample in the design\n \n Examples\n --------\n \n Single distribution: \n - uniform distribution, low = -1, width = 2\n \n >>> import scipy.stats as ss\n >>> d0 = ss.uniform(loc=-1,scale=2)\n >>> print lhd(dist=d0,size=5)\n [[ 0.51031081]\n [-0.28961427]\n [-0.68342107]\n [ 0.69784371]\n [ 0.12248842]]\n\n Single distribution for multiple variables:\n - normal distribution, mean = 0, stdev = 1\n \n >>> d1 = ss.norm(loc=0,scale=1)\n >>> print lhd(dist=d1,size=7,dims=5)\n [[-0.8612785 0.23034412 0.21808001]\n [ 0.0455778 0.07001606 0.31586419]\n [-0.978553 0.30394663 0.78483995]\n [-0.26415983 0.15235896 0.51462024]\n [ 0.80805686 0.38891031 0.02076505]\n [ 1.63028931 0.52104917 1.48016008]] \n \n Multiple distributions:\n - beta distribution, alpha = 2, beta = 5\n - exponential distribution, lambda = 1.5\n \n >>> d2 = ss.beta(2,5)\n >>> d3 = ss.expon(scale=1/1.5)\n >>> print lhd(dist=(d1,d2,d3),size=6)\n [[-0.8612785 0.23034412 0.21808001]\n [ 0.0455778 0.07001606 0.31586419]\n [-0.978553 0.30394663 0.78483995]\n [-0.26415983 0.15235896 0.51462024]\n [ 0.80805686 0.38891031 0.02076505]\n [ 1.63028931 0.52104917 1.48016008]]\n \"\"\"\n assert dims > 0, 'kwarg \"dims\" must be at least 1'\n if not size or not dist:\n return None\n\n def _lhs(x, samples=20):\n \"\"\"\n _lhs(x) returns a latin-hypercube matrix (each row is a different\n set of sample inputs) using a default sample size of 20 for each column\n of X. X must be a 2xN matrix that contains the lower and upper bounds of\n each column. The lower bound(s) should be in the first row and the upper\n bound(s) should be in the second row.\n \n _lhs(x,samples=N) uses the sample size of N instead of the default (20).\n \n Example:\n >>> x = np.array([[0,-1,3],[1,2,6]])\n >>> print 'x:'; print x\n x:\n [[ 0 -1 3]\n [ 1 2 6]]\n\n >>> print 'lhs(x):'; print _lhs(x)\n lhs(x):\n [[ 0.02989122 -0.93918734 3.14432618]\n [ 0.08869833 -0.82140706 3.19875152]\n [ 0.10627442 -0.66999234 3.33814979]\n [ 0.15202861 -0.44157763 3.57036894]\n [ 0.2067089 -0.34845384 3.66930908]\n [ 0.26542056 -0.23706445 3.76361414]\n [ 0.34201421 -0.00779306 3.90818257]\n [ 0.37891646 0.15458423 4.15031708]\n [ 0.43501575 0.23561118 4.20320064]\n [ 0.4865449 0.36350601 4.45792314]\n [ 0.54804367 0.56069855 4.60911539]\n [ 0.59400712 0.7468415 4.69923486]\n [ 0.63708876 0.9159176 4.83611204]\n [ 0.68819855 0.98596354 4.97659182]\n [ 0.7368695 1.18923511 5.11135111]\n [ 0.78885724 1.28369441 5.2900157 ]\n [ 0.80966513 1.47415703 5.4081971 ]\n [ 0.86196731 1.57844205 5.61067689]\n [ 0.94784517 1.71823504 5.78021164]\n [ 0.96739728 1.94169017 5.88604772]]\n\n >>> print 'lhs(x,samples=5):'; print _lhs(x,samples=5)\n lhs(x,samples=5):\n [[ 0.1949127 -0.54124725 3.49238369]\n [ 0.21128576 -0.13439798 3.65652016]\n [ 0.47516308 0.39957406 4.5797308 ]\n [ 0.64400392 0.90890999 4.92379431]\n [ 0.96279472 1.79415307 5.52028238]] \n \"\"\"\n\n # determine the segment size\n segmentSize = 1.0 / samples\n\n # get the number of dimensions to sample (number of columns)\n numVars = x.shape[1]\n\n # populate each dimension\n out = np.zeros((samples, numVars))\n pointValue = np.zeros(samples)\n\n for n in range(numVars):\n for i in range(samples):\n segmentMin = i * segmentSize\n point = segmentMin + (np.random.random() * segmentSize)\n pointValue[i] = (point * (x[1, n] - x[0, n])) + x[0, n]\n out[:, n] = pointValue\n\n # now randomly arrange the different segments\n return _mix(out)\n\n def _mix(data, dim=\"cols\"):\n \"\"\"\n Takes a data matrix and mixes up the values along dim (either \"rows\" or \n \"cols\"). In other words, if dim='rows', then each row's data is mixed\n ONLY WITHIN ITSELF. Likewise, if dim='cols', then each column's data is\n mixed ONLY WITHIN ITSELF.\n \"\"\"\n data = np.atleast_2d(data)\n n = data.shape[0]\n\n if dim == \"rows\":\n data = data.T\n\n data_rank = list(range(n))\n for i in range(data.shape[1]):\n new_data_rank = np.random.permutation(data_rank)\n vals, order = np.unique(\n np.hstack((data_rank, new_data_rank)), return_inverse=True\n )\n old_order = order[:n]\n new_order = order[-n:]\n tmp = data[np.argsort(old_order), i][new_order]\n data[:, i] = tmp[:]\n\n if dim == \"rows\":\n data = data.T\n\n return data\n\n if form is \"randomized\":\n if hasattr(dist, \"__getitem__\"): # if multiple distributions were input\n nvars = len(dist)\n x = np.vstack((np.zeros(nvars), np.ones(nvars)))\n unif_data = _lhs(x, samples=size)\n dist_data = np.empty_like(unif_data)\n for i, d in enumerate(dist):\n dist_data[:, i] = d.ppf(unif_data[:, i])\n\n else: # if a single distribution was input\n nvars = dims\n x = np.vstack((np.zeros(nvars), np.ones(nvars)))\n unif_data = _lhs(x, samples=size)\n dist_data = np.empty_like(unif_data)\n for i in range(nvars):\n dist_data[:, i] = dist.ppf(unif_data[:, i])\n\n elif form is \"spacefilling\":\n\n def euclid_distance(arr):\n n = arr.shape[0]\n ans = 0.0\n for i in range(n - 1):\n for j in range(i + 1, n):\n d = np.sqrt(\n np.sum(\n [(arr[i, k] - arr[j, k]) ** 2 for k in range(arr.shape[1])]\n )\n )\n ans += 1.0 / d ** 2\n return ans\n\n def fill_space(data):\n best = 1e8\n for it in range(iterations):\n d = euclid_distance(data)\n if d < best:\n d_opt = d\n data_opt = data.copy()\n\n data = _mix(data)\n\n print(\"Optimized Distance:\", d_opt)\n return data_opt\n\n if hasattr(dist, \"__getitem__\"): # if multiple distributions were input\n nvars = len(dist)\n x = np.vstack((np.zeros(nvars), np.ones(nvars)))\n unif_data = fill_space(_lhs(x, samples=size))\n dist_data = np.empty_like(unif_data)\n for i, d in enumerate(dist):\n dist_data[:, i] = d.ppf(unif_data[:, i])\n\n else: # if a single distribution was input\n nvars = dims\n x = np.vstack((np.zeros(nvars), np.ones(nvars)))\n unif_data = fill_space(_lhs(x, samples=size))\n dist_data = np.empty_like(unif_data)\n for i in range(nvars):\n dist_data[:, i] = dist.ppf(unif_data[:, i])\n\n elif form is \"orthogonal\":\n raise NotImplementedError(\n \"Sorry. The orthogonal space-filling algorithm hasn't been implemented yet.\"\n )\n else:\n raise ValueError('Invalid \"form\" value: %s' % (form))\n\n if dist_data.shape[1] > 1:\n cor_matrix = np.zeros((nvars, nvars))\n for i in range(nvars):\n for j in range(nvars):\n x_data = dist_data[:, i].copy()\n y_data = dist_data[:, j].copy()\n x_mean = x_data.mean()\n y_mean = y_data.mean()\n num = np.sum((x_data - x_mean) * (y_data - y_mean))\n den = np.sqrt(\n np.sum((x_data - x_mean) ** 2) * np.sum((y_data - y_mean) ** 2)\n )\n cor_matrix[i, j] = num / den\n cor_matrix[j, i] = num / den\n inv_cor_matrix = np.linalg.pinv(cor_matrix)\n VIF = np.max(np.diag(inv_cor_matrix))\n\n if showcorrelations:\n print(\"Correlation Matrix:\\n\", cor_matrix)\n print(\"Inverted Correlation Matrix:\\n\", inv_cor_matrix)\n print(\"Variance Inflation Factor (VIF):\", VIF)\n\n return dist_data", "response": "Returns a new Latin - Hypercube sample design for the objects in the specified object."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_uncertain_func(x):\n if isinstance(x, UncertainFunction):\n return x\n\n # ! In Python 2.6+, numbers.Number could be used instead, here:\n elif isinstance(x, CONSTANT_TYPES):\n # No variable => no derivative to define:\n return UncertainFunction([x] * npts)\n\n raise NotUpcast(\"%s cannot be converted to a number with\" \" uncertainty\" % type(x))", "response": "Converts x into an UncertainFunction - compatible object."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a random variate of the beta distribution.", "response": "def Beta(alpha, beta, low=0, high=1, tag=None):\n \"\"\"\n A Beta random variate\n \n Parameters\n ----------\n alpha : scalar\n The first shape parameter\n beta : scalar\n The second shape parameter\n \n Optional\n --------\n low : scalar\n Lower bound of the distribution support (default=0)\n high : scalar\n Upper bound of the distribution support (default=1)\n \"\"\"\n assert (\n alpha > 0 and beta > 0\n ), 'Beta \"alpha\" and \"beta\" parameters must be greater than zero'\n assert low < high, 'Beta \"low\" must be less than \"high\"'\n return uv(ss.beta(alpha, beta, loc=low, scale=high - low), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a random variate of the given size with the Bradford distribution.", "response": "def Bradford(q, low=0, high=1, tag=None):\n \"\"\"\n A Bradford random variate\n \n Parameters\n ----------\n q : scalar\n The shape parameter\n low : scalar\n The lower bound of the distribution (default=0)\n high : scalar\n The upper bound of the distribution (default=1)\n \"\"\"\n assert q > 0, 'Bradford \"q\" parameter must be greater than zero'\n assert low < high, 'Bradford \"low\" parameter must be less than \"high\"'\n return uv(ss.bradford(q, loc=low, scale=high - low), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a random variate of the Chi - Squared distribution.", "response": "def ChiSquared(k, tag=None):\n \"\"\"\n A Chi-Squared random variate\n \n Parameters\n ----------\n k : int\n The degrees of freedom of the distribution (must be greater than one)\n \"\"\"\n assert int(k) == k and k >= 1, 'Chi-Squared \"k\" must be an integer greater than 0'\n return uv(ss.chi2(k), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a new random variate with the specified size and scale.", "response": "def Erlang(k, lamda, tag=None):\n \"\"\"\n An Erlang random variate.\n \n This distribution is the same as a Gamma(k, theta) distribution, but \n with the restriction that k must be a positive integer. This\n is provided for greater compatibility with other simulation tools, but\n provides no advantage over the Gamma distribution in its applications.\n \n Parameters\n ----------\n k : int\n The shape parameter (must be a positive integer)\n lamda : scalar\n The scale parameter (must be greater than zero)\n \"\"\"\n assert int(k) == k and k > 0, 'Erlang \"k\" must be a positive integer'\n assert lamda > 0, 'Erlang \"lamda\" must be greater than zero'\n return Gamma(k, lamda, tag)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Exponential(lamda, tag=None):\n assert lamda > 0, 'Exponential \"lamda\" must be greater than zero'\n return uv(ss.expon(scale=1.0 / lamda), tag=tag)", "response": "Returns a random variate with exponential distribution."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the Extreme Value maximum random variate.", "response": "def ExtValueMax(mu, sigma, tag=None):\n \"\"\"\n An Extreme Value Maximum random variate.\n \n Parameters\n ----------\n mu : scalar\n The location parameter\n sigma : scalar\n The scale parameter (must be greater than zero)\n \"\"\"\n assert sigma > 0, 'ExtremeValueMax \"sigma\" must be greater than zero'\n p = U(0, 1)._mcpts[:]\n return UncertainFunction(mu - sigma * np.log(-np.log(p)), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a random variate of the Fisher factor.", "response": "def Fisher(d1, d2, tag=None):\n \"\"\"\n An F (fisher) random variate\n \n Parameters\n ----------\n d1 : int\n Numerator degrees of freedom\n d2 : int\n Denominator degrees of freedom\n \"\"\"\n assert (\n int(d1) == d1 and d1 >= 1\n ), 'Fisher (F) \"d1\" must be an integer greater than 0'\n assert (\n int(d2) == d2 and d2 >= 1\n ), 'Fisher (F) \"d2\" must be an integer greater than 0'\n return uv(ss.f(d1, d2), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a random variate of the Gamma distribution.", "response": "def Gamma(k, theta, tag=None):\n \"\"\"\n A Gamma random variate\n \n Parameters\n ----------\n k : scalar\n The shape parameter (must be positive and non-zero)\n theta : scalar\n The scale parameter (must be positive and non-zero)\n \"\"\"\n assert (\n k > 0 and theta > 0\n ), 'Gamma \"k\" and \"theta\" parameters must be greater than zero'\n return uv(ss.gamma(k, scale=theta), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef LogNormal(mu, sigma, tag=None):\n assert sigma > 0, 'Log-Normal \"sigma\" must be positive'\n return uv(ss.lognorm(sigma, loc=mu), tag=tag)", "response": "Returns a random variate with lognormal distribution."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a random variate with a normal distribution.", "response": "def Normal(mu, sigma, tag=None):\n \"\"\"\n A Normal (or Gaussian) random variate\n \n Parameters\n ----------\n mu : scalar\n The mean value of the distribution\n sigma : scalar\n The standard deviation (must be positive and non-zero)\n \"\"\"\n assert sigma > 0, 'Normal \"sigma\" must be greater than zero'\n return uv(ss.norm(loc=mu, scale=sigma), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Pareto(q, a, tag=None):\n assert q > 0 and a > 0, 'Pareto \"q\" and \"a\" must be positive scalars'\n p = Uniform(0, 1, tag)\n return a * (1 - p) ** (-1.0 / q)", "response": "Returns a random variate of the given size q and a."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a Pareto random variate.", "response": "def Pareto2(q, b, tag=None):\n \"\"\"\n A Pareto random variate (second kind). This form always starts at the\n origin.\n \n Parameters\n ----------\n q : scalar\n The scale parameter\n b : scalar\n The shape parameter\n \"\"\"\n assert q > 0 and b > 0, 'Pareto2 \"q\" and \"b\" must be positive scalars'\n return Pareto(q, b, tag) - b"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a random variate of the given degree of freedom.", "response": "def StudentT(v, tag=None):\n \"\"\"\n A Student-T random variate\n \n Parameters\n ----------\n v : int\n The degrees of freedom of the distribution (must be greater than one)\n \"\"\"\n assert int(v) == v and v >= 1, 'Student-T \"v\" must be an integer greater than 0'\n return uv(ss.t(v), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Triangular(low, peak, high, tag=None):\n assert low <= peak <= high, 'Triangular \"peak\" must lie between \"low\" and \"high\"'\n low, peak, high = [float(x) for x in [low, peak, high]]\n return uv(\n ss.triang((1.0 * peak - low) / (high - low), loc=low, scale=(high - low)),\n tag=tag,\n )", "response": "Returns a random variate of the given lower and upper bound of the triangle."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a random variate with uniform values.", "response": "def Uniform(low, high, tag=None):\n \"\"\"\n A Uniform random variate\n \n Parameters\n ----------\n low : scalar\n Lower bound of the distribution support.\n high : scalar\n Upper bound of the distribution support.\n \"\"\"\n assert low < high, 'Uniform \"low\" must be less than \"high\"'\n return uv(ss.uniform(loc=low, scale=high - low), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a random weibull random variate.", "response": "def Weibull(lamda, k, tag=None):\n \"\"\"\n A Weibull random variate\n \n Parameters\n ----------\n lamda : scalar\n The scale parameter\n k : scalar\n The shape parameter\n \"\"\"\n assert (\n lamda > 0 and k > 0\n ), 'Weibull \"lamda\" and \"k\" parameters must be greater than zero'\n return uv(ss.exponweib(lamda, k), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Bernoulli(p, tag=None):\n assert (\n 0 < p < 1\n ), 'Bernoulli probability \"p\" must be between zero and one, non-inclusive'\n return uv(ss.bernoulli(p), tag=tag)", "response": "A Bernoulli random variate from a random variate."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Binomial(n, p, tag=None):\n assert (\n int(n) == n and n > 0\n ), 'Binomial number of trials \"n\" must be an integer greater than zero'\n assert (\n 0 < p < 1\n ), 'Binomial probability \"p\" must be between zero and one, non-inclusive'\n return uv(ss.binom(n, p), tag=tag)", "response": "Returns a random variate of the given number of trials and probability p."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the geometric random variate of the given probability.", "response": "def Geometric(p, tag=None):\n \"\"\"\n A Geometric random variate\n \n Parameters\n ----------\n p : scalar\n The probability of success\n \"\"\"\n assert (\n 0 < p < 1\n ), 'Geometric probability \"p\" must be between zero and one, non-inclusive'\n return uv(ss.geom(p), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Hypergeometric(N, n, K, tag=None):\n assert (\n int(N) == N and N > 0\n ), 'Hypergeometric total population size \"N\" must be an integer greater than zero.'\n assert (\n int(n) == n and 0 < n <= N\n ), 'Hypergeometric interest population size \"n\" must be an integer greater than zero and no more than the total population size.'\n assert (\n int(K) == K and 0 < K <= N\n ), 'Hypergeometric chosen population size \"K\" must be an integer greater than zero and no more than the total population size.'\n return uv(ss.hypergeom(N, n, K), tag=tag)", "response": "Returns a Hypergeometric random variate from the given population size n and K."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a random variate with a Poisson distribution.", "response": "def Poisson(lamda, tag=None):\n \"\"\"\n A Poisson random variate\n \n Parameters\n ----------\n lamda : scalar\n The rate of an occurance within a specified interval of time or space.\n \"\"\"\n assert lamda > 0, 'Poisson \"lamda\" must be greater than zero.'\n return uv(ss.poisson(lamda), tag=tag)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the covariance matrix of uncertain variables.", "response": "def covariance_matrix(nums_with_uncert):\n \"\"\"\n Calculate the covariance matrix of uncertain variables, oriented by the\n order of the inputs\n \n Parameters\n ----------\n nums_with_uncert : array-like\n A list of variables that have an associated uncertainty\n \n Returns\n -------\n cov_matrix : 2d-array-like\n A nested list containing covariance values\n \n Example\n -------\n \n >>> x = N(1, 0.1)\n >>> y = N(10, 0.1)\n >>> z = x + 2*y\n >>> covariance_matrix([x,y,z])\n [[ 9.99694861e-03 2.54000840e-05 1.00477488e-02]\n [ 2.54000840e-05 9.99823207e-03 2.00218642e-02]\n [ 1.00477488e-02 2.00218642e-02 5.00914772e-02]]\n\n \"\"\"\n ufuncs = list(map(to_uncertain_func, nums_with_uncert))\n cov_matrix = []\n for (i1, expr1) in enumerate(ufuncs):\n coefs_expr1 = []\n mean1 = expr1.mean\n for (i2, expr2) in enumerate(ufuncs[: i1 + 1]):\n mean2 = expr2.mean\n coef = np.mean((expr1._mcpts - mean1) * (expr2._mcpts - mean2))\n coefs_expr1.append(coef)\n cov_matrix.append(coefs_expr1)\n\n # We symmetrize the matrix:\n for (i, covariance_coefs) in enumerate(cov_matrix):\n covariance_coefs.extend(cov_matrix[j][i] for j in range(i + 1, len(cov_matrix)))\n\n return cov_matrix"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating the correlation matrix of uncertain variables oriented by the order of the inputs.", "response": "def correlation_matrix(nums_with_uncert):\n \"\"\"\n Calculate the correlation matrix of uncertain variables, oriented by the\n order of the inputs\n \n Parameters\n ----------\n nums_with_uncert : array-like\n A list of variables that have an associated uncertainty\n \n Returns\n -------\n corr_matrix : 2d-array-like\n A nested list containing covariance values\n \n Example\n -------\n \n >>> x = N(1, 0.1)\n >>> y = N(10, 0.1)\n >>> z = x + 2*y\n >>> correlation_matrix([x,y,z])\n [[ 0.99969486 0.00254001 0.4489385 ]\n [ 0.00254001 0.99982321 0.89458702]\n [ 0.4489385 0.89458702 1. ]]\n\n \"\"\"\n ufuncs = list(map(to_uncertain_func, nums_with_uncert))\n data = np.vstack([ufunc._mcpts for ufunc in ufuncs])\n return np.corrcoef(data.T, rowvar=0)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef var(self):\n mn = self.mean\n vr = np.mean((self._mcpts - mn) ** 2)\n return vr", "response": "Returns the variance value of the current object"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef kurt(self):\n mn = self.mean\n sd = self.std\n kt = 0.0 if abs(sd) <= 1e-8 else np.mean((self._mcpts - mn) ** 4) / sd ** 4\n return kt", "response": "Returns the Kurtosis coefficient value as a result of an uncertainty calculation"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a list of the first four standard moments of a distribution.", "response": "def stats(self):\n \"\"\"\n The first four standard moments of a distribution: mean, variance, and\n standardized skewness and kurtosis coefficients.\n \"\"\"\n mn = self.mean\n vr = self.var\n sk = self.skew\n kt = self.kurt\n return [mn, vr, sk, kt]"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the actual distribution value at a given percentile or set of percentiles.", "response": "def percentile(self, val):\n \"\"\"\n Get the distribution value at a given percentile or set of percentiles.\n This follows the NIST method for calculating percentiles.\n \n Parameters\n ----------\n val : scalar or array\n Either a single value or an array of values between 0 and 1.\n \n Returns\n -------\n out : scalar or array\n The actual distribution value that appears at the requested\n percentile value or values\n \n \"\"\"\n try:\n # test to see if an input is given as an array\n out = [self.percentile(vi) for vi in val]\n except (ValueError, TypeError):\n if val <= 0:\n out = float(min(self._mcpts))\n elif val >= 1:\n out = float(max(self._mcpts))\n else:\n tmp = np.sort(self._mcpts)\n n = val * (len(tmp) + 1)\n k, d = int(n), n - int(n)\n out = float(tmp[k] + d * (tmp[k + 1] - tmp[k]))\n if isinstance(val, np.ndarray):\n out = np.array(out)\n return out"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nprints the description of the object.", "response": "def describe(self, name=None):\n \"\"\"\n Cleanly show what the four displayed distribution moments are:\n - Mean\n - Variance\n - Standardized Skewness Coefficient\n - Standardized Kurtosis Coefficient\n \n For a standard Normal distribution, these are [0, 1, 0, 3].\n \n If the object has an associated tag, this is presented. If the optional\n ``name`` kwarg is utilized, this is presented as with the moments.\n Otherwise, no unique name is presented.\n \n Example\n =======\n ::\n \n >>> x = N(0, 1, 'x')\n >>> x.describe() # print tag since assigned\n MCERP Uncertain Value (x):\n ...\n\n >>> x.describe('foobar') # 'name' kwarg takes precedence\n MCERP Uncertain Value (foobar):\n ...\n \n >>> y = x**2\n >>> y.describe('y') # print name since assigned\n MCERP Uncertain Value (y):\n ...\n\n >>> y.describe() # print nothing since no tag\n MCERP Uncertain Value:\n ...\n\n \"\"\"\n mn, vr, sk, kt = self.stats\n if name is not None:\n s = \"MCERP Uncertain Value (\" + name + \"):\\n\"\n elif self.tag is not None:\n s = \"MCERP Uncertain Value (\" + self.tag + \"):\\n\"\n else:\n s = \"MCERP Uncertain Value:\\n\"\n s += \" > Mean................... {: }\\n\".format(mn)\n s += \" > Variance............... {: }\\n\".format(vr)\n s += \" > Skewness Coefficient... {: }\\n\".format(sk)\n s += \" > Kurtosis Coefficient... {: }\\n\".format(kt)\n print(s)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nplotting the distribution of the UncertainFunction. By default the plot is displayed with a density estimate estimate.", "response": "def plot(self, hist=False, show=False, **kwargs):\n \"\"\"\n Plot the distribution of the UncertainFunction. By default, the\n distribution is shown with a kernel density estimate (kde).\n \n Optional\n --------\n hist : bool\n If true, a density histogram is displayed (histtype='stepfilled')\n show : bool\n If ``True``, the figure will be displayed after plotting the \n distribution. If ``False``, an explicit call to ``plt.show()`` is\n required to display the figure.\n kwargs : any valid matplotlib.pyplot.plot or .hist kwarg\n \n \"\"\"\n import matplotlib.pyplot as plt\n\n vals = self._mcpts\n low = min(vals)\n high = max(vals)\n\n p = ss.kde.gaussian_kde(vals)\n xp = np.linspace(low, high, 100)\n\n if hist:\n h = plt.hist(\n vals,\n bins=int(np.sqrt(len(vals)) + 0.5),\n histtype=\"stepfilled\",\n normed=True,\n **kwargs\n )\n plt.ylim(0, 1.1 * h[0].max())\n else:\n plt.plot(xp, p.evaluate(xp), **kwargs)\n\n plt.xlim(low - (high - low) * 0.1, high + (high - low) * 0.1)\n\n if show:\n self.show()"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef plot(self, hist=False, show=False, **kwargs):\n import matplotlib.pyplot as plt\n\n if hist:\n vals = self._mcpts\n low = vals.min()\n high = vals.max()\n h = plt.hist(\n vals,\n bins=int(np.sqrt(len(vals)) + 0.5),\n histtype=\"stepfilled\",\n normed=True,\n **kwargs\n )\n plt.ylim(0, 1.1 * h[0].max())\n else:\n bound = 0.0001\n low = self.rv.ppf(bound)\n high = self.rv.ppf(1 - bound)\n if hasattr(self.rv.dist, \"pmf\"):\n low = int(low)\n high = int(high)\n vals = list(range(low, high + 1))\n plt.plot(vals, self.rv.pmf(vals), \"o\", **kwargs)\n else:\n vals = np.linspace(low, high, 500)\n plt.plot(vals, self.rv.pdf(vals), **kwargs)\n plt.xlim(low - (high - low) * 0.1, high + (high - low) * 0.1)\n\n if show:\n self.show()", "response": "Plots the distribution of the UncertainVariable."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads the hat from a picture at path.", "response": "def load_hat(self, path): # pylint: disable=no-self-use\n \"\"\"Loads the hat from a picture at path.\n\n Args:\n path: The path to load from\n\n Returns:\n The hat data.\n \"\"\"\n hat = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n if hat is None:\n raise ValueError('No hat image found at `{}`'.format(path))\n b, g, r, a = cv2.split(hat)\n return cv2.merge((r, g, b, a))"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nuses a haarcascade to detect faces inside an image.", "response": "def find_faces(self, image, draw_box=False):\n \"\"\"Uses a haarcascade to detect faces inside an image.\n\n Args:\n image: The image.\n draw_box: If True, the image will be marked with a rectangle.\n\n Return:\n The faces as returned by OpenCV's detectMultiScale method for\n cascades.\n \"\"\"\n frame_gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n faces = self.cascade.detectMultiScale(\n frame_gray,\n scaleFactor=1.3,\n minNeighbors=5,\n minSize=(50, 50),\n flags=0)\n\n if draw_box:\n for x, y, w, h in faces:\n cv2.rectangle(image, (x, y),\n (x + w, y + h), (0, 255, 0), 2)\n return faces"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfind instances of rsrc_type that match the filter in kwargs", "response": "def find_resources(self, rsrc_type, sort=None, yield_pages=False, **kwargs):\n \"\"\"Find instances of `rsrc_type` that match the filter in `**kwargs`\"\"\"\n return rsrc_type.find(self, sort=sort, yield_pages=yield_pages, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef changed(self, message=None, *args):\n if message is not None:\n self.logger.debug('%s: %s', self._repr(), message % args)\n self.logger.debug('%s: changed', self._repr())\n if self.parent is not None:\n self.parent.changed()\n elif isinstance(self, Mutable):\n super(TrackedObject, self).changed()", "response": "Marks the object as changed."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef register(cls, origin_type):\n def decorator(tracked_type):\n \"\"\"Adds the decorated class to the `_type_mapping` dictionary.\"\"\"\n cls._type_mapping[origin_type] = tracked_type\n return tracked_type\n return decorator", "response": "Decorator for mutation tracker registration."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef convert(cls, obj, parent):\n replacement_type = cls._type_mapping.get(type(obj))\n if replacement_type is not None:\n new = replacement_type(obj)\n new.parent = parent\n return new\n return obj", "response": "Converts objects to registered tracked types."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef convert_items(self, items):\n return ((key, self.convert(value, self)) for key, value in items)", "response": "Generator like convert_iterable but for 2 - tuple iterators."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef convert_mapping(self, mapping):\n if isinstance(mapping, dict):\n return self.convert_items(iteritems(mapping))\n return self.convert_items(mapping)", "response": "Convenience method to track either a dict or a 2 - tuple iterator."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nredirect to the changelist view if we only have one preference object redirect to it otherwise display listing.", "response": "def changelist_view(self, request, extra_context=None):\n \"\"\"\n If we only have a single preference object redirect to it,\n otherwise display listing.\n \"\"\"\n model = self.model\n if model.objects.all().count() > 1:\n return super(PreferencesAdmin, self).changelist_view(request)\n else:\n obj = model.singleton.get()\n return redirect(\n reverse(\n 'admin:%s_%s_change' % (\n model._meta.app_label, model._meta.model_name\n ),\n args=(obj.id,)\n )\n )"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef aslist(generator):\n 'Function decorator to transform a generator into a list'\n def wrapper(*args, **kwargs):\n return list(generator(*args, **kwargs))\n return wrapper", "response": "Function decorator to transform a generator into a list"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting a package release from a Python package.", "response": "def get_package_release_from_pypi(pkg_name, version, pypi_json_api_url, allowed_classifiers):\n \"\"\"\n No classifier-based selection of Python packages is currently implemented: for now we don't fetch any .whl or .egg\n Eventually, we should select the best release available, based on the classifier & PEP 425: https://www.python.org/dev/peps/pep-0425/\n E.g. a wheel when available but NOT for tornado 4.3 for example, where available wheels are only for Windows.\n Note also that some packages don't have .whl distributed, e.g. https://bugs.launchpad.net/lxml/+bug/1176147\n \"\"\"\n matching_releases = get_package_releases_matching_version(pkg_name, version, pypi_json_api_url)\n src_releases = [release for release in matching_releases if release['python_version'] == 'source']\n if src_releases:\n return select_src_release(src_releases, pkg_name, target_classifiers=('py2.py3-none-any',), select_arbitrary_version_if_none_match=True)\n if allowed_classifiers:\n return select_src_release(matching_releases, pkg_name, target_classifiers=allowed_classifiers)\n raise PypiQueryError('No source supported found for package {} version {}'.format(pkg_name, version))"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nextract a PEP425 - compliant classifier and extension from a file.", "response": "def extract_classifier_and_extension(pkg_name, filename):\n \"\"\"\n Returns a PEP425-compliant classifier (or 'py2.py3-none-any' if it cannot be extracted),\n and the file extension\n TODO: return a classifier 3-members namedtuple instead of a single string\n \"\"\"\n basename, _, extension = filename.rpartition('.')\n if extension == 'gz' and filename.endswith('.tar.gz'):\n extension = 'tar.gz'\n basename = filename[:-7]\n if basename == pkg_name or basename[len(pkg_name)] != '-':\n return 'py2.py3-none-any', extension\n basename = basename[len(pkg_name)+1:]\n classifier_parts = basename.split('-')\n if len(classifier_parts) < 3:\n return 'py2.py3-none-any', extension\n if len(classifier_parts) == 3:\n _, _, classifier_parts[0] = classifier_parts[0].rpartition('.')\n return '-'.join(classifier_parts[-3:]), extension"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef coerce(cls, key, value):\n if value is None:\n return value\n if isinstance(value, cls):\n return value\n if isinstance(value, dict):\n return NestedMutableDict.coerce(key, value)\n if isinstance(value, list):\n return NestedMutableList.coerce(key, value)\n return super(cls).coerce(key, value)", "response": "Convert plain dictionary to NestedMutable."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef is_mod_function(mod, fun):\n return inspect.isfunction(fun) and inspect.getmodule(fun) == mod", "response": "Checks if a function in a module was declared in that module."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef is_mod_class(mod, cls):\n return inspect.isclass(cls) and inspect.getmodule(cls) == mod", "response": "Checks if a class in a module was declared in that module."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlists all functions declared in a module.", "response": "def list_functions(mod_name):\n \"\"\"Lists all functions declared in a module.\n\n http://stackoverflow.com/a/1107150/3004221\n\n Args:\n mod_name: the module name\n Returns:\n A list of functions declared in that module.\n \"\"\"\n mod = sys.modules[mod_name]\n return [func.__name__ for func in mod.__dict__.values()\n if is_mod_function(mod, func)]"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef list_classes(mod_name):\n mod = sys.modules[mod_name]\n return [cls.__name__ for cls in mod.__dict__.values()\n if is_mod_class(mod, cls)]", "response": "Lists all classes declared in a module."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_linenumbers(functions, module, searchstr='def {}(image):\\n'):\n lines = inspect.getsourcelines(module)[0]\n line_numbers = {}\n for function in functions:\n try:\n line_numbers[function] = lines.index(\n searchstr.format(function)) + 1\n except ValueError:\n print(r'Can not find `{}`'.format(searchstr.format(function)))\n line_numbers[function] = 0\n return line_numbers", "response": "Returns a dictionary which maps function names to line numbers."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nformatting the documentation in a nicer way and for notebook cells.", "response": "def format_doc(fun):\n \"\"\"Formats the documentation in a nicer way and for notebook cells.\"\"\"\n SEPARATOR = '============================='\n func = cvloop.functions.__dict__[fun]\n\n doc_lines = ['{}'.format(l).strip() for l in func.__doc__.split('\\n')]\n if hasattr(func, '__init__'):\n doc_lines.append(SEPARATOR)\n doc_lines += ['{}'.format(l).strip() for l in\n func.__init__.__doc__.split('\\n')]\n\n mod_lines = []\n argblock = False\n returnblock = False\n for line in doc_lines:\n if line == SEPARATOR:\n mod_lines.append('\\n#### `{}.__init__(...)`:\\n\\n'.format(fun))\n elif 'Args:' in line:\n argblock = True\n if GENERATE_ARGS:\n mod_lines.append('**{}**\\n'.format(line))\n elif 'Returns:' in line:\n returnblock = True\n mod_lines.append('\\n**{}**'.format(line))\n elif not argblock and not returnblock:\n mod_lines.append('{}\\n'.format(line))\n elif argblock and not returnblock and ':' in line:\n if GENERATE_ARGS:\n mod_lines.append('- *{}:* {}\\n'.format(\n *line.split(':')))\n elif returnblock:\n mod_lines.append(line)\n else:\n mod_lines.append('{}\\n'.format(line))\n return mod_lines"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef prepare_axes(axes, title, size, cmap=None):\n if axes is None:\n return None\n\n # prepare axis itself\n axes.set_xlim([0, size[1]])\n axes.set_ylim([size[0], 0])\n axes.set_aspect('equal')\n\n axes.axis('off')\n if isinstance(cmap, str):\n title = '{} (cmap: {})'.format(title, cmap)\n axes.set_title(title)\n\n # prepare image data\n axes_image = image.AxesImage(axes, cmap=cmap,\n extent=(0, size[1], size[0], 0))\n axes_image.set_data(np.random.random((size[0], size[1], 3)))\n\n axes.add_image(axes_image)\n return axes_image", "response": "Prepares an axes object for clean plotting."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef connect_event_handlers(self):\n self.figure.canvas.mpl_connect('close_event', self.evt_release)\n self.figure.canvas.mpl_connect('pause_event', self.evt_toggle_pause)", "response": "Connects event handlers to the figure."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef evt_toggle_pause(self, *args): # pylint: disable=unused-argument\n if self.event_source._timer is None: # noqa: e501 pylint: disable=protected-access\n self.event_source.start()\n else:\n self.event_source.stop()", "response": "Pauses and resumes the video source."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nprint information about the unprocessed image.", "response": "def print_info(self, capture):\n \"\"\"Prints information about the unprocessed image.\n\n Reads one frame from the source to determine image colors, dimensions\n and data types.\n\n Args:\n capture: the source to read from.\n \"\"\"\n self.frame_offset += 1\n ret, frame = capture.read()\n if ret:\n print('Capture Information')\n print('\\tDimensions (HxW): {}x{}'.format(*frame.shape[0:2]))\n print('\\tColor channels: {}'.format(frame.shape[2] if\n len(frame.shape) > 2 else 1))\n print('\\tColor range: {}-{}'.format(np.min(frame),\n np.max(frame)))\n print('\\tdtype: {}'.format(frame.dtype))\n else:\n print('No source found.')"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndetermine the height and width of the image source.", "response": "def determine_size(self, capture):\n \"\"\"Determines the height and width of the image source.\n\n If no dimensions are available, this method defaults to a resolution of\n 640x480, thus returns (480, 640).\n If capture has a get method it is assumed to understand\n `cv2.CAP_PROP_FRAME_WIDTH` and `cv2.CAP_PROP_FRAME_HEIGHT` to get the\n information. Otherwise it reads one frame from the source to determine\n image dimensions.\n\n Args:\n capture: the source to read from.\n\n Returns:\n A tuple containing integers of height and width (simple casts).\n \"\"\"\n width = 640\n height = 480\n if capture and hasattr(capture, 'get'):\n width = capture.get(cv2.CAP_PROP_FRAME_WIDTH)\n height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT)\n else:\n self.frame_offset += 1\n ret, frame = capture.read()\n if ret:\n width = frame.shape[1]\n height = frame.shape[0]\n return (int(height), int(width))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ninitializing the drawing of the images by setting the images to random colors.", "response": "def _init_draw(self):\n \"\"\"Initializes the drawing of the frames by setting the images to\n random colors.\n\n This function is called by TimedAnimation.\n \"\"\"\n if self.original is not None:\n self.original.set_data(np.random.random((10, 10, 3)))\n self.processed.set_data(np.random.random((10, 10, 3)))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read_frame(self):\n ret, frame = self.capture.read()\n if not ret:\n self.event_source.stop()\n try:\n self.capture.release()\n except AttributeError:\n # has no release method, thus just pass\n pass\n return None\n if self.convert_color != -1 and is_color_image(frame):\n return cv2.cvtColor(frame, self.convert_color)\n return frame", "response": "Reads a frame and converts it to a color if needed."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nannotate the processed axis with given annotations for the current frame number.", "response": "def annotate(self, framedata):\n \"\"\"Annotates the processed axis with given annotations for\n the provided framedata.\n\n Args:\n framedata: The current frame number.\n \"\"\"\n for artist in self.annotation_artists:\n artist.remove()\n self.annotation_artists = []\n for annotation in self.annotations:\n if annotation[2] > framedata:\n return\n if annotation[2] == framedata:\n pos = annotation[0:2]\n shape = self.annotations_default['shape']\n color = self.annotations_default['color']\n size = self.annotations_default['size']\n line = self.annotations_default['line']\n if len(annotation) > 3:\n shape = annotation[3].get('shape', shape)\n color = annotation[3].get('color', color)\n size = annotation[3].get('size', size)\n line = annotation[3].get('line', line)\n if shape == 'CIRC' and hasattr(size, '__len__'):\n size = 30\n\n if not hasattr(color, '__len__'):\n color = (color,) * 3\n\n if shape == 'RECT':\n patch = patches.Rectangle((pos[0] - size[0] // 2,\n pos[1] - size[1] // 2),\n size[0], size[1], fill=False,\n lw=line, fc='none', ec=color)\n elif shape == 'CIRC':\n patch = patches.CirclePolygon(pos, radius=size, fc='none',\n ec=color, lw=line)\n self.annotation_artists.append(patch)\n self.axes_processed.add_artist(self.annotation_artists[-1])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading processes and draws the frames.", "response": "def _draw_frame(self, framedata):\n \"\"\"Reads, processes and draws the frames.\n\n If needed for color maps, conversions to gray scale are performed. In\n case the images are no color images and no custom color maps are\n defined, the colormap `gray` is applied.\n\n This function is called by TimedAnimation.\n\n Args:\n framedata: The frame data.\n \"\"\"\n original = self.read_frame()\n if original is None:\n self.update_info(self.info_string(message='Finished.',\n frame=framedata))\n return\n\n if self.original is not None:\n processed = self.process_frame(original.copy())\n\n if self.cmap_original is not None:\n original = to_gray(original)\n elif not is_color_image(original):\n self.original.set_cmap('gray')\n self.original.set_data(original)\n else:\n processed = self.process_frame(original)\n\n if self.cmap_processed is not None:\n processed = to_gray(processed)\n elif not is_color_image(processed):\n self.processed.set_cmap('gray')\n\n if self.annotations:\n self.annotate(framedata)\n\n self.processed.set_data(processed)\n\n self.update_info(self.info_string(frame=framedata))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef update_info(self, custom=None):\n self.figure.suptitle(self.info_string() if custom is None else custom)", "response": "Updates the figure s suptitle."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef info_string(self, size=None, message='', frame=-1):\n info = []\n if size is not None:\n info.append('Size: {1}x{0}'.format(*size))\n elif self.size is not None:\n info.append('Size: {1}x{0}'.format(*self.size))\n if frame >= 0:\n info.append('Frame: {}'.format(frame))\n if message != '':\n info.append('{}'.format(message))\n return ' '.join(info)", "response": "Returns a string containing size frame number and info messages."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef main():\n with open(sys.argv[1], 'r') as nbfile:\n notebook = json.load(nbfile)\n\n # remove kernelspec (venvs)\n try:\n del notebook['metadata']['kernelspec']\n except KeyError:\n pass\n\n # remove outputs and metadata, set execution counts to None\n for cell in notebook['cells']:\n try:\n if cell['cell_type'] == 'code':\n cell['outputs'] = []\n cell['execution_count'] = None\n cell['metadata'] = {}\n except KeyError:\n pass\n\n with open(sys.argv[1], 'w') as nbfile:\n json.dump(notebook, nbfile, indent=1)", "response": "Sanitizes the loaded *. ipynb."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating comment :param comment: :param mentions: list of pair of code and type(\"USER\", \"GROUP\", and so on) :return:", "response": "def create(self, comment, mentions=()):\n \"\"\"\n create comment\n :param comment:\n :param mentions: list of pair of code and type(\"USER\", \"GROUP\", and so on)\n :return:\n \"\"\"\n\n data = {\n \"app\": self.app_id,\n \"record\": self.record_id,\n \"comment\": {\n \"text\": comment,\n }\n }\n\n if len(mentions) > 0:\n _mentions = []\n for m in mentions:\n if isinstance(m, (list, tuple)):\n if len(m) == 2:\n _mentions.append({\n \"code\": m[0],\n \"type\": m[1]\n })\n else:\n raise Exception(\"mention have to have code and target type. ex.[('user_1', 'USER')]\")\n elif isinstance(m, Mention):\n _mentions.append(m.serialize())\n\n data[\"comment\"][\"mentions\"] = _mentions\n\n resp = self._request(\"POST\", self._url, data)\n r = cr.CreateCommentResult(resp)\n return r"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _consume(iterator, n=None):\n # Use functions that consume iterators at C speed.\n if n is None:\n # feed the entire iterator into a zero-length deque\n collections.deque(iterator, maxlen=0)\n else:\n # advance to the empty slice starting at position n\n next(itertools.islice(iterator, n, n), None)", "response": "Advance the iterator n - steps ahead."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates how many items must be in the collection to satisfy this slice", "response": "def _slice_required_len(slice_obj):\n \"\"\"\n Calculate how many items must be in the collection to satisfy this slice\n\n returns `None` for slices may vary based on the length of the underlying collection\n such as `lst[-1]` or `lst[::]`\n \"\"\"\n if slice_obj.step and slice_obj.step != 1:\n return None\n # (None, None, *) requires the entire list\n if slice_obj.start is None and slice_obj.stop is None:\n return None\n\n # Negative indexes are hard without knowing the collection length\n if slice_obj.start and slice_obj.start < 0:\n return None\n if slice_obj.stop and slice_obj.stop < 0:\n return None\n\n if slice_obj.stop:\n if slice_obj.start and slice_obj.start > slice_obj.stop:\n return 0\n return slice_obj.stop\n return slice_obj.start + 1"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstylize text with given styles.", "response": "def stylize_interactive(text, styles, reset=True):\n \"\"\"stylize() variant that adds C0 control codes (SOH/STX) for readline\n safety.\"\"\"\n # problem: readline includes bare ANSI codes in width calculations.\n # solution: wrap nonprinting codes in SOH/STX when necessary.\n # see: https://github.com/dslackw/colored/issues/5\n terminator = _c0wrap(attr(\"reset\")) if reset else \"\"\n return \"{}{}{}\".format(_c0wrap(styles), text, terminator)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef attribute(self):\n\n paint = {\n \"bold\": self.ESC + \"1\" + self.END,\n 1: self.ESC + \"1\" + self.END,\n \"dim\": self.ESC + \"2\" + self.END,\n 2: self.ESC + \"2\" + self.END,\n \"underlined\": self.ESC + \"4\" + self.END,\n 4: self.ESC + \"4\" + self.END,\n \"blink\": self.ESC + \"5\" + self.END,\n 5: self.ESC + \"5\" + self.END,\n \"reverse\": self.ESC + \"7\" + self.END,\n 7: self.ESC + \"7\" + self.END,\n \"hidden\": self.ESC + \"8\" + self.END,\n 8: self.ESC + \"8\" + self.END,\n \"reset\": self.ESC + \"0\" + self.END,\n 0: self.ESC + \"0\" + self.END,\n \"res_bold\": self.ESC + \"21\" + self.END,\n 21: self.ESC + \"21\" + self.END,\n \"res_dim\": self.ESC + \"22\" + self.END,\n 22: self.ESC + \"22\" + self.END,\n \"res_underlined\": self.ESC + \"24\" + self.END,\n 24: self.ESC + \"24\" + self.END,\n \"res_blink\": self.ESC + \"25\" + self.END,\n 25: self.ESC + \"25\" + self.END,\n \"res_reverse\": self.ESC + \"27\" + self.END,\n 27: self.ESC + \"27\" + self.END,\n \"res_hidden\": self.ESC + \"28\" + self.END,\n 28: self.ESC + \"28\" + self.END,\n }\n return paint[self.color]", "response": "Set or reset attributes of the class."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprint 256 foreground colors", "response": "def foreground(self):\n \"\"\"Print 256 foreground colors\"\"\"\n code = self.ESC + \"38;5;\"\n if str(self.color).isdigit():\n self.reverse_dict()\n color = self.reserve_paint[str(self.color)]\n return code + self.paint[color] + self.END\n elif self.color.startswith(\"#\"):\n return code + str(self.HEX) + self.END\n else:\n return code + self.paint[self.color] + self.END"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nperform a reset and check for presence pulse.", "response": "def reset(self, required=False):\n \"\"\"\n Perform a reset and check for presence pulse.\n\n :param bool required: require presence pulse\n \"\"\"\n reset = self._ow.reset()\n if required and reset:\n raise OneWireError(\"No presence pulse found. Check devices and wiring.\")\n return not reset"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nread from the device into a byte array.", "response": "def readinto(self, buf, *, start=0, end=None):\n \"\"\"\n Read into ``buf`` from the device. The number of bytes read will be the\n length of ``buf``.\n\n If ``start`` or ``end`` is provided, then the buffer will be sliced\n as if ``buf[start:end]``. This will not cause an allocation like\n ``buf[start:end]`` will so it saves memory.\n\n :param bytearray buf: buffer to write into\n :param int start: Index to start writing at\n :param int end: Index to write up to but not include\n \"\"\"\n if end is None:\n end = len(buf)\n for i in range(start, end):\n buf[i] = self._readbyte()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write(self, buf, *, start=0, end=None):\n if end is None:\n end = len(buf)\n for i in range(start, end):\n self._writebyte(buf[i])", "response": "Write the bytes from buf to the device."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef scan(self):\n devices = []\n diff = 65\n rom = False\n count = 0\n for _ in range(0xff):\n rom, diff = self._search_rom(rom, diff)\n if rom:\n count += 1\n if count > self.maximum_devices:\n raise RuntimeError(\n \"Maximum device count of {} exceeded.\"\\\n .format(self.maximum_devices))\n devices.append(OneWireAddress(rom))\n if diff == 0:\n break\n return devices", "response": "Scan for devices on the bus and return a list of addresses."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms the 1 - Wire CRC check on the provided data.", "response": "def crc8(data):\n \"\"\"\n Perform the 1-Wire CRC check on the provided data.\n\n :param bytearray data: 8 byte array representing 64 bit ROM code\n \"\"\"\n crc = 0\n\n for byte in data:\n crc ^= byte\n for _ in range(8):\n if crc & 0x01:\n crc = (crc >> 1) ^ 0x8C\n else:\n crc >>= 1\n crc &= 0xFF\n return crc"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _deserialize(cls, json_body, get_value_and_type):\n\n instance = cls()\n is_set = False\n properties = cls._get_property_names(instance)\n\n def get_property_detail(name):\n p = [p for p in instance._property_details if p.name == name or p.field_name == name]\n return None if len(p) == 0 else p[0]\n\n for k in json_body:\n field = json_body[k]\n pd = get_property_detail(k)\n pn = k if not pd else pd.to_property_name(k)\n if pn in properties:\n v, t = get_value_and_type(field)\n initial_value = getattr(instance, pn)\n value = instance._field_to_property(v, t, pd, initial_value)\n setattr(instance, pn, value)\n is_set = True\n\n return instance if is_set else None", "response": "Deserialize json to model\n "} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _serialize(self, convert_to_key_and_value, ignore_missing=False):\n\n serialized = {}\n properties = self._get_property_names(self)\n\n def get_property_detail(name):\n p = [p for p in self._property_details if p.name == name]\n return None if len(p) == 0 else p[0]\n\n for p in properties:\n pd = get_property_detail(p)\n value = self._property_to_field(p, pd)\n field_name = p if not pd else pd.to_field_name()\n\n if value is None or (ignore_missing and not value) or (pd and pd.unsent):\n continue\n else:\n key, value = convert_to_key_and_value(field_name, value, pd)\n if key:\n serialized[key] = value\n\n return serialized", "response": "Serialize the current object to a dictionary."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread from the device into a buffer.", "response": "def readinto(self, buf, *, start=0, end=None):\n \"\"\"\n Read into ``buf`` from the device. The number of bytes read will be the\n length of ``buf``.\n\n If ``start`` or ``end`` is provided, then the buffer will be sliced\n as if ``buf[start:end]``. This will not cause an allocation like\n ``buf[start:end]`` will so it saves memory.\n\n :param bytearray buf: buffer to write into\n :param int start: Index to start writing at\n :param int end: Index to write up to but not include\n \"\"\"\n self._bus.readinto(buf, start=start, end=end)\n if start == 0 and end is None and len(buf) >= 8:\n if self._bus.crc8(buf):\n raise RuntimeError('CRC error.')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting the bytes from buf to the device.", "response": "def write(self, buf, *, start=0, end=None):\n \"\"\"\n Write the bytes from ``buf`` to the device.\n\n If ``start`` or ``end`` is provided, then the buffer will be sliced\n as if ``buffer[start:end]``. This will not cause an allocation like\n ``buffer[start:end]`` will so it saves memory.\n\n :param bytearray buf: buffer containing the bytes to write\n :param int start: Index to start writing from\n :param int end: Index to read up to but not include\n \"\"\"\n return self._bus.write(buf, start=start, end=end)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef preferences_class_prepared(sender, *args, **kwargs):\n cls = sender\n if issubclass(cls, Preferences):\n # Add singleton manager to subclasses.\n cls.add_to_class('singleton', SingletonManager())\n # Add property for preferences object to preferences.preferences.\n setattr(preferences.Preferences, cls._meta.object_name, property(lambda x: cls.singleton.get()))", "response": "Add singleton manager to preferences. preferences and preferences. preferences."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef site_cleanup(sender, action, instance, **kwargs):\n if action == 'post_add':\n if isinstance(instance, Preferences) \\\n and hasattr(instance.__class__, 'objects'):\n site_conflicts = instance.__class__.objects.filter(\n sites__in=instance.sites.all()\n ).only('id').distinct()\n\n for conflict in site_conflicts:\n if conflict.id != instance.id:\n for site in instance.sites.all():\n conflict.sites.remove(site)", "response": "Remove sites from pre - existing preferences objects."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the first preferences object for the current site. If preferences do not exist create it.", "response": "def get_queryset(self):\n \"\"\"\n Return the first preferences object for the current site.\n If preferences do not exist create it.\n \"\"\"\n\n queryset = super(SingletonManager, self).get_queryset()\n\n # Get current site\n current_site = None\n if getattr(settings, 'SITE_ID', None) is not None:\n current_site = Site.objects.get_current()\n\n # If site found limit queryset to site.\n if current_site is not None:\n queryset = queryset.filter(sites=settings.SITE_ID)\n\n if not queryset.exists():\n # Create object (for current site) if it doesn't exist.\n obj = self.model.objects.create()\n if current_site is not None:\n obj.sites.add(current_site)\n\n return queryset"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_iterable(self, iterable, session=None):\n '''Load an ``iterable``.\n\n By default it returns a generator of data loaded via the\n :meth:`loads` method.\n\n :param iterable: an iterable over data to load.\n :param session: Optional :class:`stdnet.odm.Session`.\n :return: an iterable over decoded data.\n '''\n data = []\n load = self.loads\n for v in iterable:\n data.append(load(v))\n return data", "response": "Load an iterable over data."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef search_model(self, q, text, lookup=None):\r\n '''Implements :meth:`stdnet.odm.SearchEngine.search_model`.\r\nIt return a new :class:`stdnet.odm.QueryElem` instance from\r\nthe input :class:`Query` and the *text* to search.'''\r\n words = self.words_from_text(text, for_search=True)\r\n if not words:\r\n return q\r\n qs = self._search(words, include=(q.model,), lookup=lookup)\r\n qs = tuple((q.get_field('object_id') for q in qs))\r\n return odm.intersect((q,)+qs)", "response": "Implements : meth : stdnet. ODM. SearchEngine. search_model.\r\nIt returns a new : class : QueryElem instance from the input : class : Query and the text to search."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _search(self, words, include=None, exclude=None, lookup=None):\r\n '''Full text search. Return a list of queries to intersect.'''\r\n lookup = lookup or 'contains'\r\n query = self.router.worditem.query()\r\n if include:\r\n query = query.filter(model_type__in=include)\r\n if exclude:\r\n query = query.exclude(model_type__in=include)\r\n if not words:\r\n return [query]\r\n qs = []\r\n if lookup == 'in':\r\n # we are looking for items with at least one word in it\r\n qs.append(query.filter(word__in=words))\r\n elif lookup == 'contains':\r\n #we want to match every single words\r\n for word in words:\r\n qs.append(query.filter(word=word))\r\n else:\r\n raise ValueError('Unknown lookup \"{0}\"'.format(lookup))\r\n return qs", "response": "Full text search. Return a list of queries to intersect."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets a new Redis client.", "response": "def redis_client(address=None, connection_pool=None, timeout=None,\n parser=None, **kwargs):\n '''Get a new redis client.\n\n :param address: a ``host``, ``port`` tuple.\n :param connection_pool: optional connection pool.\n :param timeout: socket timeout.\n :param timeout: socket timeout.\n '''\n if not connection_pool:\n if timeout == 0:\n if not async:\n raise ImportError('Asynchronous connection requires async '\n 'bindings installed.')\n return async.pool.redis(address, **kwargs)\n else:\n kwargs['socket_timeout'] = timeout\n return Redis(address[0], address[1], **kwargs)\n else:\n return Redis(connection_pool=connection_pool)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a bytestring version of s with the specified encoding as specified in encoding.", "response": "def to_bytes(s, encoding=None, errors='strict'):\r\n \"\"\"Returns a bytestring version of 's',\r\nencoded as specified in 'encoding'.\"\"\"\r\n encoding = encoding or 'utf-8'\r\n if isinstance(s, bytes):\r\n if encoding != 'utf-8':\r\n return s.decode('utf-8', errors).encode(encoding, errors)\r\n else:\r\n return s\r\n if not is_string(s):\r\n s = string_type(s)\r\n return s.encode(encoding, errors)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef date_decimal_hook(dct):\n '''The default JSON decoder hook. It is the inverse of\n:class:`stdnet.utils.jsontools.JSONDateDecimalEncoder`.'''\n if '__datetime__' in dct:\n return todatetime(dct['__datetime__'])\n elif '__date__' in dct:\n return todatetime(dct['__date__']).date()\n elif '__decimal__' in dct:\n return Decimal(dct['__decimal__'])\n else:\n return dct", "response": "The default JSON decoder hook. It is the inverse of\n : class : stdnet. utils. jsontools. JSONDateDecimalEncoder."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef flat_to_nested(data, instance=None, attname=None,\n separator=None, loads=None):\n '''Convert a flat representation of a dictionary to\na nested representation. Fields in the flat representation are separated\nby the *splitter* parameters.\n\n:parameter data: a flat dictionary of key value pairs.\n:parameter instance: optional instance of a model.\n:parameter attribute: optional attribute of a model.\n:parameter separator: optional separator. Default ``\"__\"``.\n:parameter loads: optional data unserializer.\n:rtype: a nested dictionary'''\n separator = separator or JSPLITTER\n val = {}\n flat_vals = {}\n for key, value in iteritems(data):\n if value is None:\n continue\n keys = key.split(separator)\n # first key equal to the attribute name\n if attname:\n if keys.pop(0) != attname:\n continue\n if loads:\n value = loads(value)\n # if an instance is available, inject the flat attribute\n if not keys:\n if value is None:\n val = flat_vals = {}\n break\n else:\n continue\n else:\n flat_vals[key] = value\n\n d = val\n lk = keys[-1]\n for k in keys[:-1]:\n if k not in d:\n nd = {}\n d[k] = nd\n else:\n nd = d[k]\n if not isinstance(nd, dict):\n nd = {'': nd}\n d[k] = nd\n d = nd\n if lk not in d:\n d[lk] = value\n else:\n d[lk][''] = value\n\n if instance and flat_vals:\n for attr, value in iteritems(flat_vals):\n setattr(instance, attr, value)\n\n return val", "response": "Convert a flat representation of a dictionary to a nested representation."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dict_flat_generator(value, attname=None, splitter=JSPLITTER,\n dumps=None, prefix=None, error=ValueError,\n recursive=True):\n '''Convert a nested dictionary into a flat dictionary representation'''\n if not isinstance(value, dict) or not recursive:\n if not prefix:\n raise error('Cannot assign a non dictionary to a JSON field')\n else:\n name = '%s%s%s' % (attname, splitter,\n prefix) if attname else prefix\n yield name, dumps(value) if dumps else value\n else:\n # loop over dictionary\n for field in value:\n val = value[field]\n key = prefix\n if field:\n key = '%s%s%s' % (prefix, splitter,\n field) if prefix else field\n for k, v2 in dict_flat_generator(val, attname, splitter, dumps,\n key, error, field):\n yield k, v2", "response": "Convert a nested dictionary into a flat dictionary representation"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmultiplies dictionaries by a numeric values and add them together.", "response": "def addmul_number_dicts(series):\n '''Multiply dictionaries by a numeric values and add them together.\n\n:parameter series: a tuple of two elements tuples. Each serie is of the form::\n\n (weight,dictionary)\n\n where ``weight`` is a number and ``dictionary`` is a dictionary with\n numeric values.\n:parameter skip: optional list of field names to skip.\n\nOnly common fields are aggregated. If a field has a non-numeric value it is\nnot included either.'''\n if not series:\n return\n vtype = value_type((s[1] for s in series))\n if vtype == 1:\n return sum((weight*float(d) for weight, d in series))\n elif vtype == 3:\n keys = set(series[0][1])\n for serie in series[1:]:\n keys.intersection_update(serie[1])\n results = {}\n for key in keys:\n key_series = tuple((weight, d[key]) for weight, d in series)\n result = addmul_number_dicts(key_series)\n if result is not None:\n results[key] = result\n return results"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndownloads all TPFs for a given K2 campaign.", "response": "def Download(campaign=0, queue='build', email=None, walltime=8, **kwargs):\n '''\n Submits a cluster job to the build queue to download all TPFs for a given\n campaign.\n\n :param int campaign: The `K2` campaign to run\n :param str queue: The name of the queue to submit to. Default `build`\n :param str email: The email to send job status notifications to. \\\n Default `None`\n :param int walltime: The number of hours to request. Default `8`\n\n '''\n\n # Figure out the subcampaign\n if type(campaign) is int:\n subcampaign = -1\n elif type(campaign) is float:\n x, y = divmod(campaign, 1)\n campaign = int(x)\n subcampaign = round(y * 10)\n # Submit the cluster job\n pbsfile = os.path.join(EVEREST_SRC, 'missions', 'k2', 'download.pbs')\n str_w = 'walltime=%d:00:00' % walltime\n str_v = 'EVEREST_DAT=%s,CAMPAIGN=%d,SUBCAMPAIGN=%d' % (\n EVEREST_DAT, campaign, subcampaign)\n if subcampaign == -1:\n str_name = 'download_c%02d' % campaign\n else:\n str_name = 'download_c%02d.%d' % (campaign, subcampaign)\n str_out = os.path.join(EVEREST_DAT, 'k2', str_name + '.log')\n qsub_args = ['qsub', pbsfile,\n '-q', queue,\n '-v', str_v,\n '-o', str_out,\n '-j', 'oe',\n '-N', str_name,\n '-l', str_w]\n if email is not None:\n qsub_args.append(['-M', email, '-m', 'ae'])\n # Now we submit the job\n print(\"Submitting the job...\")\n subprocess.call(qsub_args)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _Download(campaign, subcampaign):\n '''\n Download all stars from a given campaign. This is\n called from ``missions/k2/download.pbs``\n\n '''\n\n # Are we doing a subcampaign?\n if subcampaign != -1:\n campaign = campaign + 0.1 * subcampaign\n # Get all star IDs for this campaign\n stars = [s[0] for s in GetK2Campaign(campaign)]\n nstars = len(stars)\n # Download the TPF data for each one\n for i, EPIC in enumerate(stars):\n print(\"Downloading data for EPIC %d (%d/%d)...\" %\n (EPIC, i + 1, nstars))\n if not os.path.exists(os.path.join(EVEREST_DAT, 'k2',\n 'c%02d' % int(campaign),\n ('%09d' % EPIC)[:4] + '00000',\n ('%09d' % EPIC)[4:],\n 'data.npz')):\n try:\n GetData(EPIC, season=campaign, download_only=True)\n except KeyboardInterrupt:\n sys.exit()\n except:\n # Some targets could be corrupted...\n print(\"ERROR downloading EPIC %d.\" % EPIC)\n exctype, value, tb = sys.exc_info()\n for line in traceback.format_exception_only(exctype, value):\n ln = line.replace('\\n', '')\n print(ln)\n continue", "response": "Download all stars from a given campaign. This is the main function for the main function."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsubmit a cluster job to compute and plot data for all targets in a given campaign. :param campaign: The K2 campaign number. If this is an :py:class:`int`, \\ returns all targets in that campaign. If a :py:class:`float` \\ in the form `X.Y`, runs the `Y^th` decile of campaign `X`. :param str queue: The queue to submit to. Default `None` (default queue) :param str email: The email to send job status notifications to. \\ Default `None` :param int walltime: The number of hours to request. Default `100` :param int nodes: The number of nodes to request. Default `5` :param int ppn: The number of processors per node to request. Default `12` :param int mpn: Memory per node in gb to request. Default no setting.", "response": "def Run(campaign=0, EPIC=None, nodes=5, ppn=12, walltime=100,\n mpn=None, email=None, queue=None, **kwargs):\n '''\n Submits a cluster job to compute and plot data for all targets\n in a given campaign.\n\n :param campaign: The K2 campaign number. If this is an :py:class:`int`, \\\n returns all targets in that campaign. If a :py:class:`float` \\\n in the form `X.Y`, runs the `Y^th` decile of campaign `X`.\n :param str queue: The queue to submit to. Default `None` (default queue)\n :param str email: The email to send job status notifications to. \\\n Default `None`\n :param int walltime: The number of hours to request. Default `100`\n :param int nodes: The number of nodes to request. Default `5`\n :param int ppn: The number of processors per node to request. Default `12`\n :param int mpn: Memory per node in gb to request. Default no setting.\n\n '''\n\n # Figure out the subcampaign\n if type(campaign) is int:\n subcampaign = -1\n elif type(campaign) is float:\n x, y = divmod(campaign, 1)\n campaign = int(x)\n subcampaign = round(y * 10)\n\n # DEV hack: limit backfill jobs to 10 hours\n if EVEREST_DEV and (queue == 'bf'):\n walltime = min(10, walltime)\n\n # Convert kwargs to string. This is really hacky. Pickle creates an array\n # of bytes, which we must convert into a regular string to pass to the pbs\n # script and then back into python. Decoding the bytes isn't enough, since\n # we have pesky escaped characters such as newlines that don't behave well\n # when passing this string around. My braindead hack is to replace newlines\n # with '%%%', then undo the replacement when reading the kwargs. This works\n # for most cases, but sometimes pickle creates a byte array that can't be\n # decoded into utf-8; this happens when trying to pass numpy arrays around,\n # for instance. This needs to be fixed in the future, but for now we'll\n # restrict the kwargs to be ints, floats, lists, and strings.\n try:\n strkwargs = pickle.dumps(kwargs, 0).decode(\n 'utf-8').replace('\\n', '%%%')\n except UnicodeDecodeError:\n raise ValueError('Unable to pickle `kwargs`. Currently the `kwargs` ' +\n 'values may only be `int`s, `float`s, `string`s, ' +\n '`bool`s, or lists of these.')\n\n # Submit the cluster job\n pbsfile = os.path.join(EVEREST_SRC, 'missions', 'k2', 'run.pbs')\n if mpn is not None:\n str_n = 'nodes=%d:ppn=%d,feature=%dcore,mem=%dgb' % (\n nodes, ppn, ppn, mpn * nodes)\n else:\n str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)\n str_w = 'walltime=%d:00:00' % walltime\n str_v = \"EVEREST_DAT=%s,NODES=%d,\" % (EVEREST_DAT, nodes) + \\\n \"EPIC=%d,\" % (0 if EPIC is None else EPIC) + \\\n \"CAMPAIGN=%d,SUBCAMPAIGN=%d,STRKWARGS='%s'\" % \\\n (campaign, subcampaign, strkwargs)\n\n if EPIC is None:\n if subcampaign == -1:\n str_name = 'c%02d' % campaign\n else:\n str_name = 'c%02d.%d' % (campaign, subcampaign)\n else:\n str_name = 'EPIC%d' % EPIC\n str_out = os.path.join(EVEREST_DAT, 'k2', str_name + '.log')\n qsub_args = ['qsub', pbsfile,\n '-v', str_v,\n '-o', str_out,\n '-j', 'oe',\n '-N', str_name,\n '-l', str_n,\n '-l', str_w]\n if email is not None:\n qsub_args.append(['-M', email, '-m', 'ae'])\n if queue is not None:\n qsub_args += ['-q', queue]\n\n # Now we submit the job\n print(\"Submitting the job...\")\n subprocess.call(qsub_args)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _Publish(campaign, subcampaign, strkwargs):\n '''\n The actual function that publishes a given campaign; this must\n be called from ``missions/k2/publish.pbs``.\n\n '''\n\n # Get kwargs from string\n kwargs = pickle.loads(strkwargs.replace('%%%', '\\n').encode('utf-8'))\n\n # Check the cadence\n cadence = kwargs.get('cadence', 'lc')\n\n # Model wrapper\n m = FunctionWrapper(EverestModel, season=campaign, publish=True, **kwargs)\n\n # Set up our custom exception handler\n sys.excepthook = ExceptionHook\n\n # Initialize our multiprocessing pool\n with Pool() as pool:\n # Are we doing a subcampaign?\n if subcampaign != -1:\n campaign = campaign + 0.1 * subcampaign\n # Get all the stars\n stars = GetK2Campaign(campaign, epics_only=True, cadence=cadence)\n\n # Run\n pool.map(m, stars)", "response": "The actual function that publishes a given campaign. This must\n be called from the Mission. pbs file."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Status(season=range(18), model='nPLD', purge=False, injection=False,\n cadence='lc', **kwargs):\n '''\n Shows the progress of the de-trending runs for the specified campaign(s).\n\n '''\n\n # Mission compatibility\n campaign = season\n\n # Injection?\n if injection:\n return InjectionStatus(campaign=campaign, model=model,\n purge=purge, **kwargs)\n\n # Cadence\n if cadence == 'sc':\n model = '%s.sc' % model\n\n if not hasattr(campaign, '__len__'):\n if type(campaign) is int:\n # Return the subcampaigns\n all_stars = [s for s in GetK2Campaign(\n campaign, split=True, epics_only=True, cadence=cadence)]\n campaign = [campaign + 0.1 * n for n in range(10)]\n else:\n all_stars = [[s for s in GetK2Campaign(\n campaign, epics_only=True, cadence=cadence)]]\n campaign = [campaign]\n else:\n all_stars = [[s for s in GetK2Campaign(\n c, epics_only=True, cadence=cadence)] for c in campaign]\n\n print(\"CAMP TOTAL DOWNLOADED PROCESSED FITS ERRORS\")\n print(\"---- ----- ---------- --------- ---- ------\")\n for c, stars in zip(campaign, all_stars):\n if len(stars) == 0:\n continue\n down = 0\n proc = 0\n err = 0\n fits = 0\n bad = []\n remain = []\n total = len(stars)\n if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)):\n path = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)\n for folder in [f for f in os.listdir(path) if f.endswith('00000')]:\n for subfolder in os.listdir(os.path.join(path, folder)):\n ID = int(folder[:4] + subfolder)\n if ID in stars:\n if os.path.exists(os.path.join(EVEREST_DAT,\n 'k2', 'c%02d' % c,\n folder,\n subfolder, 'data.npz')):\n down += 1\n if os.path.exists(os.path.join(EVEREST_DAT, 'k2',\n 'c%02d' % c, folder,\n subfolder, FITSFile(\n ID, c,\n cadence=cadence))):\n fits += 1\n if os.path.exists(os.path.join(EVEREST_DAT, 'k2',\n 'c%02d' % c, folder,\n subfolder,\n model + '.npz')):\n proc += 1\n elif os.path.exists(os.path.join(EVEREST_DAT, 'k2',\n 'c%02d' % c, folder,\n subfolder,\n model + '.err')):\n err += 1\n bad.append(folder[:4] + subfolder)\n if purge:\n os.remove(os.path.join(\n EVEREST_DAT, 'k2', 'c%02d' % c,\n folder, subfolder, model + '.err'))\n else:\n remain.append(folder[:4] + subfolder)\n if proc == total:\n cc = ct = cp = ce = GREEN\n cd = BLACK if down < total else GREEN\n else:\n cc = BLACK\n ct = BLACK\n cd = BLACK if down < total else BLUE\n cp = BLACK if proc < down or proc == 0 else BLUE\n ce = RED if err > 0 else BLACK\n cf = BLACK if fits < total else GREEN\n if type(c) is int:\n print(\"%s{:>4d} \\033[0m%s{:>8d}\\033[0m%s{:>16d}\\033[0m%s{:>13d}\\033[0m%s{:>10d}\\033[0m%s{:>10d}\\033[0m\".format(c, total, down, proc, fits, err)\n % (cc, ct, cd, cp, cf, ce))\n else:\n print(\"%s{:>4.1f} \\033[0m%s{:>8d}\\033[0m%s{:>16d}\\033[0m%s{:>13d}\\033[0m%s{:>10d}\\033[0m%s{:>10d}\\033[0m\".format(c, total, down, proc, fits, err)\n % (cc, ct, cd, cp, cf, ce))\n if len(remain) <= 25 and len(remain) > 0 and len(campaign) == 1:\n remain.extend([\" \"] * (4 - (len(remain) % 4)))\n print()\n for A, B, C, D in zip(remain[::4], remain[1::4],\n remain[2::4], remain[3::4]):\n if A == remain[0]:\n print(\"REMAIN: %s %s %s %s\" % (A, B, C, D))\n print()\n else:\n print(\" %s %s %s %s\" % (A, B, C, D))\n print()\n if len(bad) and len(campaign) == 1:\n bad.extend([\" \"] * (4 - (len(bad) % 4)))\n print()\n for A, B, C, D in zip(bad[::4], bad[1::4], bad[2::4], bad[3::4]):\n if A == bad[0]:\n print(\"ERRORS: %s %s %s %s\" % (A, B, C, D))\n print()\n else:\n print(\" %s %s %s %s\" % (A, B, C, D))\n print()", "response": "Return the status of the K2 sub - campaigns."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef InjectionStatus(campaign=range(18), model='nPLD', purge=False,\n depths=[0.01, 0.001, 0.0001], **kwargs):\n '''\n Shows the progress of the injection de-trending runs for\n the specified campaign(s).\n\n '''\n\n if not hasattr(campaign, '__len__'):\n if type(campaign) is int:\n # Return the subcampaigns\n all_stars = [s for s in GetK2Campaign(\n campaign, split=True, epics_only=True)]\n campaign = [campaign + 0.1 * n for n in range(10)]\n else:\n all_stars = [[s for s in GetK2Campaign(campaign, epics_only=True)]]\n campaign = [campaign]\n else:\n all_stars = [[s for s in GetK2Campaign(\n c, epics_only=True)] for c in campaign]\n print(\"CAMP MASK DEPTH TOTAL DONE ERRORS\")\n print(\"---- ---- ----- ----- ---- ------\")\n for c, stars in zip(campaign, all_stars):\n if len(stars) == 0:\n continue\n done = [[0 for d in depths], [0 for d in depths]]\n err = [[0 for d in depths], [0 for d in depths]]\n total = len(stars)\n if os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)):\n path = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % c)\n for folder in os.listdir(path):\n for subfolder in os.listdir(os.path.join(path, folder)):\n ID = int(folder[:4] + subfolder)\n for m, mask in enumerate(['U', 'M']):\n for d, depth in enumerate(depths):\n if os.path.exists(\n os.path.join(\n EVEREST_DAT, 'k2', 'c%02d' % c, folder,\n subfolder, '%s_Inject_%s%g.npz' %\n (model, mask, depth))):\n done[m][d] += 1\n elif os.path.exists(\n os.path.join(\n EVEREST_DAT, 'k2', 'c%02d' % c, folder,\n subfolder, '%s_Inject_%s%g.err' %\n (model, mask, depth))):\n err[m][d] += 1\n for d, depth in enumerate(depths):\n for m, mask in enumerate(['F', 'T']):\n if done[m][d] == total:\n color = GREEN\n else:\n color = BLACK\n if err[m][d] > 0:\n errcolor = RED\n else:\n errcolor = ''\n if type(c) is int:\n print(\"%s{:>4d}{:>8s}{:>14g}{:>10d}{:>10d}%s{:>9d}\\033[0m\".format(\n c, mask, depth, total, done[m][d], err[m][d]) % (color, errcolor))\n else:\n print(\"%s{:>4.1f}{:>8s}{:>14g}{:>10d}{:>10d}%s{:>9d}\\033[0m\".format(\n c, mask, depth, total, done[m][d], err[m][d]) % (color, errcolor))", "response": "A simple status message for the injection of a single resource."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconstruct the primary HDU file containing basic header info.", "response": "def PrimaryHDU(model):\n '''\n Construct the primary HDU file containing basic header info.\n\n '''\n\n # Get mission cards\n cards = model._mission.HDUCards(model.meta, hdu=0)\n if 'KEPMAG' not in [c[0] for c in cards]:\n cards.append(('KEPMAG', model.mag, 'Kepler magnitude'))\n\n # Add EVEREST info\n cards.append(('COMMENT', '************************'))\n cards.append(('COMMENT', '* EVEREST INFO *'))\n cards.append(('COMMENT', '************************'))\n cards.append(('MISSION', model.mission, 'Mission name'))\n cards.append(('VERSION', EVEREST_MAJOR_MINOR, 'EVEREST pipeline version'))\n cards.append(('SUBVER', EVEREST_VERSION, 'EVEREST pipeline subversion'))\n cards.append(('DATE', strftime('%Y-%m-%d'),\n 'EVEREST file creation date (YYYY-MM-DD)'))\n\n # Create the HDU\n header = pyfits.Header(cards=cards)\n hdu = pyfits.PrimaryHDU(header=header)\n\n return hdu"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconstruct the data HDU file containing the arrays and observing info.", "response": "def LightcurveHDU(model):\n '''\n Construct the data HDU file containing the arrays and the observing info.\n\n '''\n\n # Get mission cards\n cards = model._mission.HDUCards(model.meta, hdu=1)\n\n # Add EVEREST info\n cards.append(('COMMENT', '************************'))\n cards.append(('COMMENT', '* EVEREST INFO *'))\n cards.append(('COMMENT', '************************'))\n cards.append(('MISSION', model.mission, 'Mission name'))\n cards.append(('VERSION', EVEREST_MAJOR_MINOR, 'EVEREST pipeline version'))\n cards.append(('SUBVER', EVEREST_VERSION, 'EVEREST pipeline subversion'))\n cards.append(('DATE', strftime('%Y-%m-%d'),\n 'EVEREST file creation date (YYYY-MM-DD)'))\n cards.append(('MODEL', model.name, 'Name of EVEREST model used'))\n cards.append(('APNAME', model.aperture_name, 'Name of aperture used'))\n cards.append(('BPAD', model.bpad, 'Chunk overlap in cadences'))\n for c in range(len(model.breakpoints)):\n cards.append(\n ('BRKPT%02d' % (c + 1), model.breakpoints[c],\n 'Light curve breakpoint'))\n cards.append(('CBVNUM', model.cbv_num, 'Number of CBV signals to recover'))\n cards.append(('CBVNITER', model.cbv_niter,\n 'Number of CBV SysRem iterations'))\n cards.append(('CBVWIN', model.cbv_win, 'Window size for smoothing CBVs'))\n cards.append(('CBVORD', model.cbv_order, 'Order when smoothing CBVs'))\n cards.append(('CDIVS', model.cdivs, 'Cross-validation subdivisions'))\n cards.append(('CDPP', model.cdpp, 'Average de-trended CDPP'))\n cards.append(('CDPPR', model.cdppr, 'Raw CDPP'))\n cards.append(('CDPPV', model.cdppv, 'Average validation CDPP'))\n cards.append(('CDPPG', model.cdppg, 'Average GP-de-trended CDPP'))\n for i in range(99):\n try:\n cards.append(('CDPP%02d' % (i + 1),\n model.cdpp_arr[i] if not np.isnan(\n model.cdpp_arr[i]) else 0, 'Chunk de-trended CDPP'))\n cards.append(('CDPPR%02d' % (\n i + 1), model.cdppr_arr[i] if not np.isnan(\n model.cdppr_arr[i]) else 0, 'Chunk raw CDPP'))\n cards.append(('CDPPV%02d' % (i + 1),\n model.cdppv_arr[i] if not np.isnan(\n model.cdppv_arr[i]) else 0, 'Chunk validation CDPP'))\n except:\n break\n cards.append(\n ('CVMIN', model.cv_min, 'Cross-validation objective function'))\n cards.append(\n ('GITER', model.giter, 'Number of GP optimiziation iterations'))\n cards.append(\n ('GMAXF', model.giter, 'Max number of GP function evaluations'))\n cards.append(('GPFACTOR', model.gp_factor,\n 'GP amplitude initialization factor'))\n cards.append(('KERNEL', model.kernel, 'GP kernel name'))\n if model.kernel == 'Basic':\n cards.append(\n ('GPWHITE', model.kernel_params[0],\n 'GP white noise amplitude (e-/s)'))\n cards.append(\n ('GPRED', model.kernel_params[1],\n 'GP red noise amplitude (e-/s)'))\n cards.append(\n ('GPTAU', model.kernel_params[2],\n 'GP red noise timescale (days)'))\n elif model.kernel == 'QuasiPeriodic':\n cards.append(\n ('GPWHITE', model.kernel_params[0],\n 'GP white noise amplitude (e-/s)'))\n cards.append(\n ('GPRED', model.kernel_params[1], 'GP red noise amplitude (e-/s)'))\n cards.append(('GPGAMMA', model.kernel_params[2], 'GP scale factor'))\n cards.append(('GPPER', model.kernel_params[3], 'GP period (days)'))\n for c in range(len(model.breakpoints)):\n for o in range(model.pld_order):\n cards.append(('LAMB%02d%02d' % (c + 1, o + 1),\n model.lam[c][o], 'Cross-validation parameter'))\n if model.name == 'iPLD':\n cards.append(('RECL%02d%02d' % (c + 1, o + 1),\n model.reclam[c][o],\n 'Cross-validation parameter'))\n cards.append(('LEPS', model.leps, 'Cross-validation tolerance'))\n cards.append(('MAXPIX', model.max_pixels, 'Maximum size of TPF aperture'))\n for i, source in enumerate(model.nearby[:99]):\n cards.append(('NRBY%02dID' %\n (i + 1), source['ID'], 'Nearby source ID'))\n cards.append(\n ('NRBY%02dX' % (i + 1), source['x'], 'Nearby source X position'))\n cards.append(\n ('NRBY%02dY' % (i + 1), source['y'], 'Nearby source Y position'))\n cards.append(\n ('NRBY%02dM' % (i + 1), source['mag'], 'Nearby source magnitude'))\n cards.append(('NRBY%02dX0' %\n (i + 1), source['x0'], 'Nearby source reference X'))\n cards.append(('NRBY%02dY0' %\n (i + 1), source['y0'], 'Nearby source reference Y'))\n for i, n in enumerate(model.neighbors):\n cards.append(\n ('NEIGH%02d' % i, model.neighbors[i],\n 'Neighboring star used to de-trend'))\n cards.append(('OITER', model.oiter, 'Number of outlier search iterations'))\n cards.append(('OPTGP', model.optimize_gp, 'GP optimization performed?'))\n cards.append(\n ('OSIGMA', model.osigma, 'Outlier tolerance (standard deviations)'))\n for i, planet in enumerate(model.planets):\n cards.append(\n ('P%02dT0' % (i + 1), planet[0], 'Planet transit time (days)'))\n cards.append(\n ('P%02dPER' % (i + 1), planet[1], 'Planet transit period (days)'))\n cards.append(\n ('P%02dDUR' % (i + 1), planet[2],\n 'Planet transit duration (days)'))\n cards.append(('PLDORDER', model.pld_order, 'PLD de-trending order'))\n cards.append(('SATUR', model.saturated, 'Is target saturated?'))\n cards.append(('SATTOL', model.saturation_tolerance,\n 'Fractional saturation tolerance'))\n\n # Add the EVEREST quality flags to the QUALITY array\n quality = np.array(model.quality)\n quality[np.array(model.badmask, dtype=int)] += 2 ** (QUALITY_BAD - 1)\n quality[np.array(model.nanmask, dtype=int)] += 2 ** (QUALITY_NAN - 1)\n quality[np.array(model.outmask, dtype=int)] += 2 ** (QUALITY_OUT - 1)\n quality[np.array(model.recmask, dtype=int)] += 2 ** (QUALITY_REC - 1)\n quality[np.array(model.transitmask, dtype=int)] += 2 ** (QUALITY_TRN - 1)\n\n # When de-trending, we interpolated to fill in NaN fluxes. Here\n # we insert the NaNs back in, since there's no actual physical\n # information at those cadences.\n flux = np.array(model.flux)\n flux[model.nanmask] = np.nan\n\n # Create the arrays list\n arrays = [pyfits.Column(name='CADN', format='D', array=model.cadn),\n pyfits.Column(name='FLUX', format='D', array=flux, unit='e-/s'),\n pyfits.Column(name='FRAW', format='D',\n array=model.fraw, unit='e-/s'),\n pyfits.Column(name='FRAW_ERR', format='D',\n array=model.fraw_err, unit='e-/s'),\n pyfits.Column(name='QUALITY', format='J', array=quality),\n pyfits.Column(name='TIME', format='D',\n array=model.time, unit='BJD - 2454833')]\n\n # Add the CBVs\n if model.fcor is not None:\n arrays += [pyfits.Column(name='FCOR', format='D',\n array=model.fcor, unit='e-/s')]\n for n in range(model.XCBV.shape[1]):\n arrays += [pyfits.Column(name='CBV%02d' %\n (n + 1), format='D',\n array=model.XCBV[:, n])]\n\n # Did we subtract a background term?\n if hasattr(model.bkg, '__len__'):\n arrays.append(pyfits.Column(name='BKG', format='D',\n array=model.bkg, unit='e-/s'))\n\n # Create the HDU\n header = pyfits.Header(cards=cards)\n cols = pyfits.ColDefs(arrays)\n hdu = pyfits.BinTableHDU.from_columns(cols, header=header, name='ARRAYS')\n\n return hdu"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconstructing the HDU containing the pixel - level light curve.", "response": "def PixelsHDU(model):\n '''\n Construct the HDU containing the pixel-level light curve.\n\n '''\n\n # Get mission cards\n cards = model._mission.HDUCards(model.meta, hdu=2)\n\n # Add EVEREST info\n cards = []\n cards.append(('COMMENT', '************************'))\n cards.append(('COMMENT', '* EVEREST INFO *'))\n cards.append(('COMMENT', '************************'))\n cards.append(('MISSION', model.mission, 'Mission name'))\n cards.append(('VERSION', EVEREST_MAJOR_MINOR, 'EVEREST pipeline version'))\n cards.append(('SUBVER', EVEREST_VERSION, 'EVEREST pipeline subversion'))\n cards.append(('DATE', strftime('%Y-%m-%d'),\n 'EVEREST file creation date (YYYY-MM-DD)'))\n\n # Create the HDU\n header = pyfits.Header(cards=cards)\n\n # The pixel timeseries\n arrays = [pyfits.Column(name='FPIX', format='%dD' %\n model.fpix.shape[1], array=model.fpix)]\n\n # The first order PLD vectors for all the neighbors (npixels, ncadences)\n X1N = model.X1N\n if X1N is not None:\n arrays.append(pyfits.Column(name='X1N', format='%dD' %\n X1N.shape[1], array=X1N))\n\n cols = pyfits.ColDefs(arrays)\n hdu = pyfits.BinTableHDU.from_columns(cols, header=header, name='PIXELS')\n\n return hdu"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ApertureHDU(model):\n '''\n Construct the HDU containing the aperture used to de-trend.\n\n '''\n\n # Get mission cards\n cards = model._mission.HDUCards(model.meta, hdu=3)\n\n # Add EVEREST info\n cards.append(('COMMENT', '************************'))\n cards.append(('COMMENT', '* EVEREST INFO *'))\n cards.append(('COMMENT', '************************'))\n cards.append(('MISSION', model.mission, 'Mission name'))\n cards.append(('VERSION', EVEREST_MAJOR_MINOR, 'EVEREST pipeline version'))\n cards.append(('SUBVER', EVEREST_VERSION, 'EVEREST pipeline subversion'))\n cards.append(('DATE', strftime('%Y-%m-%d'),\n 'EVEREST file creation date (YYYY-MM-DD)'))\n\n # Create the HDU\n header = pyfits.Header(cards=cards)\n hdu = pyfits.ImageHDU(data=model.aperture,\n header=header, name='APERTURE MASK')\n\n return hdu", "response": "Construct the HDU containing the aperture used to de - trend."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ImagesHDU(model):\n '''\n Construct the HDU containing sample postage stamp images of the target.\n\n '''\n\n # Get mission cards\n cards = model._mission.HDUCards(model.meta, hdu=4)\n\n # Add EVEREST info\n cards.append(('COMMENT', '************************'))\n cards.append(('COMMENT', '* EVEREST INFO *'))\n cards.append(('COMMENT', '************************'))\n cards.append(('MISSION', model.mission, 'Mission name'))\n cards.append(('VERSION', EVEREST_MAJOR_MINOR, 'EVEREST pipeline version'))\n cards.append(('SUBVER', EVEREST_VERSION, 'EVEREST pipeline subversion'))\n cards.append(('DATE', strftime('%Y-%m-%d'),\n 'EVEREST file creation date (YYYY-MM-DD)'))\n\n # The images\n format = '%dD' % model.pixel_images[0].shape[1]\n arrays = [pyfits.Column(name='STAMP1', format=format,\n array=model.pixel_images[0]),\n pyfits.Column(name='STAMP2', format=format,\n array=model.pixel_images[1]),\n pyfits.Column(name='STAMP3', format=format,\n array=model.pixel_images[2])]\n\n # Create the HDU\n header = pyfits.Header(cards=cards)\n cols = pyfits.ColDefs(arrays)\n hdu = pyfits.BinTableHDU.from_columns(\n cols, header=header, name='POSTAGE STAMPS')\n\n return hdu", "response": "Construct the HDU containing sample postage stamp images of the target."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconstructs the HDU containing the hi res image of the target.", "response": "def HiResHDU(model):\n '''\n Construct the HDU containing the hi res image of the target.\n\n '''\n\n # Get mission cards\n cards = model._mission.HDUCards(model.meta, hdu=5)\n\n # Add EVEREST info\n cards.append(('COMMENT', '************************'))\n cards.append(('COMMENT', '* EVEREST INFO *'))\n cards.append(('COMMENT', '************************'))\n cards.append(('MISSION', model.mission, 'Mission name'))\n cards.append(('VERSION', EVEREST_MAJOR_MINOR, 'EVEREST pipeline version'))\n cards.append(('SUBVER', EVEREST_VERSION, 'EVEREST pipeline subversion'))\n cards.append(('DATE', strftime('%Y-%m-%d'),\n 'EVEREST file creation date (YYYY-MM-DD)'))\n\n # Create the HDU\n header = pyfits.Header(cards=cards)\n if model.hires is not None:\n hdu = pyfits.ImageHDU(\n data=model.hires, header=header, name='HI RES IMAGE')\n else:\n hdu = pyfits.ImageHDU(data=np.empty(\n (0, 0), dtype=float), header=header, name='HI RES IMAGE')\n return hdu"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef MakeFITS(model, fitsfile=None):\n '''\n Generate a FITS file for a given :py:mod:`everest` run.\n\n :param model: An :py:mod:`everest` model instance\n\n '''\n\n # Get the fits file name\n if fitsfile is None:\n outfile = os.path.join(model.dir, model._mission.FITSFile(\n model.ID, model.season, model.cadence))\n else:\n outfile = os.path.join(model.dir, fitsfile)\n if os.path.exists(outfile) and not model.clobber:\n return\n elif os.path.exists(outfile):\n os.remove(outfile)\n\n log.info('Generating FITS file...')\n\n # Create the HDUs\n primary = PrimaryHDU(model)\n lightcurve = LightcurveHDU(model)\n pixels = PixelsHDU(model)\n aperture = ApertureHDU(model)\n images = ImagesHDU(model)\n hires = HiResHDU(model)\n\n # Combine to get the HDUList\n hdulist = pyfits.HDUList(\n [primary, lightcurve, pixels, aperture, images, hires])\n\n # Output to the FITS file\n hdulist.writeto(outfile)\n\n return", "response": "Generates a FITS file for a given model instance."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve a serializer register as name.", "response": "def get_serializer(name, **options):\r\n '''Retrieve a serializer register as *name*. If the serializer is not\r\navailable a ``ValueError`` exception will raise.\r\nA common usage pattern::\r\n\r\n qs = MyModel.objects.query().sort_by('id')\r\n s = odm.get_serializer('json')\r\n s.dump(qs)\r\n'''\r\n if name in _serializers:\r\n serializer = _serializers[name]\r\n return serializer(**options)\r\n else:\r\n raise ValueError('Unknown serializer {0}.'.format(name))"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nregisters a new serializer to the library.", "response": "def register_serializer(name, serializer):\r\n '''\\\r\nRegister a new serializer to the library.\r\n\r\n:parameter name: serializer name (it can override existing serializers).\r\n:parameter serializer: an instance or a derived class of a\r\n :class:`stdnet.odm.Serializer` class or a callable.\r\n'''\r\n if not isclass(serializer):\r\n serializer = serializer.__class__\r\n _serializers[name] = serializer"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsolves the linear problem A with a set of contiguous w - sized masks applied to the rows and columns of b.", "response": "def MaskSolve(A, b, w=5, progress=True, niter=None):\r\n '''\r\n Finds the solution `x` to the linear problem\r\n\r\n A x = b\r\n\r\n for all contiguous `w`-sized masks applied to\r\n the rows and columns of `A` and to the entries\r\n of `b`.\r\n\r\n Returns an array `X` of shape `(N - w + 1, N - w)`,\r\n where the `nth` row is the solution to the equation\r\n\r\n A[![n,n+w)] x = b[![n,n+w)]\r\n\r\n where ![n,n+w) indicates that indices in the range\r\n [n,n+w) have been masked.\r\n\r\n '''\r\n\r\n # Ensure we have choldate installed\r\n if cholupdate is None:\r\n log.info(\"Running the slow version of `MaskSolve`.\")\r\n log.info(\"Install the `choldate` package for better performance.\")\r\n log.info(\"https://github.com/rodluger/choldate\")\r\n return MaskSolveSlow(A, b, w=w, progress=progress, niter=niter)\r\n\r\n # Number of data points\r\n N = b.shape[0]\r\n\r\n # How many iterations? Default is to go through\r\n # the entire dataset\r\n if niter is None:\r\n niter = N - w + 1\r\n\r\n # Our result matrix\r\n X = np.empty((niter, N - w))\r\n\r\n # Solve the first two steps explicitly.\r\n for n in range(2):\r\n mask = np.arange(n, w + n)\r\n A_ = np.delete(np.delete(A, mask, axis=0), mask, axis=1)\r\n b_ = np.delete(b, mask)\r\n U = cholesky(A_)\r\n X[n] = cho_solve((U, False), b_)\r\n\r\n # Iterate!\r\n for n in prange(1, niter - 1):\r\n\r\n # Update the data vector.\r\n b_[n] = b[n]\r\n\r\n # Remove a row.\r\n S33 = U[n + 1:, n + 1:]\r\n S23 = U[n, n + 1:]\r\n cholupdate(S33, S23)\r\n\r\n # Add a row.\r\n A12 = A[:n, n]\r\n A22 = A[n, n]\r\n A23 = A[n, n + w + 1:]\r\n S11 = U[:n, :n]\r\n S12 = solve_triangular(S11.T, A12, lower=True,\r\n check_finite=False, trans=0, overwrite_b=True)\r\n S22 = np.sqrt(A22 - np.dot(S12.T, S12))\r\n S13 = U[:n, n + 1:]\r\n S23 = (A23 - np.dot(S12.T, S13)) / S22\r\n choldowndate(S33, np.array(S23))\r\n U[:n, n] = S12\r\n U[n, n] = S22\r\n U[n, n + 1:] = S23\r\n U[n + 1:, n + 1:] = S33\r\n\r\n # Now we can solve our linear equation\r\n X[n + 1] = cho_solve((U, False), b_)\r\n\r\n # Return the matrix\r\n return X"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MaskSolveSlow(A, b, w=5, progress=True, niter=None):\r\n '''\r\n Identical to `MaskSolve`, but computes the solution\r\n the brute-force way.\r\n\r\n '''\r\n\r\n # Number of data points\r\n N = b.shape[0]\r\n\r\n # How many iterations? Default is to go through\r\n # the entire dataset\r\n if niter is None:\r\n niter = N - w + 1\r\n\r\n # Our result matrix\r\n X = np.empty((niter, N - w))\r\n\r\n # Iterate! The mask at step `n` goes from\r\n # data index `n` to data index `n+w-1` (inclusive).\r\n for n in prange(niter):\r\n mask = np.arange(n, n + w)\r\n An = np.delete(np.delete(A, mask, axis=0), mask, axis=1)\r\n Un = cholesky(An)\r\n bn = np.delete(b, mask)\r\n X[n] = cho_solve((Un, False), bn)\r\n\r\n return X", "response": "Identical to MaskSolve but computes the solution of A and b in a brute - force way."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef unmasked(self, depth=0.01):\n return 1 - (np.hstack(self._O2) +\n np.hstack(self._O3) / depth) / np.hstack(self._O1)", "response": "Return the unmasked overfitting metric for a given transit depth."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef show(self):\n try:\n if platform.system().lower().startswith('darwin'):\n subprocess.call(['open', self.pdf])\n elif os.name == 'nt':\n os.startfile(self.pdf)\n elif os.name == 'posix':\n subprocess.call(['xdg-open', self.pdf])\n else:\n raise IOError(\"\")\n except IOError:\n log.info(\"Unable to open the pdf. Try opening it manually:\")\n log.info(self.pdf)", "response": "Show the overfitting PDF summary."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the current season for this target.", "response": "def season(self):\n \"\"\"\n Return the current observing season.\n\n For *K2*, this is the observing campaign, while for *Kepler*,\n it is the current quarter.\n\n \"\"\"\n try:\n self._season\n except AttributeError:\n self._season = self._mission.Season(self.ID)\n if hasattr(self._season, '__len__'):\n raise AttributeError(\n \"Please choose a campaign/season for this target: %s.\" %\n self._season)\n return self._season"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncomputes the design matrix at the given indices and returns the X1 matrix.", "response": "def X(self, i, j=slice(None, None, None)):\n '''\n Computes the design matrix at the given *PLD* order and the given\n indices. The columns are the *PLD* vectors for the target at the\n corresponding order, computed as the product of the fractional pixel\n flux of all sets of :py:obj:`n` pixels, where :py:obj:`n` is the *PLD*\n order.\n\n '''\n\n X1 = self.fpix[j] / self.norm[j].reshape(-1, 1)\n X = np.product(list(multichoose(X1.T, i + 1)), axis=1).T\n if self.X1N is not None:\n return np.hstack([X, self.X1N[j] ** (i + 1)])\n else:\n return X"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nplots miscellaneous de - trending information on the validation summary figure.", "response": "def plot_info(self, dvs):\n '''\n Plots miscellaneous de-trending information on the data\n validation summary figure.\n\n :param dvs: A :py:class:`dvs.DVS` figure instance\n\n '''\n\n axl, axc, axr = dvs.title()\n axc.annotate(\"%s %d\" % (self._mission.IDSTRING, self.ID),\n xy=(0.5, 0.5), xycoords='axes fraction',\n ha='center', va='center', fontsize=18)\n\n axc.annotate(r\"%.2f ppm $\\rightarrow$ %.2f ppm\" %\n (self.cdppr, self.cdpp),\n xy=(0.5, 0.2), xycoords='axes fraction',\n ha='center', va='center', fontsize=8, color='k',\n fontstyle='italic')\n\n axl.annotate(\"%s %s%02d: %s\" %\n (self.mission.upper(),\n self._mission.SEASONCHAR, self.season, self.name),\n xy=(0.5, 0.5), xycoords='axes fraction',\n ha='center', va='center', fontsize=12,\n color='k')\n\n axl.annotate(self.aperture_name if len(self.neighbors) == 0\n else \"%s, %d neighbors\" %\n (self.aperture_name, len(self.neighbors)),\n xy=(0.5, 0.2), xycoords='axes fraction',\n ha='center', va='center', fontsize=8, color='k',\n fontstyle='italic')\n\n axr.annotate(\"%s %.3f\" % (self._mission.MAGSTRING, self.mag),\n xy=(0.5, 0.5), xycoords='axes fraction',\n ha='center', va='center', fontsize=12,\n color='k')\n\n if not np.isnan(self.cdppg) and self.cdppg > 0:\n axr.annotate(r\"GP %.3f ppm\" % (self.cdppg),\n xy=(0.5, 0.2), xycoords='axes fraction',\n ha='center', va='center', fontsize=8, color='k',\n fontstyle='italic')"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the model for the current value of lambda.", "response": "def compute(self):\n '''\n Compute the model for the current value of lambda.\n\n '''\n\n # Is there a transit model?\n if self.transit_model is not None:\n return self.compute_joint()\n\n log.info('Computing the model...')\n\n # Loop over all chunks\n model = [None for b in self.breakpoints]\n for b, brkpt in enumerate(self.breakpoints):\n\n # Masks for current chunk\n m = self.get_masked_chunk(b)\n c = self.get_chunk(b)\n\n # This block of the masked covariance matrix\n mK = GetCovariance(self.kernel, self.kernel_params,\n self.time[m], self.fraw_err[m])\n\n # Get median\n med = np.nanmedian(self.fraw[m])\n\n # Normalize the flux\n f = self.fraw[m] - med\n\n # The X^2 matrices\n A = np.zeros((len(m), len(m)))\n B = np.zeros((len(c), len(m)))\n\n # Loop over all orders\n for n in range(self.pld_order):\n\n # Only compute up to the current PLD order\n if (self.lam_idx >= n) and (self.lam[b][n] is not None):\n XM = self.X(n, m)\n XC = self.X(n, c)\n A += self.lam[b][n] * np.dot(XM, XM.T)\n B += self.lam[b][n] * np.dot(XC, XM.T)\n del XM, XC\n\n # Compute the model\n W = np.linalg.solve(mK + A, f)\n model[b] = np.dot(B, W)\n\n # Free up some memory\n del A, B, W\n\n # Join the chunks after applying the correct offset\n if len(model) > 1:\n\n # First chunk\n self.model = model[0][:-self.bpad]\n\n # Center chunks\n for m in model[1:-1]:\n # Join the chunks at the first non-outlier cadence\n i = 1\n while len(self.model) - i in self.mask:\n i += 1\n offset = self.model[-i] - m[self.bpad - i]\n self.model = np.concatenate(\n [self.model, m[self.bpad:-self.bpad] + offset])\n\n # Last chunk\n i = 1\n while len(self.model) - i in self.mask:\n i += 1\n offset = self.model[-i] - model[-1][self.bpad - i]\n self.model = np.concatenate(\n [self.model, model[-1][self.bpad:] + offset])\n\n else:\n\n self.model = model[0]\n\n # Subtract the global median\n self.model -= np.nanmedian(self.model)\n\n # Get the CDPP and reset the weights\n self.cdpp_arr = self.get_cdpp_arr()\n self.cdpp = self.get_cdpp()\n self._weights = None"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef compute_joint(self):\n '''\n Compute the model in a single step, allowing for a light curve-wide\n transit model. This is a bit more expensive to compute.\n\n '''\n\n # Init\n log.info('Computing the joint model...')\n A = [None for b in self.breakpoints]\n B = [None for b in self.breakpoints]\n\n # We need to make sure that we're not masking the transits we are\n # trying to fit!\n # NOTE: If there happens to be an index that *SHOULD* be masked during\n # a transit (cosmic ray, detector anomaly), update `self.badmask`\n # to include that index.\n # Bad data points are *never* used in the regression.\n if self.transit_model is not None:\n outmask = np.array(self.outmask)\n transitmask = np.array(self.transitmask)\n transit_inds = np.where(\n np.sum([tm(self.time) for tm in self.transit_model],\n axis=0) < 0)[0]\n self.outmask = np.array(\n [i for i in self.outmask if i not in transit_inds])\n self.transitmask = np.array(\n [i for i in self.transitmask if i not in transit_inds])\n\n # Loop over all chunks\n for b, brkpt in enumerate(self.breakpoints):\n\n # Masks for current chunk\n m = self.get_masked_chunk(b, pad=False)\n c = self.get_chunk(b, pad=False)\n\n # The X^2 matrices\n A[b] = np.zeros((len(m), len(m)))\n B[b] = np.zeros((len(c), len(m)))\n\n # Loop over all orders\n for n in range(self.pld_order):\n\n # Only compute up to the current PLD order\n if (self.lam_idx >= n) and (self.lam[b][n] is not None):\n XM = self.X(n, m)\n XC = self.X(n, c)\n A[b] += self.lam[b][n] * np.dot(XM, XM.T)\n B[b] += self.lam[b][n] * np.dot(XC, XM.T)\n del XM, XC\n\n # Merge chunks. BIGA and BIGB are sparse, but unfortunately\n # scipy.sparse doesn't handle sparse matrix inversion all that\n # well when the *result* is not itself sparse. So we're sticking\n # with regular np.linalg.\n BIGA = block_diag(*A)\n del A\n BIGB = block_diag(*B)\n del B\n\n # Compute the full covariance matrix\n mK = GetCovariance(self.kernel, self.kernel_params, self.apply_mask(\n self.time), self.apply_mask(self.fraw_err))\n\n # The normalized, masked flux array\n f = self.apply_mask(self.fraw)\n med = np.nanmedian(f)\n f -= med\n\n # Are we computing a joint transit model?\n if self.transit_model is not None:\n\n # Get the unmasked indices\n m = self.apply_mask()\n\n # Subtract off the mean total transit model\n mean_transit_model = med * \\\n np.sum([tm.depth * tm(self.time[m])\n for tm in self.transit_model], axis=0)\n f -= mean_transit_model\n\n # Now add each transit model to the matrix of regressors\n for tm in self.transit_model:\n XM = tm(self.time[m]).reshape(-1, 1)\n XC = tm(self.time).reshape(-1, 1)\n BIGA += med ** 2 * tm.var_depth * np.dot(XM, XM.T)\n BIGB += med ** 2 * tm.var_depth * np.dot(XC, XM.T)\n del XM, XC\n\n # Dot the inverse of the covariance matrix\n W = np.linalg.solve(mK + BIGA, f)\n self.model = np.dot(BIGB, W)\n\n # Compute the transit weights and maximum likelihood transit model\n w_trn = med ** 2 * np.concatenate([tm.var_depth * np.dot(\n tm(self.time[m]).reshape(1, -1), W)\n for tm in self.transit_model])\n self.transit_depth = np.array(\n [med * tm.depth + w_trn[i] for i, tm in\n enumerate(self.transit_model)]) / med\n\n # Remove the transit prediction from the model\n self.model -= np.dot(np.hstack([tm(self.time).reshape(-1, 1)\n for tm in self.transit_model]),\n w_trn)\n\n else:\n\n # No transit model to worry about\n W = np.linalg.solve(mK + BIGA, f)\n self.model = np.dot(BIGB, W)\n\n # Subtract the global median\n self.model -= np.nanmedian(self.model)\n\n # Restore the mask\n if self.transit_model is not None:\n self.outmask = outmask\n self.transitmask = transitmask\n\n # Get the CDPP and reset the weights\n self.cdpp_arr = self.get_cdpp_arr()\n self.cdpp = self.get_cdpp()\n self._weights = None", "response": "Compute the joint model for the current set of breakpoints."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef apply_mask(self, x=None):\n '''\n Returns the outlier mask, an array of indices corresponding to the\n non-outliers.\n\n :param numpy.ndarray x: If specified, returns the masked version of \\\n :py:obj:`x` instead. Default :py:obj:`None`\n\n '''\n\n if x is None:\n return np.delete(np.arange(len(self.time)), self.mask)\n else:\n return np.delete(x, self.mask, axis=0)", "response": "Returns the outlier mask an array of indices corresponding to the\n non - outliers."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_chunk(self, b, x=None, pad=True):\n '''\n Returns the indices corresponding to a given light curve chunk.\n\n :param int b: The index of the chunk to return\n :param numpy.ndarray x: If specified, applies the mask to array \\\n :py:obj:`x`. Default :py:obj:`None`\n\n '''\n\n M = np.arange(len(self.time))\n if b > 0:\n res = M[(M > self.breakpoints[b - 1] - int(pad) * self.bpad)\n & (M <= self.breakpoints[b] + int(pad) * self.bpad)]\n else:\n res = M[M <= self.breakpoints[b] + int(pad) * self.bpad]\n if x is None:\n return res\n else:\n return x[res]", "response": "Returns the indices corresponding to a given light curve chunk."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_weights(self):\n '''\n Computes the PLD weights vector :py:obj:`w`.\n\n ..warning :: Deprecated and not thoroughly tested.\n\n '''\n\n log.info(\"Computing PLD weights...\")\n\n # Loop over all chunks\n weights = [None for i in range(len(self.breakpoints))]\n for b, brkpt in enumerate(self.breakpoints):\n\n # Masks for current chunk\n m = self.get_masked_chunk(b)\n c = self.get_chunk(b)\n\n # This block of the masked covariance matrix\n _mK = GetCovariance(self.kernel, self.kernel_params,\n self.time[m], self.fraw_err[m])\n\n # This chunk of the normalized flux\n f = self.fraw[m] - np.nanmedian(self.fraw)\n\n # Loop over all orders\n _A = [None for i in range(self.pld_order)]\n for n in range(self.pld_order):\n if self.lam_idx >= n:\n X = self.X(n, m)\n _A[n] = np.dot(X, X.T)\n del X\n\n # Compute the weights\n A = np.sum([l * a for l, a in zip(self.lam[b], _A)\n if l is not None], axis=0)\n W = np.linalg.solve(_mK + A, f)\n weights[b] = [l * np.dot(self.X(n, m).T, W)\n for n, l in enumerate(self.lam[b]) if l is not None]\n\n self._weights = weights", "response": "Computes the PLD weights vector for all breakpoints."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_cdpp_arr(self, flux=None):\n '''\n Returns the CDPP value in *ppm* for each of the\n chunks in the light curve.\n\n '''\n\n if flux is None:\n flux = self.flux\n return np.array([self._mission.CDPP(flux[self.get_masked_chunk(b)],\n cadence=self.cadence)\n for b, _ in enumerate(self.breakpoints)])", "response": "Returns the CDPP value in ppm for each of the logarithmic chunks in the light curve."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_cdpp(self, flux=None):\n '''\n Returns the scalar CDPP for the light curve.\n\n '''\n\n if flux is None:\n flux = self.flux\n return self._mission.CDPP(self.apply_mask(flux), cadence=self.cadence)", "response": "Returns the scalar CDPP for the light curve."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef plot_aperture(self, axes, labelsize=8):\n '''\n Plots the aperture and the pixel images at the beginning, middle,\n and end of the time series. Also plots a high resolution image of\n the target, if available.\n\n '''\n\n log.info('Plotting the aperture...')\n\n # Get colormap\n plasma = pl.get_cmap('plasma')\n plasma.set_bad(alpha=0)\n\n # Get aperture contour\n def PadWithZeros(vector, pad_width, iaxis, kwargs):\n vector[:pad_width[0]] = 0\n vector[-pad_width[1]:] = 0\n return vector\n ny, nx = self.pixel_images[0].shape\n contour = np.zeros((ny, nx))\n contour[np.where(self.aperture)] = 1\n contour = np.lib.pad(contour, 1, PadWithZeros)\n highres = zoom(contour, 100, order=0, mode='nearest')\n extent = np.array([-1, nx, -1, ny])\n\n # Plot first, mid, and last TPF image\n title = ['start', 'mid', 'end']\n for i, image in enumerate(self.pixel_images):\n ax = axes[i]\n ax.imshow(image, aspect='auto',\n interpolation='nearest', cmap=plasma)\n ax.contour(highres, levels=[0.5], extent=extent,\n origin='lower', colors='r', linewidths=1)\n\n # Check for saturated columns\n for x in range(self.aperture.shape[0]):\n for y in range(self.aperture.shape[1]):\n if self.aperture[x][y] == AP_SATURATED_PIXEL:\n ax.fill([y - 0.5, y + 0.5, y + 0.5, y - 0.5],\n [x - 0.5, x - 0.5, x + 0.5, x + 0.5],\n fill=False, hatch='xxxxx', color='r', lw=0)\n\n ax.axis('off')\n ax.set_xlim(-0.7, nx - 0.3)\n ax.set_ylim(-0.7, ny - 0.3)\n ax.annotate(title[i], xy=(0.5, 0.975), xycoords='axes fraction',\n ha='center', va='top', size=labelsize, color='w')\n if i == 1:\n for source in self.nearby:\n ax.annotate('%.1f' % source['mag'],\n xy=(source['x'] - source['x0'],\n source['y'] - source['y0']),\n ha='center', va='center', size=labelsize - 2,\n color='w', fontweight='bold')\n\n # Plot hi res image\n if self.hires is not None:\n ax = axes[-1]\n ax.imshow(self.hires, aspect='auto',\n extent=(-0.5, nx - 0.5, -0.5, ny - 0.5),\n interpolation='bicubic', cmap=plasma)\n ax.contour(highres, levels=[0.5], extent=extent,\n origin='lower', colors='r', linewidths=1)\n ax.axis('off')\n ax.set_xlim(-0.7, nx - 0.3)\n ax.set_ylim(-0.7, ny - 0.3)\n ax.annotate('hires', xy=(0.5, 0.975), xycoords='axes fraction',\n ha='center', va='top', size=labelsize, color='w')\n else:\n ax = axes[-1]\n ax.axis('off')", "response": "Plots the aperture and the pixel images at the beginning middle and end of the time series."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef overfit(self, tau=None, plot=True, clobber=False, w=9, **kwargs):\n fname = os.path.join(self.dir, self.name + '_overfit.npz')\n figname = os.path.join(self.dir, self.name)\n\n # Compute\n if not os.path.exists(fname) or clobber:\n\n # Baseline\n med = np.nanmedian(self.fraw)\n\n # Default transit model\n if tau is None:\n tau = TransitShape(dur=0.1)\n\n # The overfitting metrics\n O1 = [None for brkpt in self.breakpoints]\n O2 = [None for brkpt in self.breakpoints]\n O3 = [None for brkpt in self.breakpoints]\n O4 = [None for brkpt in self.breakpoints]\n O5 = [None for brkpt in self.breakpoints]\n\n # Loop over all chunks\n for b, brkpt in enumerate(self.breakpoints):\n\n # Masks for current chunk\n m = self.get_masked_chunk(b, pad=False)\n time = self.time[m]\n ferr = self.fraw_err[m] / med\n y = self.fraw[m] / med - 1\n\n # The metrics we're computing here\n O1[b] = np.zeros(len(y)) * np.nan\n O2[b] = np.zeros(len(y)) * np.nan\n O3[b] = np.zeros(len(y)) * np.nan\n O4[b] = np.zeros(len(y)) * np.nan\n O5[b] = np.zeros(len(y)) * np.nan\n\n # Compute the astrophysical covariance and its inverse\n log.info(\"Computing the covariance...\")\n if self.kernel == 'Basic':\n wh, am, ta = self.kernel_params\n wh /= med\n am /= med\n kernel_params = [wh, am, ta]\n elif self.kernel == 'QuasiPeriodic':\n wh, am, ga, pe = self.kernel_params\n wh /= med\n am /= med\n kernel_params = [wh, am, ga, pe]\n K = GetCovariance(self.kernel, kernel_params, time, ferr)\n Kinv = cho_solve((cholesky(K), False), np.eye(len(time)))\n\n # Loop over all orders\n log.info(\"Computing some large matrices...\")\n X = [None for n in range(self.pld_order)]\n XL = [None for n in range(self.pld_order)]\n XLX = [None for n in range(self.pld_order)]\n for n in range(self.pld_order):\n if (self.lam_idx >= n) and (self.lam[b][n] is not None):\n X[n] = self.X(n, m, **kwargs)\n XL[n] = (self.lam[b][n] / med ** 2) * X[n]\n XLX[n] = np.dot(XL[n], X[n].T)\n X = np.hstack(X)\n XL = np.hstack(XL)\n XLX = np.sum(XLX, axis=0)\n\n # The full covariance\n C = XLX + K\n\n # The unmasked linear problem\n log.info(\"Solving the unmasked linear problem...\")\n m = np.dot(XLX, np.linalg.solve(C, y))\n m -= np.nanmedian(m)\n f = y - m\n R = np.linalg.solve(C, XLX.T).T\n\n # The masked linear problem\n log.info(\"Solving the masked linear problem...\")\n A = MaskSolve(C, y, w=w)\n\n # Now loop through and compute the metric\n log.info(\"Computing the overfitting metrics...\")\n for n in prange(len(y)):\n\n #\n # *** Unmasked overfitting metric ***\n #\n\n # Evaluate the sparse transit model\n TAU = tau(time, t0=time[n])\n i = np.where(TAU < 0)[0]\n TAU = TAU.reshape(-1, 1)\n\n # Fast sparse algebra\n AA = np.dot(np.dot(TAU[i].T, Kinv[i, :][:, i]), TAU[i])\n BB = np.dot(TAU[i].T, Kinv[i, :])\n CC = TAU - np.dot(R[:, i], TAU[i])\n O1[b][n] = AA\n O2[b][n] = np.dot(BB, CC)\n O3[b][n] = np.dot(BB, f)\n O4[b][n] = np.dot(BB, y)\n\n #\n # *** Masked overfitting metric ***\n #\n\n # The current mask and mask centerpoint\n mask = np.arange(n, n + w)\n j = n + (w + 1) // 2 - 1\n if j >= len(y) - w:\n continue\n\n # The regularized design matrix\n # This is the same as\n # XLmX[:, n - 1] = \\\n # np.dot(XL, np.delete(X, mask, axis=0).T)[:, n - 1]\n if n == 0:\n XLmX = np.dot(XL, np.delete(X, mask, axis=0).T)\n else:\n XLmX[:, n - 1] = np.dot(XL, X[n - 1, :].T)\n\n # The linear solution to this step\n m = np.dot(XLmX, A[n])\n\n # Evaluate the sparse transit model\n TAU = tau(time, t0=time[j])\n i = np.where(TAU < 0)[0]\n TAU = TAU[i].reshape(-1, 1)\n\n # Dot the transit model in\n den = np.dot(np.dot(TAU.T, Kinv[i, :][:, i]), TAU)\n num = np.dot(TAU.T, Kinv[i, :])\n\n # Compute the overfitting metric\n # Divide this number by a depth\n # to get the overfitting for that\n # particular depth.\n O5[b][j] = -np.dot(num, y - m) / den\n\n # Save!\n np.savez(fname, O1=O1, O2=O2, O3=O3, O4=O4, O5=O5)\n\n else:\n\n data = np.load(fname)\n O1 = data['O1']\n O2 = data['O2']\n O3 = data['O3']\n O4 = data['O4']\n O5 = data['O5']\n\n # Plot\n if plot and (clobber or not os.path.exists(figname + '_overfit.pdf')):\n\n log.info(\"Plotting the overfitting metrics...\")\n\n # Masked time array\n time = self.apply_mask(self.time)\n\n # Plot the final corrected light curve\n ovr = OVERFIT()\n self.plot_info(ovr)\n\n # Loop over the two metrics\n for kind, axes, axesh in zip(['unmasked', 'masked'],\n [ovr.axes1, ovr.axes2],\n [ovr.axes1h, ovr.axes2h]):\n\n # Loop over three depths\n for depth, ax, axh in zip([0.01, 0.001, 0.0001], axes, axesh):\n\n # Get the metric\n if kind == 'unmasked':\n metric = 1 - (np.hstack(O2) +\n np.hstack(O3) / depth) / np.hstack(O1)\n color = 'r'\n elif kind == 'masked':\n metric = np.hstack(O5) / depth\n color = 'b'\n else:\n raise ValueError(\"Invalid metric.\")\n\n # Median and median absolute deviation\n med = np.nanmedian(metric)\n mad = np.nanmedian(np.abs(metric - med))\n\n # Plot the metric as a function of time\n ax.plot(time, metric, 'k.', alpha=0.5, ms=2)\n ax.plot(time, metric, 'k-', alpha=0.1, lw=0.5)\n ylim = (-0.2, 1.0)\n ax.margins(0, None)\n ax.axhline(0, color='k', lw=1, alpha=0.5)\n ax.set_ylim(*ylim)\n if kind == 'masked' and depth == 0.0001:\n ax.set_xlabel('Time (days)', fontsize=14)\n else:\n ax.set_xticklabels([])\n\n # Plot the histogram\n rng = (max(ylim[0], np.nanmin(metric)),\n min(ylim[1], np.nanmax(metric)))\n axh.hist(metric, bins=30, range=rng,\n orientation=\"horizontal\",\n histtype=\"step\", fill=False, color='k')\n axh.axhline(med, color=color, ls='-', lw=1)\n axh.axhspan(med - mad, med + mad, color=color, alpha=0.1)\n axh.axhline(0, color='k', lw=1, alpha=0.5)\n axh.yaxis.tick_right()\n axh.set_ylim(*ax.get_ylim())\n axh.set_xticklabels([])\n\n bbox = dict(fc=\"w\", ec=\"1\", alpha=0.5)\n info = r\"$\\mathrm{med}=%.3f$\" % med + \\\n \"\\n\" + r\"$\\mathrm{mad}=%.3f$\" % mad\n axh.annotate(info, xy=(0.1, 0.925),\n xycoords='axes fraction',\n ha=\"left\", va=\"top\", bbox=bbox, color=color)\n\n bbox = dict(fc=\"w\", ec=\"1\", alpha=0.95)\n ax.annotate(\"%s overfitting metric\" % kind,\n xy=(1-0.035, 0.92),\n xycoords='axes fraction',\n ha='right', va='top',\n bbox=bbox, color=color)\n\n pl.figtext(0.025, 0.77, \"depth = 0.01\", rotation=90,\n ha='left', va='center', fontsize=18)\n pl.figtext(0.025, 0.48, \"depth = 0.001\", rotation=90,\n ha='left', va='center', fontsize=18)\n pl.figtext(0.025, 0.19, \"depth = 0.0001\", rotation=90,\n ha='left', va='center', fontsize=18)\n\n ovr.fig.savefig(figname + '_overfit.pdf')\n log.info(\"Saved plot to %s_overfit.pdf\" % figname)\n pl.close()\n\n return Overfitting(O1, O2, O3, O4, O5, figname + '_overfit.pdf')", "response": "r This routine computes the masked unmasked overfitting metrics for the light curve."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Inject(ID, inj_model='nPLD', t0=None, per=None, dur=0.1, depth=0.001,\n mask=False, trn_win=5, poly_order=3, make_fits=False, **kwargs):\n '''\n Run one of the :py:obj:`everest` models with injected transits and attempt\n to recover the transit depth at the end with a simple linear regression\n with a polynomial baseline. The depth is stored in the\n :py:obj:`inject` attribute of the model (a dictionary) as\n :py:obj:`rec_depth`. A control injection is also performed, in which the\n transits are injected into the de-trended data; the recovered depth in\n the control run is stored in :py:obj:`inject`\n as :py:obj:`rec_depth_control`.\n\n :param int ID: The target id\n :param str inj_model: The name of the :py:obj:`everest` model to run. \\\n Default `\"nPLD\"`\n :param float t0: The transit ephemeris in days. Default is to draw from \\\n the uniform distributon [0., :py:obj:`per`)\n :param float per: The injected planet period in days. Default is to draw \\\n from the uniform distribution [2, 10]\n :param float dur: The transit duration in days. Must be in the range \\\n [0.05, 0.5]. Default 0.1\n :param float depth: The fractional transit depth. Default 0.001\n :param bool mask: Explicitly mask the in-transit cadences when computing \\\n the PLD model? Default :py:obj:`False`\n :param float trn_win: The size of the transit window in units of the \\\n transit duration\n :param int poly_order: The order of the polynomial used to fit the \\\n continuum\n\n '''\n\n # Randomize the planet params\n if per is None:\n a = 3.\n b = 10.\n per = a + (b - a) * np.random.random()\n if t0 is None:\n t0 = per * np.random.random()\n\n # Get the actual class\n _model = eval(inj_model)\n inject = {'t0': t0, 'per': per, 'dur': dur, 'depth': depth, 'mask': mask,\n 'poly_order': poly_order, 'trn_win': trn_win}\n\n # Define the injection class\n class Injection(_model):\n '''\n The :py:obj:`Injection` class is a special subclass of a\n user-selected :py:obj:`everest` model.\n See :py:func:`Inject` for more details.\n\n '''\n\n def __init__(self, *args, **kwargs):\n '''\n\n '''\n\n self.inject = kwargs.pop('inject', None)\n self.parent_class = kwargs.pop('parent_class', None)\n self.kwargs = kwargs\n super(Injection, self).__init__(*args, **kwargs)\n\n @property\n def name(self):\n '''\n\n '''\n\n if self.inject['mask']:\n maskchar = 'M'\n else:\n maskchar = 'U'\n return '%s_Inject_%s%g' % (self.parent_class,\n maskchar, self.inject['depth'])\n\n def load_tpf(self):\n '''\n Loads the target pixel files and injects transits at the pixel level.\n\n '''\n\n # Load the TPF\n super(Injection, self).load_tpf()\n log.info(\"Injecting transits...\")\n\n # Inject the transits into the regular data\n transit_model = Transit(\n self.time, t0=self.inject['t0'], per=self.inject['per'],\n dur=self.inject['dur'], depth=self.inject['depth'])\n for i in range(self.fpix.shape[1]):\n self.fpix[:, i] *= transit_model\n self.fraw = np.sum(self.fpix, axis=1)\n if self.inject['mask']:\n self.transitmask = np.array(list(set(np.concatenate(\n [self.transitmask, np.where(transit_model < 1.)[0]]))),\n dtype=int)\n\n # Update the PLD normalization\n self.get_norm()\n\n def recover_depth(self):\n '''\n Recovers the injected transit depth from the long\n cadence data with a simple LLS solver.\n The results are all stored in the :py:obj:`inject`\n attribute of the model.\n\n '''\n\n # Control run\n transit_model = Transit(\n self.time, t0=self.inject['t0'], per=self.inject['per'],\n dur=self.inject['dur'], depth=self.inject['depth'])\n kwargs = dict(self.kwargs)\n kwargs.update({'clobber': False})\n control = eval(self.parent_class)(\n self.ID, is_parent=True, **kwargs)\n control.fraw *= transit_model\n\n # Get params\n log.info(\"Recovering transit depth...\")\n t0 = self.inject['t0']\n per = self.inject['per']\n dur = self.inject['dur']\n depth = self.inject['depth']\n trn_win = self.inject['trn_win']\n poly_order = self.inject['poly_order']\n\n for run, tag in zip([self, control], ['', '_control']):\n\n # Compute the model\n mask = np.array(\n list(set(np.concatenate([run.badmask, run.nanmask]))),\n dtype=int)\n flux = np.delete(run.flux / np.nanmedian(run.flux), mask)\n time = np.delete(run.time, mask)\n transit_model = (Transit(time, t0=t0, per=per,\n dur=dur, depth=depth) - 1) / depth\n\n # Count the transits\n t0 += np.ceil((time[0] - dur - t0) / per) * per\n ttimes0 = np.arange(t0, time[-1] + dur, per)\n tinds = []\n for tt in ttimes0:\n # Get indices for this chunk\n inds = np.where(np.abs(time - tt) < trn_win * dur / 2.)[0]\n # Ensure there's a transit in this chunk, and that\n # there are enough points for the polynomial fit\n if np.any(transit_model[inds] < 0.) and \\\n len(inds) > poly_order:\n tinds.append(inds)\n\n # Our design matrix\n sz = (poly_order + 1) * len(tinds)\n X = np.empty((0, 1 + sz), dtype=float)\n Y = np.array([], dtype=float)\n T = np.array([], dtype=float)\n\n # Loop over all transits\n for i, inds in enumerate(tinds):\n # Get the transit model\n trnvec = transit_model[inds].reshape(-1, 1)\n # Normalize the time array\n t = time[inds]\n t = (t - t[0]) / (t[-1] - t[0])\n # Cumulative arrays\n T = np.append(T, time[inds])\n Y = np.append(Y, flux[inds])\n # Polynomial vector\n polyvec = np.array(\n [t ** o for o in range(0, poly_order + 1)]).T\n # Update the design matrix with this chunk\n lzeros = np.zeros((len(t), i * (poly_order + 1)))\n rzeros = np.zeros(\n (len(t), sz - (i + 1) * (poly_order + 1)))\n chunk = np.hstack((trnvec, lzeros, polyvec, rzeros))\n X = np.vstack((X, chunk))\n\n # Get the relative depth\n A = np.dot(X.T, X)\n B = np.dot(X.T, Y)\n C = np.linalg.solve(A, B)\n rec_depth = C[0]\n\n # Get the uncertainties\n sig = 1.4826 * \\\n np.nanmedian(np.abs(flux - np.nanmedian(flux))\n ) / np.nanmedian(flux)\n cov = sig ** 2 * np.linalg.solve(A, np.eye(A.shape[0]))\n err = np.sqrt(np.diag(cov))\n rec_depth_err = err[0]\n\n # Store the results\n self.inject.update(\n {'rec_depth%s' % tag: rec_depth,\n 'rec_depth_err%s' % tag: rec_depth_err})\n\n # Store the detrended, folded data\n D = (Y - np.dot(C[1:], X[:, 1:].T) +\n np.nanmedian(Y)) / np.nanmedian(Y)\n T = (T - t0 - per / 2.) % per - per / 2.\n self.inject.update(\n {'fold_time%s' % tag: T, 'fold_flux%s' % tag: D})\n\n def plot_final(self, ax):\n '''\n Plots the injection recovery results.\n\n '''\n\n from mpl_toolkits.axes_grid.inset_locator import inset_axes\n ax.axis('off')\n ax1 = inset_axes(ax, width=\"47%\", height=\"100%\", loc=6)\n ax2 = inset_axes(ax, width=\"47%\", height=\"100%\", loc=7)\n\n # Plot the recovered folded transits\n ax1.plot(self.inject['fold_time'],\n self.inject['fold_flux'], 'k.', alpha=0.3)\n x = np.linspace(np.min(self.inject['fold_time']), np.max(\n self.inject['fold_time']), 500)\n try:\n y = Transit(\n x, t0=0., per=self.inject['per'], dur=self.inject['dur'],\n depth=self.inject['rec_depth'])\n except:\n # Log the error, and carry on\n exctype, value, tb = sys.exc_info()\n for line in traceback.format_exception_only(exctype, value):\n l = line.replace('\\n', '')\n log.error(l)\n y = np.ones_like(x) * np.nan\n ax1.plot(x, y, 'r-')\n ax1.annotate('INJECTED', xy=(0.98, 0.025),\n xycoords='axes fraction',\n ha='right', va='bottom', fontsize=10, alpha=0.5,\n fontweight='bold')\n ax1.annotate('True depth:\\nRecovered depth:',\n xy=(0.02, 0.025),\n xycoords='axes fraction',\n ha='left', va='bottom', fontsize=6, color='r')\n ax1.annotate('%.6f\\n%.6f' % (self.inject['depth'],\n self.inject['rec_depth']),\n xy=(0.4, 0.025),\n xycoords='axes fraction',\n ha='left', va='bottom', fontsize=6, color='r')\n ax1.margins(0, None)\n ax1.ticklabel_format(useOffset=False)\n\n # Plot the recovered folded transits (control)\n ax2.plot(self.inject['fold_time_control'],\n self.inject['fold_flux_control'], 'k.', alpha=0.3)\n x = np.linspace(np.min(self.inject['fold_time_control']), np.max(\n self.inject['fold_time_control']), 500)\n try:\n y = Transit(\n x, t0=0., per=self.inject['per'], dur=self.inject['dur'],\n depth=self.inject['rec_depth_control'])\n except:\n # Log the error, and carry on\n exctype, value, tb = sys.exc_info()\n for line in traceback.format_exception_only(exctype, value):\n l = line.replace('\\n', '')\n log.error(l)\n y = np.ones_like(x) * np.nan\n ax2.plot(x, y, 'r-')\n ax2.annotate('CONTROL', xy=(0.98, 0.025), xycoords='axes fraction',\n ha='right', va='bottom', fontsize=10, alpha=0.5,\n fontweight='bold')\n ax2.annotate('True depth:\\nRecovered depth:',\n xy=(0.02, 0.025),\n xycoords='axes fraction',\n ha='left', va='bottom', fontsize=6, color='r')\n ax2.annotate('%.6f\\n%.6f' % (self.inject['depth'],\n self.inject['rec_depth_control']),\n xy=(0.4, 0.025),\n xycoords='axes fraction',\n ha='left', va='bottom', fontsize=6, color='r')\n ax2.margins(0, None)\n ax2.ticklabel_format(useOffset=False)\n N = int(0.995 * len(self.inject['fold_flux_control']))\n hi, lo = self.inject['fold_flux_control'][np.argsort(\n self.inject['fold_flux_control'])][[N, -N]]\n fsort = self.inject['fold_flux_control'][np.argsort(\n self.inject['fold_flux_control'])]\n pad = (hi - lo) * 0.2\n ylim = (lo - 2 * pad, hi + pad)\n ax2.set_ylim(ylim)\n ax1.set_ylim(ylim)\n\n ax2.set_yticklabels([])\n for tick in ax1.get_xticklabels() + ax1.get_yticklabels() + \\\n ax2.get_xticklabels():\n tick.set_fontsize(5)\n\n def finalize(self):\n '''\n Calls the depth recovery routine at the end\n of the de-trending step.\n\n '''\n\n super(Injection, self).finalize()\n self.recover_depth()\n\n return Injection(ID, inject=inject, parent_class=inj_model,\n make_fits=make_fits, **kwargs)", "response": "This function creates a new state machine with injected transits and control runs."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a list of objects of the same type.", "response": "def object(self, session):\r\n '''Instance of :attr:`model_type` with id :attr:`object_id`.'''\r\n if not hasattr(self, '_object'):\r\n pkname = self.model_type._meta.pkname()\r\n query = session.query(self.model_type).filter(**{pkname:\r\n self.object_id})\r\n return query.items(callback=self.__set_object)\r\n else:\r\n return self._object"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetData(ID, season = None, cadence = 'lc', clobber = False, delete_raw = False, \n aperture_name = None, saturated_aperture_name = None,\n max_pixels = None, download_only = False, saturation_tolerance = None, \n bad_bits = None, **kwargs):\n '''\n Returns a :py:obj:`DataContainer` instance with the raw data for the target.\n \n :param int ID: The target ID number\n :param int season: The observing season. Default :py:obj:`None`\n :param str cadence: The light curve cadence. Default `lc`\n :param bool clobber: Overwrite existing files? Default :py:obj:`False`\n :param bool delete_raw: Delete the FITS TPF after processing it? Default :py:obj:`False`\n :param str aperture_name: The name of the aperture to use. Select `custom` to call \\\n :py:func:`GetCustomAperture`. Default :py:obj:`None`\n :param str saturated_aperture_name: The name of the aperture to use if the target is \\\n saturated. Default :py:obj:`None`\n :param int max_pixels: Maximum number of pixels in the TPF. Default :py:obj:`None`\n :param bool download_only: Download raw TPF and return? Default :py:obj:`False`\n :param float saturation_tolerance: Target is considered saturated if flux is within \\\n this fraction of the pixel well depth. Default :py:obj:`None`\n :param array_like bad_bits: Flagged :py:obj`QUALITY` bits to consider outliers when \\\n computing the model. Default :py:obj:`None`\n \n '''\n \n raise NotImplementedError('This mission is not yet supported.')", "response": "Retrieves the raw data for the target."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning neighbors from the specified model.", "response": "def GetNeighbors(ID, model = None, neighbors = None, mag_range = None, \n cdpp_range = None, aperture_name = None, \n cadence = 'lc', **kwargs):\n '''\n Return `neighbors` random bright stars on the same module as `EPIC`.\n \n :param int ID: The target ID number\n :param str model: The :py:obj:`everest` model name. Only used when imposing CDPP bounds. Default :py:obj:`None`\n :param int neighbors: Number of neighbors to return. Default None\n :param str aperture_name: The name of the aperture to use. Select `custom` to call \\\n :py:func:`GetCustomAperture`. Default :py:obj:`None`\n :param str cadence: The light curve cadence. Default `lc`\n :param tuple mag_range: (`low`, `high`) values for the Kepler magnitude. Default :py:obj:`None`\n :param tuple cdpp_range: (`low`, `high`) values for the de-trended CDPP. Default :py:obj:`None`\n \n '''\n \n raise NotImplementedError('This mission is not yet supported.')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the time and flux for a given EPIC ID and a given pipeline name.", "response": "def get(ID, pipeline='everest2', campaign=None):\n '''\n Returns the `time` and `flux` for a given EPIC `ID` and\n a given `pipeline` name.\n\n '''\n\n log.info('Downloading %s light curve for %d...' % (pipeline, ID))\n\n # Dev version hack\n if EVEREST_DEV:\n if pipeline.lower() == 'everest2' or pipeline.lower() == 'k2sff':\n from . import Season, TargetDirectory, FITSFile\n if campaign is None:\n campaign = Season(ID)\n fits = os.path.join(TargetDirectory(\n ID, campaign), FITSFile(ID, campaign))\n newdir = os.path.join(KPLR_ROOT, \"data\", \"everest\", str(ID))\n if not os.path.exists(newdir):\n os.makedirs(newdir)\n if os.path.exists(fits):\n shutil.copy(fits, newdir)\n\n if pipeline.lower() == 'everest2':\n s = k2plr.EVEREST(ID, version=2, sci_campaign=campaign)\n time = s.time\n flux = s.flux\n elif pipeline.lower() == 'everest1':\n s = k2plr.EVEREST(ID, version=1, sci_campaign=campaign)\n time = s.time\n flux = s.flux\n elif pipeline.lower() == 'k2sff':\n s = k2plr.K2SFF(ID, sci_campaign=campaign)\n time = s.time\n flux = s.fcor\n # Normalize to the median flux\n s = k2plr.EVEREST(ID, version=2, sci_campaign=campaign)\n flux *= np.nanmedian(s.flux)\n elif pipeline.lower() == 'k2sc':\n s = k2plr.K2SC(ID, sci_campaign=campaign)\n time = s.time\n flux = s.pdcflux\n elif pipeline.lower() == 'raw':\n s = k2plr.EVEREST(ID, version=2, raw=True, sci_campaign=campaign)\n time = s.time\n flux = s.flux\n else:\n raise ValueError('Invalid pipeline: `%s`.' % pipeline)\n\n return time, flux"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot(ID, pipeline='everest2', show=True, campaign=None):\n '''\n Plots the de-trended flux for the given EPIC `ID` and for\n the specified `pipeline`.\n\n '''\n\n # Get the data\n time, flux = get(ID, pipeline=pipeline, campaign=campaign)\n\n # Remove nans\n mask = np.where(np.isnan(flux))[0]\n time = np.delete(time, mask)\n flux = np.delete(flux, mask)\n\n # Plot it\n fig, ax = pl.subplots(1, figsize=(10, 4))\n fig.subplots_adjust(bottom=0.15)\n ax.plot(time, flux, \"k.\", markersize=3, alpha=0.5)\n\n # Axis limits\n N = int(0.995 * len(flux))\n hi, lo = flux[np.argsort(flux)][[N, -N]]\n pad = (hi - lo) * 0.1\n ylim = (lo - pad, hi + pad)\n ax.set_ylim(ylim)\n\n # Show the CDPP\n from .k2 import CDPP\n ax.annotate('%.2f ppm' % CDPP(flux),\n xy=(0.98, 0.975), xycoords='axes fraction',\n ha='right', va='top', fontsize=12, color='r', zorder=99)\n\n # Appearance\n ax.margins(0, None)\n ax.set_xlabel(\"Time (BJD - 2454833)\", fontsize=16)\n ax.set_ylabel(\"%s Flux\" % pipeline.upper(), fontsize=16)\n fig.canvas.set_window_title(\"%s: EPIC %d\" % (pipeline.upper(), ID))\n\n if show:\n pl.show()\n pl.close()\n else:\n return fig, ax", "response": "Plots the de - trended flux for the given EPIC ID and for\n the specified pipeline."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_cdpp(campaign, pipeline='everest2'):\n '''\n Computes the CDPP for a given `campaign` and a given `pipeline`.\n Stores the results in a file under \"/missions/k2/tables/\".\n\n '''\n\n # Imports\n from .k2 import CDPP\n from .utils import GetK2Campaign\n\n # Check pipeline\n assert pipeline.lower() in Pipelines, 'Invalid pipeline: `%s`.' % pipeline\n\n # Create file if it doesn't exist\n file = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' % (int(campaign), pipeline))\n if not os.path.exists(file):\n open(file, 'a').close()\n\n # Get all EPIC stars\n stars = GetK2Campaign(campaign, epics_only=True)\n nstars = len(stars)\n\n # Remove ones we've done\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n done = np.loadtxt(file, dtype=float)\n if len(done):\n done = [int(s) for s in done[:, 0]]\n stars = list(set(stars) - set(done))\n n = len(done) + 1\n\n # Open the output file\n with open(file, 'a', 1) as outfile:\n\n # Loop over all to get the CDPP\n for EPIC in stars:\n\n # Progress\n sys.stdout.write('\\rRunning target %d/%d...' % (n, nstars))\n sys.stdout.flush()\n n += 1\n\n # Get the CDPP\n try:\n _, flux = get(EPIC, pipeline=pipeline, campaign=campaign)\n mask = np.where(np.isnan(flux))[0]\n flux = np.delete(flux, mask)\n cdpp = CDPP(flux)\n except (urllib.error.HTTPError, urllib.error.URLError,\n TypeError, ValueError, IndexError):\n print(\"{:>09d} {:>15.3f}\".format(EPIC, 0), file=outfile)\n continue\n\n # Log to file\n print(\"{:>09d} {:>15.3f}\".format(EPIC, cdpp), file=outfile)", "response": "Compute the CDPP for a given campaign and pipeline."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompute the number of outliers for a given campaign and a given pipeline.", "response": "def get_outliers(campaign, pipeline='everest2', sigma=5):\n '''\n Computes the number of outliers for a given `campaign`\n and a given `pipeline`.\n Stores the results in a file under \"/missions/k2/tables/\".\n\n :param int sigma: The sigma level at which to clip outliers. Default 5\n\n '''\n\n # Imports\n from .utils import GetK2Campaign\n client = k2plr.API()\n\n # Check pipeline\n assert pipeline.lower() in Pipelines, 'Invalid pipeline: `%s`.' % pipeline\n\n # Create file if it doesn't exist\n file = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.out' % (int(campaign), pipeline))\n if not os.path.exists(file):\n open(file, 'a').close()\n\n # Get all EPIC stars\n stars = GetK2Campaign(campaign, epics_only=True)\n nstars = len(stars)\n\n # Remove ones we've done\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n done = np.loadtxt(file, dtype=float)\n if len(done):\n done = [int(s) for s in done[:, 0]]\n stars = list(set(stars) - set(done))\n n = len(done) + 1\n\n # Open the output file\n with open(file, 'a', 1) as outfile:\n\n # Loop over all to get the CDPP\n for EPIC in stars:\n\n # Progress\n sys.stdout.write('\\rRunning target %d/%d...' % (n, nstars))\n sys.stdout.flush()\n n += 1\n\n # Get the number of outliers\n try:\n time, flux = get(EPIC, pipeline=pipeline, campaign=campaign)\n\n # Get the raw K2 data\n tpf = os.path.join(KPLR_ROOT, \"data\", \"k2\",\n \"target_pixel_files\",\n \"%09d\" % EPIC,\n \"ktwo%09d-c%02d_lpd-targ.fits.gz\"\n % (EPIC, campaign))\n if not os.path.exists(tpf):\n client.k2_star(EPIC).get_target_pixel_files(fetch=True)\n with pyfits.open(tpf) as f:\n k2_qual = np.array(f[1].data.field('QUALITY'), dtype=int)\n k2_time = np.array(\n f[1].data.field('TIME'), dtype='float64')\n mask = []\n for b in [1, 2, 3, 4, 5, 6, 7, 8, 9,\n 11, 12, 13, 14, 16, 17]:\n mask += list(np.where(k2_qual & 2 ** (b - 1))[0])\n mask = np.array(sorted(list(set(mask))))\n\n # Fill in missing cadences, if any\n tol = 0.005\n if not ((len(time) == len(k2_time)) and (np.abs(time[0]\n - k2_time[0]) < tol) and (np.abs(time[-1]\n - k2_time[-1]) < tol)):\n ftmp = np.zeros_like(k2_time) * np.nan\n j = 0\n for i, t in enumerate(k2_time):\n if np.abs(time[j] - t) < tol:\n ftmp[i] = flux[j]\n j += 1\n if j == len(time) - 1:\n break\n flux = ftmp\n\n # Remove flagged cadences\n flux = np.delete(flux, mask)\n\n # Remove nans\n nanmask = np.where(np.isnan(flux))[0]\n flux = np.delete(flux, nanmask)\n\n # Iterative sigma clipping\n inds = np.array([], dtype=int)\n m = 1\n while len(inds) < m:\n m = len(inds)\n f = SavGol(np.delete(flux, inds))\n med = np.nanmedian(f)\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n inds = np.append(inds, np.where(\n (f > med + sigma * MAD) | (f < med - sigma * MAD))[0])\n nout = len(inds)\n ntot = len(flux)\n\n except (urllib.error.HTTPError, urllib.error.URLError,\n TypeError, ValueError, IndexError):\n print(\"{:>09d} {:>5d} {:>5d}\".format(\n EPIC, -1, -1), file=outfile)\n continue\n\n # Log to file\n print(\"{:>09d} {:>5d} {:>5d}\".format(\n EPIC, nout, ntot), file=outfile)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the name of the current Detrender subclass.", "response": "def name(self):\n '''\n Returns the name of the current :py:class:`Detrender` subclass.\n\n '''\n\n if self.cadence == 'lc':\n return self.__class__.__name__\n else:\n return '%s.sc' % self.__class__.__name__"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cv_compute(self, b, A, B, C, mK, f, m1, m2):\n '''\n Compute the model (cross-validation step only) for chunk :py:obj:`b`.\n\n '''\n\n A = np.sum([l * a for l, a in zip(self.lam[b], A)\n if l is not None], axis=0)\n B = np.sum([l * b for l, b in zip(self.lam[b], B)\n if l is not None], axis=0)\n W = np.linalg.solve(mK + A + C, f)\n if self.transit_model is None:\n model = np.dot(B, W)\n else:\n w_pld = np.concatenate([l * np.dot(self.X(n, m2).T, W)\n for n, l in enumerate(self.lam[b])\n if l is not None])\n model = np.dot(np.hstack(\n [self.X(n, m1) for n, l in enumerate(self.lam[b])\n if l is not None]), w_pld)\n model -= np.nanmedian(model)\n\n return model", "response": "Compute the model for a single chunk."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a list of outliers for the current instance of the class.", "response": "def get_outliers(self):\n '''\n Performs iterative sigma clipping to get outliers.\n\n '''\n\n log.info(\"Clipping outliers...\")\n log.info('Iter %d/%d: %d outliers' %\n (0, self.oiter, len(self.outmask)))\n\n def M(x): return np.delete(x, np.concatenate(\n [self.nanmask, self.badmask, self.transitmask]), axis=0)\n t = M(self.time)\n outmask = [np.array([-1]), np.array(self.outmask)]\n\n # Loop as long as the last two outlier arrays aren't equal\n while not np.array_equal(outmask[-2], outmask[-1]):\n\n # Check if we've done this too many times\n if len(outmask) - 1 > self.oiter:\n log.error('Maximum number of iterations in ' +\n '``get_outliers()`` exceeded. Skipping...')\n break\n\n # Check if we're going in circles\n if np.any([np.array_equal(outmask[-1], i) for i in outmask[:-1]]):\n log.error('Function ``get_outliers()`` ' +\n 'is going in circles. Skipping...')\n break\n\n # Compute the model to get the flux\n self.compute()\n\n # Get the outliers\n f = SavGol(M(self.flux))\n med = np.nanmedian(f)\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n inds = np.where((f > med + self.osigma * MAD) |\n (f < med - self.osigma * MAD))[0]\n\n # Project onto unmasked time array\n inds = np.array([np.argmax(self.time == t[i]) for i in inds])\n self.outmask = np.array(inds, dtype=int)\n\n # Add them to the running list\n outmask.append(np.array(inds))\n\n # Log\n log.info('Iter %d/%d: %d outliers' %\n (len(outmask) - 2, self.oiter, len(self.outmask)))"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncrossing - validate the current set of curves.", "response": "def cross_validate(self, ax, info=''):\n '''\n Cross-validate to find the optimal value of :py:obj:`lambda`.\n\n :param ax: The current :py:obj:`matplotlib.pyplot` axis instance to \\\n plot the cross-validation results.\n :param str info: The label to show in the bottom right-hand corner \\\n of the plot. Default `''`\n\n '''\n\n # Loop over all chunks\n ax = np.atleast_1d(ax)\n for b, brkpt in enumerate(self.breakpoints):\n\n log.info(\"Cross-validating chunk %d/%d...\" %\n (b + 1, len(self.breakpoints)))\n med_training = np.zeros_like(self.lambda_arr)\n med_validation = np.zeros_like(self.lambda_arr)\n\n # Mask for current chunk\n m = self.get_masked_chunk(b)\n\n # Check that we have enough data\n if len(m) < 3 * self.cdivs:\n self.cdppv_arr[b] = np.nan\n self.lam[b][self.lam_idx] = 0.\n log.info(\n \"Insufficient data to run cross-validation on this chunk.\")\n continue\n\n # Mask transits and outliers\n time = self.time[m]\n flux = self.fraw[m]\n ferr = self.fraw_err[m]\n med = np.nanmedian(flux)\n\n # The precision in the validation set\n validation = [[] for k, _ in enumerate(self.lambda_arr)]\n\n # The precision in the training set\n training = [[] for k, _ in enumerate(self.lambda_arr)]\n\n # Setup the GP\n gp = GP(self.kernel, self.kernel_params, white=False)\n gp.compute(time, ferr)\n\n # The masks\n masks = list(Chunks(np.arange(0, len(time)),\n len(time) // self.cdivs))\n\n # Loop over the different masks\n for i, mask in enumerate(masks):\n\n log.info(\"Section %d/%d...\" % (i + 1, len(masks)))\n\n # Pre-compute (training set)\n pre_t = self.cv_precompute([], b)\n\n # Pre-compute (validation set)\n pre_v = self.cv_precompute(mask, b)\n\n # Iterate over lambda\n for k, lam in enumerate(self.lambda_arr):\n\n # Update the lambda matrix\n self.lam[b][self.lam_idx] = lam\n\n # Training set\n model = self.cv_compute(b, *pre_t)\n training[k].append(\n self.fobj(flux - model, med, time, gp, mask))\n\n # Validation set\n model = self.cv_compute(b, *pre_v)\n validation[k].append(\n self.fobj(flux - model, med, time, gp, mask))\n\n # Finalize\n training = np.array(training)\n validation = np.array(validation)\n for k, _ in enumerate(self.lambda_arr):\n\n # Take the mean\n med_validation[k] = np.nanmean(validation[k])\n med_training[k] = np.nanmean(training[k])\n\n # Compute best model\n i = self.optimize_lambda(validation)\n v_best = med_validation[i]\n t_best = med_training[i]\n self.cdppv_arr[b] = v_best / t_best\n self.lam[b][self.lam_idx] = self.lambda_arr[i]\n log.info(\"Found optimum solution at log(lambda) = %.1f.\" %\n np.log10(self.lam[b][self.lam_idx]))\n\n # Plotting: There's not enough space in the DVS to show the\n # cross-val results for more than three light curve segments.\n if len(self.breakpoints) <= 3:\n\n # Plotting hack: first x tick will be -infty\n lambda_arr = np.array(self.lambda_arr)\n lambda_arr[0] = 10 ** (np.log10(lambda_arr[1]) - 3)\n\n # Plot cross-val\n for n in range(len(masks)):\n ax[b].plot(np.log10(lambda_arr),\n validation[:, n], 'r-', alpha=0.3)\n\n ax[b].plot(np.log10(lambda_arr),\n med_training, 'b-', lw=1., alpha=1)\n ax[b].plot(np.log10(lambda_arr),\n med_validation, 'r-', lw=1., alpha=1)\n ax[b].axvline(np.log10(self.lam[b][self.lam_idx]),\n color='k', ls='--', lw=0.75, alpha=0.75)\n ax[b].axhline(v_best, color='k', ls='--', lw=0.75, alpha=0.75)\n ax[b].set_ylabel(r'Scatter (ppm)', fontsize=5)\n hi = np.max(validation[0])\n lo = np.min(training)\n rng = (hi - lo)\n ax[b].set_ylim(lo - 0.15 * rng, hi + 0.15 * rng)\n if rng > 2:\n ax[b].get_yaxis().set_major_formatter(Formatter.CDPP)\n ax[b].get_yaxis().set_major_locator(\n MaxNLocator(4, integer=True))\n elif rng > 0.2:\n ax[b].get_yaxis().set_major_formatter(Formatter.CDPP1F)\n ax[b].get_yaxis().set_major_locator(MaxNLocator(4))\n else:\n ax[b].get_yaxis().set_major_formatter(Formatter.CDPP2F)\n ax[b].get_yaxis().set_major_locator(MaxNLocator(4))\n\n # Fix the x ticks\n xticks = [np.log10(lambda_arr[0])] + list(np.linspace(\n np.log10(lambda_arr[1]), np.log10(lambda_arr[-1]), 6))\n ax[b].set_xticks(xticks)\n ax[b].set_xticklabels(['' for x in xticks])\n pad = 0.01 * \\\n (np.log10(lambda_arr[-1]) - np.log10(lambda_arr[0]))\n ax[b].set_xlim(np.log10(lambda_arr[0]) - pad,\n np.log10(lambda_arr[-1]) + pad)\n ax[b].annotate('%s.%d' % (info, b), xy=(0.02, 0.025),\n xycoords='axes fraction',\n ha='left', va='bottom', fontsize=7, alpha=0.25,\n fontweight='bold')\n\n # Finally, compute the model\n self.compute()\n\n # Tidy up\n if len(ax) == 2:\n ax[0].xaxis.set_ticks_position('top')\n for axis in ax[1:]:\n axis.spines['top'].set_visible(False)\n axis.xaxis.set_ticks_position('bottom')\n\n if len(self.breakpoints) <= 3:\n\n # A hack to mark the first xtick as -infty\n labels = ['%.1f' % x for x in xticks]\n labels[0] = r'$-\\infty$'\n ax[-1].set_xticklabels(labels)\n ax[-1].set_xlabel(r'Log $\\Lambda$', fontsize=5)\n\n else:\n\n # We're just going to plot lambda as a function of chunk number\n bs = np.arange(len(self.breakpoints))\n ax[0].plot(bs + 1, [np.log10(self.lam[b][self.lam_idx])\n for b in bs], 'r.')\n ax[0].plot(bs + 1, [np.log10(self.lam[b][self.lam_idx])\n for b in bs], 'r-', alpha=0.25)\n ax[0].set_ylabel(r'$\\log\\Lambda$', fontsize=5)\n ax[0].margins(0.1, 0.1)\n ax[0].set_xticks(np.arange(1, len(self.breakpoints) + 1))\n ax[0].set_xticklabels([])\n\n # Now plot the CDPP and approximate validation CDPP\n cdpp_arr = self.get_cdpp_arr()\n cdppv_arr = self.cdppv_arr * cdpp_arr\n ax[1].plot(bs + 1, cdpp_arr, 'b.')\n ax[1].plot(bs + 1, cdpp_arr, 'b-', alpha=0.25)\n ax[1].plot(bs + 1, cdppv_arr, 'r.')\n ax[1].plot(bs + 1, cdppv_arr, 'r-', alpha=0.25)\n ax[1].margins(0.1, 0.1)\n ax[1].set_ylabel(r'Scatter (ppm)', fontsize=5)\n ax[1].set_xlabel(r'Chunk', fontsize=5)\n if len(self.breakpoints) < 15:\n ax[1].set_xticks(np.arange(1, len(self.breakpoints) + 1))\n else:\n ax[1].set_xticks(np.arange(1, len(self.breakpoints) + 1, 2))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_ylim(self):\n '''\n Computes the ideal y-axis limits for the light curve plot. Attempts to\n set the limits equal to those of the raw light curve, but if more than\n 1% of the flux lies either above or below these limits, auto-expands\n to include those points. At the end, adds 5% padding to both the\n top and the bottom.\n\n '''\n\n bn = np.array(\n list(set(np.concatenate([self.badmask, self.nanmask]))), dtype=int)\n fraw = np.delete(self.fraw, bn)\n lo, hi = fraw[np.argsort(fraw)][[3, -3]]\n flux = np.delete(self.flux, bn)\n fsort = flux[np.argsort(flux)]\n if fsort[int(0.01 * len(fsort))] < lo:\n lo = fsort[int(0.01 * len(fsort))]\n if fsort[int(0.99 * len(fsort))] > hi:\n hi = fsort[int(0.99 * len(fsort))]\n pad = (hi - lo) * 0.05\n ylim = (lo - pad, hi + pad)\n return ylim", "response": "Computes the ideal y - axis limits for the light curve plot."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nplot the current light curve.", "response": "def plot_lc(self, ax, info_left='', info_right='', color='b'):\n '''\n Plots the current light curve. This is called at several stages to\n plot the de-trending progress as a function of the different\n *PLD* orders.\n\n :param ax: The current :py:obj:`matplotlib.pyplot` axis instance\n :param str info_left: Information to display at the left of the \\\n plot. Default `''`\n :param str info_right: Information to display at the right of the \\\n plot. Default `''`\n :param str color: The color of the data points. Default `'b'`\n\n '''\n\n # Plot\n if (self.cadence == 'lc') or (len(self.time) < 4000):\n ax.plot(self.apply_mask(self.time), self.apply_mask(self.flux),\n ls='none', marker='.', color=color,\n markersize=2, alpha=0.5)\n ax.plot(self.time[self.transitmask], self.flux[self.transitmask],\n ls='none', marker='.', color=color,\n markersize=2, alpha=0.5)\n else:\n ax.plot(self.apply_mask(self.time), self.apply_mask(\n self.flux), ls='none', marker='.', color=color,\n markersize=2, alpha=0.03, zorder=-1)\n ax.plot(self.time[self.transitmask], self.flux[self.transitmask],\n ls='none', marker='.', color=color,\n markersize=2, alpha=0.03, zorder=-1)\n ax.set_rasterization_zorder(0)\n ylim = self.get_ylim()\n\n # Plot the outliers, but not the NaNs\n badmask = [i for i in self.badmask if i not in self.nanmask]\n\n def O1(x): return x[self.outmask]\n\n def O2(x): return x[badmask]\n if self.cadence == 'lc':\n ax.plot(O1(self.time), O1(self.flux), ls='none',\n color=\"#777777\", marker='.', markersize=2, alpha=0.5)\n ax.plot(O2(self.time), O2(self.flux),\n 'r.', markersize=2, alpha=0.25)\n else:\n ax.plot(O1(self.time), O1(self.flux), ls='none', color=\"#777777\",\n marker='.', markersize=2, alpha=0.25, zorder=-1)\n ax.plot(O2(self.time), O2(self.flux), 'r.',\n markersize=2, alpha=0.125, zorder=-1)\n for i in np.where(self.flux < ylim[0])[0]:\n if i in badmask:\n color = \"#ffcccc\"\n elif i in self.outmask:\n color = \"#cccccc\"\n elif i in self.nanmask:\n continue\n else:\n color = \"#ccccff\"\n ax.annotate('', xy=(self.time[i], ylim[0]), xycoords='data',\n xytext=(0, 15), textcoords='offset points',\n arrowprops=dict(arrowstyle=\"-|>\", color=color))\n for i in np.where(self.flux > ylim[1])[0]:\n if i in badmask:\n color = \"#ffcccc\"\n elif i in self.outmask:\n color = \"#cccccc\"\n elif i in self.nanmask:\n continue\n else:\n color = \"#ccccff\"\n ax.annotate('', xy=(self.time[i], ylim[1]), xycoords='data',\n xytext=(0, -15), textcoords='offset points',\n arrowprops=dict(arrowstyle=\"-|>\", color=color))\n\n # Plot the breakpoints\n for brkpt in self.breakpoints[:-1]:\n if len(self.breakpoints) <= 5:\n ax.axvline(self.time[brkpt], color='r', ls='--', alpha=0.5)\n else:\n ax.axvline(self.time[brkpt], color='r', ls='-', alpha=0.025)\n\n # Appearance\n if len(self.cdpp_arr) == 2:\n ax.annotate('%.2f ppm' % self.cdpp_arr[0], xy=(0.02, 0.975),\n xycoords='axes fraction',\n ha='left', va='top', fontsize=10)\n ax.annotate('%.2f ppm' % self.cdpp_arr[1], xy=(0.98, 0.975),\n xycoords='axes fraction',\n ha='right', va='top', fontsize=10)\n elif len(self.cdpp_arr) < 6:\n for n in range(len(self.cdpp_arr)):\n if n > 0:\n x = (self.time[self.breakpoints[n - 1]] - self.time[0]\n ) / (self.time[-1] - self.time[0]) + 0.02\n else:\n x = 0.02\n ax.annotate('%.2f ppm' % self.cdpp_arr[n], xy=(x, 0.975),\n xycoords='axes fraction',\n ha='left', va='top', fontsize=8)\n else:\n ax.annotate('%.2f ppm' % self.cdpp, xy=(0.02, 0.975),\n xycoords='axes fraction',\n ha='left', va='top', fontsize=10)\n ax.annotate(info_right, xy=(0.98, 0.025), xycoords='axes fraction',\n ha='right', va='bottom', fontsize=10, alpha=0.5,\n fontweight='bold')\n ax.annotate(info_left, xy=(0.02, 0.025), xycoords='axes fraction',\n ha='left', va='bottom', fontsize=8)\n ax.set_xlabel(r'Time (%s)' % self._mission.TIMEUNITS, fontsize=5)\n ax.margins(0.01, 0.1)\n ax.set_ylim(*ylim)\n ax.get_yaxis().set_major_formatter(Formatter.Flux)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef plot_final(self, ax):\n '''\n Plots the final de-trended light curve.\n\n '''\n\n # Plot the light curve\n bnmask = np.array(\n list(set(np.concatenate([self.badmask, self.nanmask]))), dtype=int)\n\n def M(x): return np.delete(x, bnmask)\n if (self.cadence == 'lc') or (len(self.time) < 4000):\n ax.plot(M(self.time), M(self.flux), ls='none',\n marker='.', color='k', markersize=2, alpha=0.3)\n else:\n ax.plot(M(self.time), M(self.flux), ls='none', marker='.',\n color='k', markersize=2, alpha=0.03, zorder=-1)\n ax.set_rasterization_zorder(0)\n # Hack: Plot invisible first and last points to ensure\n # the x axis limits are the\n # same in the other plots, where we also plot outliers!\n ax.plot(self.time[0], np.nanmedian(M(self.flux)), marker='.', alpha=0)\n ax.plot(self.time[-1], np.nanmedian(M(self.flux)), marker='.', alpha=0)\n\n # Plot the GP (long cadence only)\n if self.cadence == 'lc':\n gp = GP(self.kernel, self.kernel_params, white=False)\n gp.compute(self.apply_mask(self.time),\n self.apply_mask(self.fraw_err))\n med = np.nanmedian(self.apply_mask(self.flux))\n y, _ = gp.predict(self.apply_mask(self.flux) - med, self.time)\n y += med\n ax.plot(M(self.time), M(y), 'r-', lw=0.5, alpha=0.5)\n\n # Compute the CDPP of the GP-detrended flux\n self.cdppg = self._mission.CDPP(self.apply_mask(\n self.flux - y + med), cadence=self.cadence)\n\n else:\n\n # We're not going to calculate this\n self.cdppg = 0.\n\n # Appearance\n ax.annotate('Final', xy=(0.98, 0.025), xycoords='axes fraction',\n ha='right', va='bottom', fontsize=10, alpha=0.5,\n fontweight='bold')\n ax.margins(0.01, 0.1)\n\n # Get y lims that bound 99% of the flux\n flux = np.delete(self.flux, bnmask)\n N = int(0.995 * len(flux))\n hi, lo = flux[np.argsort(flux)][[N, -N]]\n fsort = flux[np.argsort(flux)]\n pad = (hi - lo) * 0.1\n ylim = (lo - pad, hi + pad)\n ax.set_ylim(ylim)\n ax.get_yaxis().set_major_formatter(Formatter.Flux)", "response": "Plots the final de - trended light curve."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_cbv(self, ax, flux, info, show_cbv=False):\n '''\n Plots the final CBV-corrected light curve.\n\n '''\n\n # Plot the light curve\n bnmask = np.array(\n list(set(np.concatenate([self.badmask, self.nanmask]))), dtype=int)\n\n def M(x): return np.delete(x, bnmask)\n if self.cadence == 'lc':\n ax.plot(M(self.time), M(flux), ls='none', marker='.',\n color='k', markersize=2, alpha=0.45)\n else:\n ax.plot(M(self.time), M(flux), ls='none', marker='.',\n color='k', markersize=2, alpha=0.03, zorder=-1)\n ax.set_rasterization_zorder(0)\n # Hack: Plot invisible first and last points to ensure\n # the x axis limits are the\n # same in the other plots, where we also plot outliers!\n ax.plot(self.time[0], np.nanmedian(M(flux)), marker='.', alpha=0)\n ax.plot(self.time[-1], np.nanmedian(M(flux)), marker='.', alpha=0)\n\n # Show CBV fit?\n if show_cbv:\n ax.plot(self.time, self._mission.FitCBVs(\n self) + np.nanmedian(flux), 'r-', alpha=0.2)\n\n # Appearance\n ax.annotate(info, xy=(0.98, 0.025), xycoords='axes fraction',\n ha='right', va='bottom', fontsize=10, alpha=0.5,\n fontweight='bold')\n ax.margins(0.01, 0.1)\n\n # Get y lims that bound 99% of the flux\n flux = np.delete(flux, bnmask)\n N = int(0.995 * len(flux))\n hi, lo = flux[np.argsort(flux)][[N, -N]]\n fsort = flux[np.argsort(flux)]\n pad = (hi - lo) * 0.2\n ylim = (lo - pad, hi + pad)\n ax.set_ylim(ylim)\n ax.get_yaxis().set_major_formatter(Formatter.Flux)\n ax.set_xlabel(r'Time (%s)' % self._mission.TIMEUNITS, fontsize=9)\n for tick in ax.get_xticklabels() + ax.get_yticklabels():\n tick.set_fontsize(7)", "response": "Plots the final CBV - correcteded light curve."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_tpf(self):\n '''\n Loads the target pixel file.\n\n '''\n\n if not self.loaded:\n if self._data is not None:\n data = self._data\n else:\n data = self._mission.GetData(\n self.ID, season=self.season,\n cadence=self.cadence,\n clobber=self.clobber_tpf,\n aperture_name=self.aperture_name,\n saturated_aperture_name=self.saturated_aperture_name,\n max_pixels=self.max_pixels,\n saturation_tolerance=self.saturation_tolerance,\n get_hires=self.get_hires,\n get_nearby=self.get_nearby)\n if data is None:\n raise Exception(\"Unable to retrieve target data.\")\n self.cadn = data.cadn\n self.time = data.time\n self.model = np.zeros_like(self.time)\n self.fpix = data.fpix\n self.fraw = np.sum(self.fpix, axis=1)\n self.fpix_err = data.fpix_err\n self.fraw_err = np.sqrt(np.sum(self.fpix_err ** 2, axis=1))\n self.nanmask = data.nanmask\n self.badmask = data.badmask\n self.transitmask = np.array([], dtype=int)\n self.outmask = np.array([], dtype=int)\n self.aperture = data.aperture\n self.aperture_name = data.aperture_name\n self.apertures = data.apertures\n self.quality = data.quality\n self.Xpos = data.Xpos\n self.Ypos = data.Ypos\n self.mag = data.mag\n self.pixel_images = data.pixel_images\n self.nearby = data.nearby\n self.hires = data.hires\n self.saturated = data.saturated\n self.meta = data.meta\n self.bkg = data.bkg\n\n # Update the last breakpoint to the correct value\n self.breakpoints[-1] = len(self.time) - 1\n\n # Get PLD normalization\n self.get_norm()\n\n self.loaded = True", "response": "Loads the target pixel file."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load_model(self, name=None):\n '''\n Loads a saved version of the model.\n\n '''\n\n if self.clobber:\n return False\n\n if name is None:\n name = self.name\n file = os.path.join(self.dir, '%s.npz' % name)\n if os.path.exists(file):\n if not self.is_parent:\n log.info(\"Loading '%s.npz'...\" % name)\n try:\n data = np.load(file)\n for key in data.keys():\n try:\n setattr(self, key, data[key][()])\n except NotImplementedError:\n pass\n\n # HACK: Backwards compatibility. Previous version stored\n # the CDPP in the `cdpp6`\n # and `cdpp6_arr` attributes. Let's move them over.\n if hasattr(self, 'cdpp6'):\n self.cdpp = self.cdpp6\n del self.cdpp6\n if hasattr(self, 'cdpp6_arr'):\n self.cdpp_arr = np.array(self.cdpp6_arr)\n del self.cdpp6_arr\n if hasattr(self, 'gppp'):\n self.cdppg = self.gppp\n del self.gppp\n\n # HACK: At one point we were saving the figure instances,\n # so loading the .npz\n # opened a plotting window. I don't think this is the case\n # any more, so this\n # next line should be removed in the future...\n pl.close()\n\n return True\n except:\n log.warn(\"Error loading '%s.npz'.\" % name)\n exctype, value, tb = sys.exc_info()\n for line in traceback.format_exception_only(exctype, value):\n ln = line.replace('\\n', '')\n log.warn(ln)\n os.rename(file, file + '.bad')\n\n if self.is_parent:\n raise Exception(\n 'Unable to load `%s` model for target %d.'\n % (self.name, self.ID))\n\n return False", "response": "Loads a saved version of the model."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef save_model(self):\n '''\n Saves all of the de-trending information to disk in an `npz` file\n and saves the DVS as a `pdf`.\n\n '''\n\n # Save the data\n log.info(\"Saving data to '%s.npz'...\" % self.name)\n d = dict(self.__dict__)\n d.pop('_weights', None)\n d.pop('_A', None)\n d.pop('_B', None)\n d.pop('_f', None)\n d.pop('_mK', None)\n d.pop('K', None)\n d.pop('dvs', None)\n d.pop('clobber', None)\n d.pop('clobber_tpf', None)\n d.pop('_mission', None)\n d.pop('debug', None)\n d.pop('transit_model', None)\n d.pop('_transit_model', None)\n np.savez(os.path.join(self.dir, self.name + '.npz'), **d)\n\n # Save the DVS\n pdf = PdfPages(os.path.join(self.dir, self.name + '.pdf'))\n pdf.savefig(self.dvs.fig)\n pl.close(self.dvs.fig)\n d = pdf.infodict()\n d['Title'] = 'EVEREST: %s de-trending of %s %d' % (\n self.name, self._mission.IDSTRING, self.ID)\n d['Author'] = 'Rodrigo Luger'\n pdf.close()", "response": "Save all of the de - trending information to disk in an npz file and saves the DVS as a pdf."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef update_gp(self):\n '''\n Calls :py:func:`gp.GetKernelParams` to optimize the GP and obtain the\n covariance matrix for the regression.\n\n '''\n\n self.kernel_params = GetKernelParams(self.time, self.flux,\n self.fraw_err,\n mask=self.mask,\n guess=self.kernel_params,\n kernel=self.kernel,\n giter=self.giter,\n gmaxf=self.gmaxf)", "response": "Updates the kernel parameters of the current object."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef init_kernel(self):\n '''\n Initializes the covariance matrix with a guess at\n the GP kernel parameters.\n\n '''\n\n if self.kernel_params is None:\n X = self.apply_mask(self.fpix / self.flux.reshape(-1, 1))\n y = self.apply_mask(self.flux) - np.dot(X, np.linalg.solve(\n np.dot(X.T, X), np.dot(X.T, self.apply_mask(self.flux))))\n white = np.nanmedian([np.nanstd(c) for c in Chunks(y, 13)])\n amp = self.gp_factor * np.nanstd(y)\n tau = 30.0\n if self.kernel == 'Basic':\n self.kernel_params = [white, amp, tau]\n elif self.kernel == 'QuasiPeriodic':\n self.kernel_params = [white, amp, 1., 20.]", "response": "Initializes the covariance matrix with a guess at\n the GP kernel parameters."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run(self):\n '''\n Runs the de-trending step.\n\n '''\n\n try:\n\n # Load raw data\n log.info(\"Loading target data...\")\n self.load_tpf()\n self.mask_planets()\n self.plot_aperture([self.dvs.top_right() for i in range(4)])\n self.init_kernel()\n M = self.apply_mask(np.arange(len(self.time)))\n self.cdppr_arr = self.get_cdpp_arr()\n self.cdpp_arr = np.array(self.cdppr_arr)\n self.cdppv_arr = np.array(self.cdppr_arr)\n self.cdppr = self.get_cdpp()\n self.cdpp = self.cdppr\n self.cdppv = self.cdppr\n\n log.info(\"%s (Raw): CDPP = %s\" % (self.name, self.cdpps))\n self.plot_lc(self.dvs.left(), info_right='Raw', color='k')\n\n # Loop\n for n in range(self.pld_order):\n self.lam_idx += 1\n self.get_outliers()\n if n > 0 and self.optimize_gp:\n self.update_gp()\n self.cross_validate(self.dvs.right(), info='CV%d' % n)\n self.cdpp_arr = self.get_cdpp_arr()\n self.cdppv_arr *= self.cdpp_arr\n self.cdpp = self.get_cdpp()\n self.cdppv = np.nanmean(self.cdppv_arr)\n log.info(\"%s (%d/%d): CDPP = %s\" %\n (self.name, n + 1, self.pld_order, self.cdpps))\n self.plot_lc(self.dvs.left(), info_right='LC%d' % (\n n + 1), info_left='%d outliers' % len(self.outmask))\n\n # Save\n self.finalize()\n self.plot_final(self.dvs.top_left())\n self.plot_info(self.dvs)\n self.save_model()\n\n except:\n\n self.exception_handler(self.debug)", "response": "Runs the de - trending step."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npublishes the current state of the object to the DVS figure and generates a FITS file for the publication.", "response": "def publish(self, **kwargs):\n '''\n Correct the light curve with the CBVs, generate a\n cover page for the DVS figure,\n and produce a FITS file for publication.\n\n '''\n\n try:\n\n # HACK: Force these params for publication\n self.cbv_win = 999\n self.cbv_order = 3\n self.cbv_num = 1\n\n # Get the CBVs\n self._mission.GetTargetCBVs(self)\n\n # Plot the final corrected light curve\n cbv = CBV()\n self.plot_info(cbv)\n self.plot_cbv(cbv.body(), self.fcor, 'Corrected')\n self.plot_cbv(cbv.body(), self.flux, 'De-trended', show_cbv=True)\n self.plot_cbv(cbv.body(), self.fraw, 'Raw')\n\n # Save the CBV pdf\n pdf = PdfPages(os.path.join(self.dir, 'cbv.pdf'))\n pdf.savefig(cbv.fig)\n pl.close(cbv.fig)\n d = pdf.infodict()\n d['Title'] = 'EVEREST: %s de-trending of %s %d' % (\n self.name, self._mission.IDSTRING, self.ID)\n d['Author'] = 'Rodrigo Luger'\n pdf.close()\n\n # Now merge the two PDFs\n assert os.path.exists(os.path.join(\n self.dir, self.name + '.pdf')), \\\n \"Unable to locate %s.pdf.\" % self.name\n output = PdfFileWriter()\n pdfOne = PdfFileReader(os.path.join(self.dir, 'cbv.pdf'))\n pdfTwo = PdfFileReader(os.path.join(self.dir, self.name + '.pdf'))\n # Add the CBV page\n output.addPage(pdfOne.getPage(0))\n # Add the original DVS page\n output.addPage(pdfTwo.getPage(pdfTwo.numPages - 1))\n # Write the final PDF\n outputStream = open(os.path.join(self.dir, self._mission.DVSFile(\n self.ID, self.season, self.cadence)), \"wb\")\n output.write(outputStream)\n outputStream.close()\n os.remove(os.path.join(self.dir, 'cbv.pdf'))\n\n # Make the FITS file\n MakeFITS(self)\n\n except:\n\n self.exception_handler(self.debug)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setup(self, **kwargs):\n '''\n This is called during production de-trending, prior to\n calling the :py:obj:`Detrender.run()` method.\n\n :param tuple cdpp_range: If :py:obj:`parent_model` is set, \\\n neighbors are selected only if \\\n their de-trended CDPPs fall within this range. Default `None`\n :param tuple mag_range: Only select neighbors whose magnitudes are \\\n within this range. Default (11., 13.)\n :param int neighbors: The number of neighboring stars to use in \\\n the de-trending. The higher this number, the more signals \\\n there are and hence the more de-trending information there is. \\\n However, the neighboring star signals are regularized together \\\n with the target's signals, so adding too many neighbors will \\\n inevitably reduce the contribution of the target's own \\\n signals, which may reduce performance. Default `10`\n :param str parent_model: By default, :py:class:`nPLD` is run in \\\n stand-alone mode. The neighbor signals are computed directly \\\n from their TPFs, so there is no need to have run *PLD* on them \\\n beforehand. However, if :py:obj:`parent_model` \\\n is set, :py:class:`nPLD` will use information from the \\\n :py:obj:`parent_model` model of each neighboring star when \\\n de-trending. This is particularly useful for identifying \\\n outliers in the neighbor signals and preventing them from \\\n polluting the current target. Setting :py:obj:`parent_model` \\\n to :py:class:`rPLD`, for instance, will use the \\\n outlier information in the :py:class:`rPLD` model of the \\\n neighbors (this must have been run ahead of time). \\\n Note, however, that tests with *K2* data show that including \\\n outliers in the neighbor signals actually \\\n *improves* the performance, since many of these outliers \\\n are associated with events such as thruster firings and are \\\n present in all light curves, and therefore *help* in the \\\n de-trending. Default `None`\n\n ..note :: Optionally, the :py:obj:`neighbors` may be specified \\\n directly as a list of target IDs to use. \\\n In this case, users may also provide a list of \\\n :py:class:`everest.utils.DataContainer` instances \\\n corresponding to each of the neighbors in the \\\n :py:obj:`neighbors_data` kwarg.\n '''\n\n # Get neighbors\n self.parent_model = kwargs.get('parent_model', None)\n neighbors = kwargs.get('neighbors', 10)\n neighbors_data = kwargs.get('neighbors_data', None)\n if hasattr(neighbors, '__len__'):\n self.neighbors = neighbors\n else:\n num_neighbors = neighbors\n self.neighbors = \\\n self._mission.GetNeighbors(self.ID,\n season=self.season,\n cadence=self.cadence,\n model=self.parent_model,\n neighbors=num_neighbors,\n mag_range=kwargs.get(\n 'mag_range', (11., 13.)),\n cdpp_range=kwargs.get(\n 'cdpp_range', None),\n aperture_name=self.aperture_name)\n if len(self.neighbors):\n if len(self.neighbors) < num_neighbors:\n log.warn(\"%d neighbors requested, but only %d found.\" %\n (num_neighbors, len(self.neighbors)))\n elif num_neighbors > 0:\n log.warn(\"No neighbors found! Running standard PLD...\")\n\n for n, neighbor in enumerate(self.neighbors):\n log.info(\"Loading data for neighboring target %d...\" % neighbor)\n if neighbors_data is not None:\n data = neighbors_data[n]\n data.mask = np.array(\n list(set(np.concatenate([data.badmask, data.nanmask]))),\n dtype=int)\n data.fraw = np.sum(data.fpix, axis=1)\n elif self.parent_model is not None and self.cadence == 'lc':\n # We load the `parent` model. The advantage here is\n # that outliers have properly been identified and masked.\n # I haven't tested this on short\n # cadence data, so I'm going to just forbid it...\n data = eval(self.parent_model)(\n neighbor, mission=self.mission, is_parent=True)\n else:\n # We load the data straight from the TPF. Much quicker,\n # since no model must be run in advance. Downside is we\n # don't know where the outliers are. But based\n # on tests with K2 data, the de-trending is actually\n # *better* if the outliers are\n # included! These are mostly thruster fire events and other\n # artifacts common to\n # all the stars, so it makes sense that we might want\n # to keep them in the design matrix.\n data = self._mission.GetData(neighbor, season=self.season,\n clobber=self.clobber_tpf,\n cadence=self.cadence,\n aperture_name=self.aperture_name,\n saturated_aperture_name=\n self.saturated_aperture_name,\n max_pixels=self.max_pixels,\n saturation_tolerance=\n self.saturation_tolerance,\n get_hires=False, get_nearby=False)\n if data is None:\n raise Exception(\n \"Unable to retrieve data for neighboring target.\")\n data.mask = np.array(\n list(set(np.concatenate([data.badmask, data.nanmask]))),\n dtype=int)\n data.fraw = np.sum(data.fpix, axis=1)\n\n # Compute the linear PLD vectors and interpolate over\n # outliers, NaNs and bad timestamps\n X1 = data.fpix / data.fraw.reshape(-1, 1)\n X1 = Interpolate(data.time, data.mask, X1)\n if self.X1N is None:\n self.X1N = np.array(X1)\n else:\n self.X1N = np.hstack([self.X1N, X1])\n del X1\n del data", "response": "This method is called by the detector to setup the internal state of the detector."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run(self):\n '''\n Runs the de-trending.\n\n '''\n\n try:\n\n # Plot original\n self.plot_aperture([self.dvs.top_right() for i in range(4)])\n self.plot_lc(self.dvs.left(), info_right='nPLD', color='k')\n\n # Cross-validate\n self.cross_validate(self.dvs.right())\n self.compute()\n self.cdpp_arr = self.get_cdpp_arr()\n self.cdpp = self.get_cdpp()\n\n # Plot new\n self.plot_lc(self.dvs.left(), info_right='Powell', color='k')\n\n # Save\n self.plot_final(self.dvs.top_left())\n self.plot_info(self.dvs)\n self.save_model()\n\n except:\n\n self.exception_handler(self.debug)", "response": "Runs the de - trending.\n "} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nperforms the cross - validation step.", "response": "def cross_validate(self, ax):\n '''\n Performs the cross-validation step.\n\n '''\n\n # The CDPP to beat\n cdpp_opt = self.get_cdpp_arr()\n\n # Loop over all chunks\n for b, brkpt in enumerate(self.breakpoints):\n\n log.info(\"Cross-validating chunk %d/%d...\" %\n (b + 1, len(self.breakpoints)))\n\n # Mask for current chunk\n m = self.get_masked_chunk(b)\n\n # Mask transits and outliers\n time = self.time[m]\n flux = self.fraw[m]\n ferr = self.fraw_err[m]\n med = np.nanmedian(self.fraw)\n\n # Setup the GP\n gp = GP(self.kernel, self.kernel_params, white=False)\n gp.compute(time, ferr)\n\n # The masks\n masks = list(Chunks(np.arange(0, len(time)),\n len(time) // self.cdivs))\n\n # The pre-computed matrices\n pre_v = [self.cv_precompute(mask, b) for mask in masks]\n\n # Initialize with the nPLD solution\n log_lam_opt = np.log10(self.lam[b])\n scatter_opt = self.validation_scatter(\n log_lam_opt, b, masks, pre_v, gp, flux, time, med)\n log.info(\"Iter 0/%d: \" % (self.piter) +\n \"logL = (%s), s = %.3f\" %\n (\", \".join([\"%.3f\" % l for l in log_lam_opt]),\n scatter_opt))\n\n # Do `piter` iterations\n for p in range(self.piter):\n\n # Perturb the initial condition a bit\n log_lam = np.array(\n np.log10(self.lam[b])) * \\\n (1 + self.ppert * np.random.randn(len(self.lam[b])))\n scatter = self.validation_scatter(\n log_lam, b, masks, pre_v, gp, flux, time, med)\n log.info(\"Initializing at: \" +\n \"logL = (%s), s = %.3f\" %\n (\", \".join([\"%.3f\" % l for l in log_lam]), scatter))\n\n # Call the minimizer\n log_lam, scatter, _, _, _, _ = \\\n fmin_powell(self.validation_scatter, log_lam,\n args=(b, masks, pre_v, gp, flux, time, med),\n maxfun=self.pmaxf, disp=False,\n full_output=True)\n\n # Did it improve the CDPP?\n tmp = np.array(self.lam[b])\n self.lam[b] = 10 ** log_lam\n self.compute()\n cdpp = self.get_cdpp_arr()[b]\n self.lam[b] = tmp\n if cdpp < cdpp_opt[b]:\n cdpp_opt[b] = cdpp\n log_lam_opt = log_lam\n\n # Log it\n log.info(\"Iter %d/%d: \" % (p + 1, self.piter) +\n \"logL = (%s), s = %.3f\" %\n (\", \".join([\"%.3f\" % l for l in log_lam]), scatter))\n\n # The best solution\n log.info(\"Found minimum: logL = (%s), s = %.3f\" %\n (\", \".join([\"%.3f\" % l for l in log_lam_opt]),\n scatter_opt))\n self.lam[b] = 10 ** log_lam_opt\n\n # We're just going to plot lambda as a function of chunk number\n bs = np.arange(len(self.breakpoints))\n color = ['k', 'b', 'r', 'g', 'y']\n for n in range(self.pld_order):\n ax[0].plot(bs + 1, [np.log10(self.lam[b][n])\n for b in bs], '.', color=color[n])\n ax[0].plot(bs + 1, [np.log10(self.lam[b][n])\n for b in bs], '-', color=color[n], alpha=0.25)\n ax[0].set_ylabel(r'$\\log\\Lambda$', fontsize=5)\n ax[0].margins(0.1, 0.1)\n ax[0].set_xticks(np.arange(1, len(self.breakpoints) + 1))\n ax[0].set_xticklabels([])\n\n # Now plot the CDPP\n cdpp_arr = self.get_cdpp_arr()\n ax[1].plot(bs + 1, cdpp_arr, 'b.')\n ax[1].plot(bs + 1, cdpp_arr, 'b-', alpha=0.25)\n ax[1].margins(0.1, 0.1)\n ax[1].set_ylabel(r'Scatter (ppm)', fontsize=5)\n ax[1].set_xlabel(r'Chunk', fontsize=5)\n ax[1].set_xticks(np.arange(1, len(self.breakpoints) + 1))"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef validation_scatter(self, log_lam, b, masks, pre_v, gp, flux,\n time, med):\n '''\n Computes the scatter in the validation set.\n\n '''\n\n # Update the lambda matrix\n self.lam[b] = 10 ** log_lam\n\n # Validation set scatter\n scatter = [None for i in range(len(masks))]\n for i in range(len(masks)):\n model = self.cv_compute(b, *pre_v[i])\n try:\n gpm, _ = gp.predict(flux - model - med, time[masks[i]])\n except ValueError:\n # Sometimes the model can have NaNs if\n # `lambda` is a crazy value\n return 1.e30\n fdet = (flux - model)[masks[i]] - gpm\n scatter[i] = 1.e6 * (1.4826 * np.nanmedian(np.abs(fdet / med -\n np.nanmedian(fdet / med))) /\n np.sqrt(len(masks[i])))\n\n return np.max(scatter)", "response": "Compute the scatter for a given set of masks and the given time."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsearch for a given star in a random univariate light curve.", "response": "def Search(star, pos_tol=2.5, neg_tol=50., **ps_kwargs):\n '''\n NOTE: `pos_tol` is the positive (i.e., above the median)\n outlier tolerance in standard deviations.\n NOTE: `neg_tol` is the negative (i.e., below the median)\n outlier tolerance in standard deviations.\n\n '''\n\n # Smooth the light curve\n t = np.delete(star.time, np.concatenate([star.nanmask, star.badmask]))\n f = np.delete(star.flux, np.concatenate([star.nanmask, star.badmask]))\n f = SavGol(f)\n med = np.nanmedian(f)\n\n # Kill positive outliers\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n pos_inds = np.where((f > med + pos_tol * MAD))[0]\n pos_inds = np.array([np.argmax(star.time == t[i]) for i in pos_inds])\n\n # Kill negative outliers\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n neg_inds = np.where((f < med - neg_tol * MAD))[0]\n neg_inds = np.array([np.argmax(star.time == t[i]) for i in neg_inds])\n\n # Replace the star.outmask array\n star.outmask = np.concatenate([neg_inds, pos_inds])\n star.transitmask = np.array([], dtype=int)\n\n # Delta chi squared\n TIME = np.array([])\n DEPTH = np.array([])\n VARDEPTH = np.array([])\n DELCHISQ = np.array([])\n for b, brkpt in enumerate(star.breakpoints):\n\n # Log\n log.info('Running chunk %d/%d...' % (b + 1, len(star.breakpoints)))\n\n # Masks for current chunk\n m = star.get_masked_chunk(b, pad=False)\n\n # This block of the masked covariance matrix\n K = GetCovariance(star.kernel, star.kernel_params,\n star.time[m], star.fraw_err[m])\n\n # The masked X.L.X^T term\n A = np.zeros((len(m), len(m)))\n for n in range(star.pld_order):\n XM = star.X(n, m)\n A += star.lam[b][n] * np.dot(XM, XM.T)\n K += A\n CDK = cho_factor(K)\n\n # Baseline\n med = np.nanmedian(star.fraw[m])\n lnL0 = -0.5 * np.dot(star.fraw[m], cho_solve(CDK, star.fraw[m]))\n dt = np.median(np.diff(star.time[m]))\n\n # Create a uniform time array and get indices of missing cadences\n tol = np.nanmedian(np.diff(star.time[m])) / 5.\n tunif = np.arange(star.time[m][0], star.time[m][-1] + tol, dt)\n tnogaps = np.array(tunif)\n gaps = []\n j = 0\n for i, t in enumerate(tunif):\n if np.abs(star.time[m][j] - t) < tol:\n tnogaps[i] = star.time[m][j]\n j += 1\n if j == len(star.time[m]):\n break\n else:\n gaps.append(i)\n gaps = np.array(gaps, dtype=int)\n\n # Compute the normalized transit model for a single transit\n transit_model = TransitShape(**ps_kwargs)\n\n # Now roll the transit model across each cadence\n dchisq = np.zeros(len(tnogaps))\n d = np.zeros(len(tnogaps))\n vard = np.zeros(len(tnogaps))\n for i in prange(len(tnogaps)):\n trn = transit_model(tnogaps, tnogaps[i])\n trn = np.delete(trn, gaps)\n trn *= med\n vard[i] = 1. / np.dot(trn, cho_solve(CDK, trn))\n if not np.isfinite(vard[i]):\n vard[i] = np.nan\n d[i] = np.nan\n dchisq[i] = np.nan\n continue\n d[i] = vard[i] * np.dot(trn, cho_solve(CDK, star.fraw[m]))\n r = star.fraw[m] - trn * d[i]\n lnL = -0.5 * np.dot(r, cho_solve(CDK, r))\n dchisq[i] = -2 * (lnL0 - lnL)\n\n TIME = np.append(TIME, tnogaps)\n DEPTH = np.append(DEPTH, d)\n VARDEPTH = np.append(VARDEPTH, vard)\n DELCHISQ = np.append(DELCHISQ, dchisq)\n\n return TIME, DEPTH, VARDEPTH, DELCHISQ"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nordering iterator over dirty elements.", "response": "def iterdirty(self):\r\n '''Ordered iterator over dirty elements.'''\r\n return iter(chain(itervalues(self._new), itervalues(self._modified)))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a new instance to the session.", "response": "def add(self, instance, modified=True, persistent=None,\r\n force_update=False):\r\n '''Add a new instance to this :class:`SessionModel`.\r\n\r\n:param modified: Optional flag indicating if the ``instance`` has been\r\n modified. By default its value is ``True``.\r\n:param force_update: if ``instance`` is persistent, it forces an update of the\r\n data rather than a full replacement. This is used by the\r\n :meth:`insert_update_replace` method.\r\n:rtype: The instance added to the session'''\r\n if instance._meta.type == 'structure':\r\n return self._add_structure(instance)\r\n state = instance.get_state()\r\n if state.deleted:\r\n raise ValueError('State is deleted. Cannot add.')\r\n self.pop(state.iid)\r\n pers = persistent if persistent is not None else state.persistent\r\n pkname = instance._meta.pkname()\r\n if not pers:\r\n instance._dbdata.pop(pkname, None) # to make sure it is add action\r\n state = instance.get_state(iid=None)\r\n elif persistent:\r\n instance._dbdata[pkname] = instance.pkvalue()\r\n state = instance.get_state(iid=instance.pkvalue())\r\n else:\r\n action = 'update' if force_update else None\r\n state = instance.get_state(action=action, iid=state.iid)\r\n iid = state.iid\r\n if state.persistent:\r\n if modified:\r\n self._modified[iid] = instance\r\n else:\r\n self._new[iid] = instance\r\n return instance"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pop(self, instance):\r\n '''Remove ``instance`` from the :class:`SessionModel`. Instance\r\ncould be a :class:`Model` or an id.\r\n\r\n:parameter instance: a :class:`Model` or an ``id``.\r\n:rtype: the :class:`Model` removed from session or ``None`` if\r\n it was not in the session.\r\n'''\r\n if isinstance(instance, self.model):\r\n iid = instance.get_state().iid\r\n else:\r\n iid = instance\r\n instance = None\r\n for d in (self._new, self._modified, self._deleted):\r\n if iid in d:\r\n inst = d.pop(iid)\r\n if instance is None:\r\n instance = inst\r\n elif inst is not instance:\r\n raise ValueError('Critical error: %s is duplicated' % iid)\r\n return instance", "response": "Remove an instance from the session or None if the instance is not in the session."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef expunge(self, instance):\r\n '''Remove *instance* from the :class:`Session`. Instance could be a\r\n:class:`Model` or an id.\r\n\r\n:parameter instance: a :class:`Model` or an *id*\r\n:rtype: the :class:`Model` removed from session or ``None`` if\r\n it was not in the session.\r\n'''\r\n instance = self.pop(instance)\r\n instance.session = None\r\n return instance", "response": "Removes the given instance from the session."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nposting - commit the results of a single instance session.", "response": "def post_commit(self, results):\r\n '''\\\r\nProcess results after a commit.\r\n\r\n:parameter results: iterator over :class:`stdnet.instance_session_result`\r\n items.\r\n:rtype: a two elements tuple containing a list of instances saved and\r\n a list of ids of instances deleted.'''\r\n tpy = self._meta.pk_to_python\r\n instances = []\r\n deleted = []\r\n errors = []\r\n # The length of results must be the same as the length of\r\n # all committed instances\r\n for result in results:\r\n if isinstance(result, Exception):\r\n errors.append(result.__class__('Exception while committing %s.'\r\n ' %s' % (self._meta, result)))\r\n continue\r\n instance = self.pop(result.iid)\r\n id = tpy(result.id, self.backend)\r\n if result.deleted:\r\n deleted.append(id)\r\n else:\r\n if instance is None:\r\n raise InvalidTransaction('{0} session received id \"{1}\"\\\r\n which is not in the session.'.format(self, result.iid))\r\n setattr(instance, instance._meta.pkname(), id)\r\n instance = self.add(instance,\r\n modified=False,\r\n persistent=result.persistent)\r\n instance.get_state().score = result.score\r\n if instance.get_state().persistent:\r\n instances.append(instance)\r\n return instances, deleted, errors"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef commit(self, callback=None):\r\n '''Close the transaction and commit session to the backend.'''\r\n if self.executed:\r\n raise InvalidTransaction('Invalid operation. '\r\n 'Transaction already executed.')\r\n session = self.session\r\n self.session = None\r\n self.on_result = self._commit(session, callback)\r\n return self.on_result", "response": "Close the transaction and commit the session to the backend."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef begin(self, **options):\r\n '''Begin a new :class:`Transaction`. If this :class:`Session`\r\nis already in a :ref:`transactional state `,\r\nan error will occur. It returns the :attr:`transaction` attribute.\r\n\r\nThis method is mostly used within a ``with`` statement block::\r\n\r\n with session.begin() as t:\r\n t.add(...)\r\n ...\r\n\r\nwhich is equivalent to::\r\n\r\n t = session.begin()\r\n t.add(...)\r\n ...\r\n session.commit()\r\n\r\n``options`` parameters are passed to the :class:`Transaction` constructor.\r\n'''\r\n if self.transaction is not None:\r\n raise InvalidTransaction(\"A transaction is already begun.\")\r\n else:\r\n self.transaction = Transaction(self, **options)\r\n return self.transaction", "response": "Begin a new transaction."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate a new : class : Query for model *model*.", "response": "def query(self, model, **kwargs):\r\n '''Create a new :class:`Query` for *model*.'''\r\n sm = self.model(model)\r\n query_class = sm.manager.query_class or Query\r\n return query_class(sm._meta, self, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nupdates or create a new instance of model.", "response": "def update_or_create(self, model, **kwargs):\r\n '''Update or create a new instance of ``model``.\r\n\r\n This method can raise an exception if the ``kwargs`` dictionary\r\n contains field data that does not validate.\r\n\r\n :param model: a :class:`StdModel`\r\n :param kwargs: dictionary of parameters.\r\n :returns: A two elements tuple containing the instance and a boolean\r\n indicating if the instance was created or not.\r\n '''\r\n backend = self.model(model).backend\r\n return backend.execute(self._update_or_create(model, **kwargs))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add(self, instance, modified=True, **params):\r\n '''Add an ``instance`` to the session.\r\n\r\n If the session is not in a\r\n :ref:`transactional state `, this operation\r\n commits changes to the back-end server immediately.\r\n\r\n :parameter instance: a :class:`Model` instance. It must be registered\r\n with the :attr:`router` which created this :class:`Session`.\r\n :parameter modified: a boolean flag indicating if the instance was\r\n modified.\r\n :return: the ``instance``.\r\n\r\n If the instance is persistent (it is already stored in the database),\r\n an updated will be performed, otherwise a new entry will be created\r\n once the :meth:`commit` method is invoked.\r\n '''\r\n sm = self.model(instance)\r\n instance.session = self\r\n o = sm.add(instance, modified=modified, **params)\r\n if modified and not self.transaction:\r\n transaction = self.begin()\r\n return transaction.commit(lambda: o)\r\n else:\r\n return o", "response": "Add an instance to the session."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ndeletes an instance or a query.", "response": "def delete(self, instance_or_query):\r\n '''Delete an ``instance`` or a ``query``.\r\n\r\n Adds ``instance_or_query`` to this :class:`Session` list\r\n of data to be deleted. If the session is not in a\r\n :ref:`transactional state `, this operation\r\n commits changes to the backend server immediately.\r\n\r\n :parameter instance_or_query: a :class:`Model` instance or\r\n a :class:`Query`.\r\n '''\r\n sm = self.model(instance_or_query)\r\n # not an instance of a Model. Assume it is a query.\r\n if is_query(instance_or_query):\r\n if instance_or_query.session is not self:\r\n raise ValueError('Adding a query generated by another session')\r\n sm._delete_query.append(instance_or_query)\r\n else:\r\n instance_or_query = sm.delete(instance_or_query, self)\r\n if not self.transaction:\r\n transaction = self.begin()\r\n return transaction.commit(\r\n lambda: transaction.deleted.get(sm._meta))\r\n else:\r\n return instance_or_query"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef model(self, model, create=True):\r\n '''Returns the :class:`SessionModel` for ``model`` which\r\ncan be :class:`Model`, or a :class:`MetaClass`, or an instance\r\nof :class:`Model`.'''\r\n manager = self.manager(model)\r\n sm = self._models.get(manager)\r\n if sm is None and create:\r\n sm = SessionModel(manager)\r\n self._models[manager] = sm\r\n return sm", "response": "Returns the SessionModel for the given model which is a MetaClass or an instance of Model."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving instance from this Session.", "response": "def expunge(self, instance=None):\r\n '''Remove ``instance`` from this :class:`Session`. If ``instance``\r\nis not given, it removes all instances from this :class:`Session`.'''\r\n if instance is not None:\r\n sm = self._models.get(instance._meta)\r\n if sm:\r\n return sm.expunge(instance)\r\n else:\r\n self._models.clear()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef manager(self, model):\r\n '''Retrieve the :class:`Manager` for ``model`` which can be any of the\r\nvalues valid for the :meth:`model` method.'''\r\n try:\r\n return self.router[model]\r\n except KeyError:\r\n meta = getattr(model, '_meta', model)\r\n if meta.type == 'structure':\r\n # this is a structure\r\n if hasattr(model, 'model'):\r\n structure_model = model.model\r\n if structure_model:\r\n return self.manager(structure_model)\r\n else:\r\n manager = self.router.structure(model)\r\n if manager:\r\n return manager\r\n raise InvalidTransaction('\"%s\" not valid in this session' % meta)", "response": "Retrieve the manager for the given model which can be any of the the\r\nvalues valid for the model method."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef new(self, *args, **kwargs):\r\n '''Create a new instance of :attr:`model` and commit it to the backend\r\nserver. This a shortcut method for the more verbose::\r\n\r\n instance = manager.session().add(MyModel(**kwargs))\r\n'''\r\n return self.session().add(self.model(*args, **kwargs))", "response": "Create a new instance of the model and commit it to the backend\r\nserver."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef query(self, session=None):\r\n '''Returns a new :class:`Query` for :attr:`Manager.model`.'''\r\n if session is None or session.router is not self.router:\r\n session = self.session()\r\n return session.query(self.model)", "response": "Returns a new : class : Query object for the current model."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a new Query object for the given text search value.", "response": "def search(self, text, lookup=None):\r\n '''Returns a new :class:`Query` for :attr:`Manager.model` with\r\na full text search value.'''\r\n return self.query().search(text, lookup=lookup)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef pairs_to_dict(response, encoding):\r\n \"Create a dict given a list of key/value pairs\"\r\n it = iter(response)\r\n return dict(((k.decode(encoding), v) for k, v in zip(it, it)))", "response": "Create a dict given a list of key / value pairs"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef load_related(self, meta, fname, data, fields, encoding):\r\n '''Parse data for related objects.'''\r\n field = meta.dfields[fname]\r\n if field in meta.multifields:\r\n fmeta = field.structure_class()._meta\r\n if fmeta.name in ('hashtable', 'zset'):\r\n return ((native_str(id, encoding),\r\n pairs_to_dict(fdata, encoding)) for\r\n id, fdata in data)\r\n else:\r\n return ((native_str(id, encoding), fdata) for\r\n id, fdata in data)\r\n else:\r\n # this is data for stdmodel instances\r\n return self.build(data, meta, fields, fields, encoding)", "response": "Parse data for related objects."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _execute_query(self):\r\n '''Execute the query without fetching data. Returns the number of\r\nelements in the query.'''\r\n pipe = self.pipe\r\n if not self.card:\r\n if self.meta.ordering:\r\n self.ismember = getattr(self.backend.client, 'zrank')\r\n self.card = getattr(pipe, 'zcard')\r\n self._check_member = self.zism\r\n else:\r\n self.ismember = getattr(self.backend.client, 'sismember')\r\n self.card = getattr(pipe, 'scard')\r\n self._check_member = self.sism\r\n else:\r\n self.ismember = None\r\n self.card(self.query_key)\r\n result = yield pipe.execute()\r\n yield result[-1]", "response": "Execute the query without fetching data. Returns the number of\r\nelements in the query."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nperform ordering with respect model fields.", "response": "def order(self, last):\r\n '''Perform ordering with respect model fields.'''\r\n desc = last.desc\r\n field = last.name\r\n nested = last.nested\r\n nested_args = []\r\n while nested:\r\n meta = nested.model._meta\r\n nested_args.extend((self.backend.basekey(meta), nested.name))\r\n last = nested\r\n nested = nested.nested\r\n method = 'ALPHA' if last.field.internal_type == 'text' else ''\r\n if field == last.model._meta.pkname():\r\n field = ''\r\n return {'field': field,\r\n 'method': method,\r\n 'desc': desc,\r\n 'nested': nested_args}"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nremoving and return a range from the ordered set by rank ( index.", "response": "def ipop_range(self, start, stop=None, withscores=True, **options):\r\n '''Remove and return a range from the ordered set by rank (index).'''\r\n return self.backend.execute(\r\n self.client.zpopbyrank(self.id, start, stop,\r\n withscores=withscores, **options),\r\n partial(self._range, withscores))"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nremove and return a range from the ordered set by score.", "response": "def pop_range(self, start, stop=None, withscores=True, **options):\r\n '''Remove and return a range from the ordered set by score.'''\r\n return self.backend.execute(\r\n self.client.zpopbyscore(self.id, start, stop,\r\n withscores=withscores, **options),\r\n partial(self._range, withscores))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef meta(self, meta):\r\n '''Extract model metadata for lua script stdnet/lib/lua/odm.lua'''\r\n data = meta.as_dict()\r\n data['namespace'] = self.basekey(meta)\r\n return data", "response": "Extract model metadata for lua script stdnet / lib / lua. lua"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nexecuting a single session in redis.", "response": "def execute_session(self, session_data):\r\n '''Execute a session in redis.'''\r\n pipe = self.client.pipeline()\r\n for sm in session_data: # loop through model sessions\r\n meta = sm.meta\r\n if sm.structures:\r\n self.flush_structure(sm, pipe)\r\n delquery = None\r\n if sm.deletes is not None:\r\n delquery = sm.deletes.backend_query(pipe=pipe)\r\n self.accumulate_delete(pipe, delquery)\r\n if sm.dirty:\r\n meta_info = json.dumps(self.meta(meta))\r\n lua_data = [len(sm.dirty)]\r\n processed = []\r\n for instance in sm.dirty:\r\n state = instance.get_state()\r\n if not meta.is_valid(instance):\r\n raise FieldValueError(\r\n json.dumps(instance._dbdata['errors']))\r\n score = MIN_FLOAT\r\n if meta.ordering:\r\n if meta.ordering.auto:\r\n score = meta.ordering.name.incrby\r\n else:\r\n v = getattr(instance, meta.ordering.name, None)\r\n if v is not None:\r\n score = meta.ordering.field.scorefun(v)\r\n data = instance._dbdata['cleaned_data']\r\n action = state.action\r\n prev_id = state.iid if state.persistent else ''\r\n id = instance.pkvalue() or ''\r\n data = flat_mapping(data)\r\n lua_data.extend((action, prev_id, id, score, len(data)))\r\n lua_data.extend(data)\r\n processed.append(state.iid)\r\n self.odmrun(pipe, 'commit', meta, (), meta_info,\r\n *lua_data, iids=processed)\r\n return pipe.execute()"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nflush all model keys from the database", "response": "def flush(self, meta=None):\r\n '''Flush all model keys from the database'''\r\n pattern = self.basekey(meta) if meta else self.namespace\r\n return self.client.delpattern('%s*' % pattern)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetCovariance(kernel, kernel_params, time, errors):\n '''\n Returns the covariance matrix for a given light curve\n segment.\n\n :param array_like kernel_params: A list of kernel parameters \\\n (white noise amplitude, red noise amplitude, and red noise timescale)\n :param array_like time: The time array (*N*)\n :param array_like errors: The data error array (*N*)\n\n :returns: The covariance matrix :py:obj:`K` (*N*,*N*)\n\n '''\n\n # NOTE: We purposefully compute the covariance matrix\n # *without* the GP white noise term\n K = np.diag(errors ** 2)\n K += GP(kernel, kernel_params, white=False).get_matrix(time)\n return K", "response": "Calculates the covariance matrix for a given light curve\n segment."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\noptimize the GP by training it on the current de-trended light curve. Returns the white noise amplitude, red noise amplitude, and red noise timescale. :param array_like time: The time array :param array_like flux: The flux array :param array_like errors: The flux errors array :param array_like mask: The indices to be masked when training the GP. \\ Default `[]` :param int giter: The number of iterations. Default 3 :param int gmaxf: The maximum number of function evaluations. Default 200 :param tuple guess: The guess to initialize the minimization with. \\ Default :py:obj:`None`", "response": "def GetKernelParams(time, flux, errors, kernel='Basic', mask=[],\n giter=3, gmaxf=200, guess=None):\n '''\n Optimizes the GP by training it on the current de-trended light curve.\n Returns the white noise amplitude, red noise amplitude,\n and red noise timescale.\n\n :param array_like time: The time array\n :param array_like flux: The flux array\n :param array_like errors: The flux errors array\n :param array_like mask: The indices to be masked when training the GP. \\\n Default `[]`\n :param int giter: The number of iterations. Default 3\n :param int gmaxf: The maximum number of function evaluations. Default 200\n :param tuple guess: The guess to initialize the minimization with. \\\n Default :py:obj:`None`\n\n '''\n\n log.info(\"Optimizing the GP...\")\n\n # Save a copy of time and errors for later\n time_copy = np.array(time)\n errors_copy = np.array(errors)\n\n # Apply the mask\n time = np.delete(time, mask)\n flux = np.delete(flux, mask)\n errors = np.delete(errors, mask)\n\n # Remove 5-sigma outliers to be safe\n f = flux - savgol_filter(flux, 49, 2) + np.nanmedian(flux)\n med = np.nanmedian(f)\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n mask = np.where((f > med + 5 * MAD) | (f < med - 5 * MAD))[0]\n time = np.delete(time, mask)\n flux = np.delete(flux, mask)\n errors = np.delete(errors, mask)\n\n # Initial guesses and bounds\n white = np.nanmedian([np.nanstd(c) for c in Chunks(flux, 13)])\n amp = np.nanstd(flux)\n tau = 30.0\n if kernel == 'Basic':\n if guess is None:\n guess = [white, amp, tau]\n bounds = [[0.1 * white, 10. * white],\n [1., 10000. * amp],\n [0.5, 100.]]\n elif kernel == 'QuasiPeriodic':\n if guess is None:\n guess = [white, amp, tau, 1., 20.]\n bounds = [[0.1 * white, 10. * white],\n [1., 10000. * amp],\n [1e-5, 1e2],\n [0.02, 100.]]\n else:\n raise ValueError('Invalid value for `kernel`.')\n\n # Loop\n llbest = -np.inf\n xbest = np.array(guess)\n for i in range(giter):\n\n # Randomize an initial guess\n iguess = [np.inf for g in guess]\n for j, b in enumerate(bounds):\n tries = 0\n while (iguess[j] < b[0]) or (iguess[j] > b[1]):\n iguess[j] = (1 + 0.5 * np.random.randn()) * guess[j]\n tries += 1\n if tries > 100:\n iguess[j] = b[0] + np.random.random() * (b[1] - b[0])\n break\n\n # Optimize\n x = fmin_l_bfgs_b(NegLnLike, iguess, approx_grad=False,\n bounds=bounds, args=(time, flux, errors, kernel),\n maxfun=gmaxf)\n log.info('Iteration #%d/%d:' % (i + 1, giter))\n log.info(' ' + x[2]['task'].decode('utf-8'))\n log.info(' ' + 'Function calls: %d' % x[2]['funcalls'])\n log.info(' ' + 'Log-likelihood: %.3e' % -x[1])\n if kernel == 'Basic':\n log.info(' ' + 'White noise : %.3e (%.1f x error bars)' %\n (x[0][0], x[0][0] / np.nanmedian(errors)))\n log.info(' ' + 'Red amplitude : %.3e (%.1f x stand dev)' %\n (x[0][1], x[0][1] / np.nanstd(flux)))\n log.info(' ' + 'Red timescale : %.2f days' % x[0][2])\n elif kernel == 'QuasiPeriodic':\n log.info(' ' + 'White noise : %.3e (%.1f x error bars)' %\n (x[0][0], x[0][0] / np.nanmedian(errors)))\n log.info(' ' + 'Red amplitude : %.3e (%.1f x stand dev)' %\n (x[0][1], x[0][1] / np.nanstd(flux)))\n log.info(' ' + 'Gamma : %.3e' % x[0][2])\n log.info(' ' + 'Period : %.2f days' % x[0][3])\n if -x[1] > llbest:\n llbest = -x[1]\n xbest = np.array(x[0])\n\n return xbest"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the negative log - likelihood function and its gradient.", "response": "def NegLnLike(x, time, flux, errors, kernel):\n '''\n Returns the negative log-likelihood function and its gradient.\n\n '''\n\n gp = GP(kernel, x, white=True)\n gp.compute(time, errors)\n if OLDGEORGE:\n nll = -gp.lnlikelihood(flux)\n # NOTE: There was a bug on this next line! Used to be\n #\n # ngr = -gp.grad_lnlikelihood(flux) / gp.kernel.pars\n #\n # But I think we want\n #\n # dlogL/dx = dlogL/dlogx^2 * dlogx^2/dx^2 * dx^2/dx\n # = gp.grad_lnlikelihood() * 1/x^2 * 2x\n # = 2 * gp.grad_lnlikelihood() / x\n # = 2 * gp.grad_lnlikelihood() / np.sqrt(x^2)\n # = 2 * gp.grad_lnlikelihood() / np.sqrt(gp.kernel.pars)\n #\n # (with a negative sign out front for the negative gradient).\n # So we probably weren't optimizing the GP correctly! This affects\n # all campaigns through C13. It's not a *huge* deal, since the sign\n # of the gradient was correct and the model isn't that sensitive to\n # the value of the hyperparameters, but it may have contributed to\n # the poor performance on super variable stars. In most cases it means\n # the solver takes longer to converge and isn't as good at finding\n # the minimum.\n ngr = -2 * gp.grad_lnlikelihood(flux) / np.sqrt(gp.kernel.pars)\n else:\n nll = -gp.log_likelihood(flux)\n ngr = -2 * gp.grad_log_likelihood(flux) / \\\n np.sqrt(np.exp(gp.get_parameter_vector()))\n\n return nll, ngr"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngive a ``startdate`` and an ``enddate`` dates, evaluate the date intervals from which data is not available. It return a list of two-dimensional tuples containing start and end date for the interval. The list could countain 0,1 or 2 tuples.", "response": "def missing_intervals(startdate, enddate, start, end,\r\n dateconverter=None,\r\n parseinterval=None,\r\n intervals=None):\r\n '''Given a ``startdate`` and an ``enddate`` dates, evaluate the\r\ndate intervals from which data is not available. It return a list of\r\ntwo-dimensional tuples containing start and end date for the interval.\r\nThe list could countain 0,1 or 2 tuples.'''\r\n parseinterval = parseinterval or default_parse_interval\r\n dateconverter = dateconverter or todate\r\n startdate = dateconverter(parseinterval(startdate, 0))\r\n enddate = max(startdate, dateconverter(parseinterval(enddate, 0)))\r\n if intervals is not None and not isinstance(intervals, Intervals):\r\n intervals = Intervals(intervals)\r\n\r\n calc_intervals = Intervals()\r\n # we have some history already\r\n if start:\r\n # the startdate not available\r\n if startdate < start:\r\n calc_start = startdate\r\n calc_end = parseinterval(start, -1)\r\n if calc_end >= calc_start:\r\n calc_intervals.append(Interval(calc_start, calc_end))\r\n\r\n if enddate > end:\r\n calc_start = parseinterval(end, 1)\r\n calc_end = enddate\r\n if calc_end >= calc_start:\r\n calc_intervals.append(Interval(calc_start, calc_end))\r\n else:\r\n start = startdate\r\n end = enddate\r\n calc_intervals.append(Interval(startdate, enddate))\r\n\r\n if calc_intervals:\r\n if intervals:\r\n calc_intervals.extend(intervals)\r\n elif intervals:\r\n calc_intervals = intervals\r\n\r\n return calc_intervals"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef dategenerator(start, end, step=1, desc=False):\r\n '''Generates dates between *atrt* and *end*.'''\r\n delta = timedelta(abs(step))\r\n end = max(start, end)\r\n if desc:\r\n dt = end\r\n while dt >= start:\r\n yield dt\r\n dt -= delta\r\n else:\r\n dt = start\r\n while dt <= end:\r\n yield dt\r\n dt += delta", "response": "Generates dates between start and end."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize the logging functionality.", "response": "def InitLog(file_name=None, log_level=logging.DEBUG,\n screen_level=logging.CRITICAL, pdb=False):\n '''\n A little routine to initialize the logging functionality.\n\n :param str file_name: The name of the file to log to. \\\n Default :py:obj:`None` (set internally by :py:mod:`everest`)\n :param int log_level: The file logging level (0-50). Default 10 (debug)\n :param int screen_level: The screen logging level (0-50). \\\n Default 50 (critical)\n\n '''\n\n # Initialize the logging\n root = logging.getLogger()\n root.handlers = []\n root.setLevel(logging.DEBUG)\n\n # File handler\n if file_name is not None:\n if not os.path.exists(os.path.dirname(file_name)):\n os.makedirs(os.path.dirname(file_name))\n fh = logging.FileHandler(file_name)\n fh.setLevel(log_level)\n fh_formatter = logging.Formatter(\n \"%(asctime)s %(levelname)-5s [%(name)s.%(funcName)s()]: %(message)s\",\n datefmt=\"%m/%d/%y %H:%M:%S\")\n fh.setFormatter(fh_formatter)\n fh.addFilter(NoPILFilter())\n root.addHandler(fh)\n\n # Screen handler\n sh = logging.StreamHandler(sys.stdout)\n if pdb:\n sh.setLevel(logging.DEBUG)\n else:\n sh.setLevel(screen_level)\n sh_formatter = logging.Formatter(\n \"%(levelname)-5s [%(name)s.%(funcName)s()]: %(message)s\")\n sh.setFormatter(sh_formatter)\n sh.addFilter(NoPILFilter())\n root.addHandler(sh)\n\n # Set exception hook\n if pdb:\n sys.excepthook = ExceptionHookPDB\n else:\n sys.excepthook = ExceptionHook"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sort_like(l, col1, col2):\n '''\n Sorts the list :py:obj:`l` by comparing :py:obj:`col2` to :py:obj:`col1`.\n Specifically, finds the indices :py:obj:`i` such that ``col2[i] = col1``\n and returns ``l[i]``. This is useful when comparing the CDPP values of\n catalogs generated by different pipelines. The\n target IDs are all the same, but won't necessarily be in the same order.\n This allows :py:obj:`everest` to sort the CDPP arrays so that the\n targets match.\n\n :param array_like l: The list or array to sort\n :param array_like col1: A list or array (same length as :py:obj:`l`)\n :param array_like col2: A second list or array containing the same \\\n elements as :py:obj:`col1` but in a different order\n\n '''\n\n s = np.zeros_like(col1) * np.nan\n for i, c in enumerate(col1):\n j = np.argmax(col2 == c)\n if j == 0:\n if col2[0] != c:\n continue\n s[i] = l[j]\n return s", "response": "Sorts the list l by comparing col1 to col2."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nprogressing bar range with tqdm", "response": "def prange(*x):\n '''\n Progress bar range with `tqdm`\n\n '''\n\n try:\n root = logging.getLogger()\n if len(root.handlers):\n for h in root.handlers:\n if (type(h) is logging.StreamHandler) and \\\n (h.level != logging.CRITICAL):\n from tqdm import tqdm\n return tqdm(range(*x))\n return range(*x)\n else:\n from tqdm import tqdm\n return tqdm(range(*x))\n except ImportError:\n return range(*x)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef front(self, *fields):\r\n '''Return the front pair of the structure'''\r\n ts = self.irange(0, 0, fields=fields)\r\n if ts:\r\n return ts.start(), ts[0]", "response": "Return the front pair of the structure"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the back pair of the structure", "response": "def back(self, *fields):\r\n '''Return the back pair of the structure'''\r\n ts = self.irange(-1, -1, fields=fields)\r\n if ts:\r\n return ts.end(), ts[0]"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconverts the backend into the database connection parameters. It returns a tuple of scheme host params.", "response": "def parse_backend(backend):\r\n \"\"\"Converts the \"backend\" into the database connection parameters.\r\nIt returns a (scheme, host, params) tuple.\"\"\"\r\n r = urlparse.urlsplit(backend)\r\n scheme, host = r.scheme, r.netloc\r\n path, query = r.path, r.query\r\n if path and not query:\r\n query, path = path, ''\r\n if query:\r\n if query.find('?'):\r\n path = query\r\n else:\r\n query = query[1:]\r\n if query:\r\n params = dict(urlparse.parse_qsl(query))\r\n else:\r\n params = {}\r\n\r\n return scheme, host, params"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget a : class : BackendDataServer.", "response": "def getdb(backend=None, **kwargs):\r\n '''get a :class:`BackendDataServer`.'''\r\n if isinstance(backend, BackendDataServer):\r\n return backend\r\n backend = backend or settings.DEFAULT_BACKEND\r\n if not backend:\r\n return None\r\n scheme, address, params = parse_backend(backend)\r\n params.update(kwargs)\r\n if 'timeout' in params:\r\n params['timeout'] = int(params['timeout'])\r\n return _getdb(scheme, address, params)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef basekey(self, meta, *args):\r\n key = '%s%s' % (self.namespace, meta.modelkey)\r\n postfix = ':'.join((str(p) for p in args if p is not None))\r\n return '%s:%s' % (key, postfix) if postfix else key", "response": "Calculate the basekey to access model data."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef make_objects(self, meta, data, related_fields=None):\r\n '''Generator of :class:`stdnet.odm.StdModel` instances with data\r\nfrom database.\r\n\r\n:parameter meta: instance of model :class:`stdnet.odm.Metaclass`.\r\n:parameter data: iterator over instances data.\r\n'''\r\n make_object = meta.make_object\r\n related_data = []\r\n if related_fields:\r\n for fname, fdata in iteritems(related_fields):\r\n field = meta.dfields[fname]\r\n if field in meta.multifields:\r\n related = dict(fdata)\r\n multi = True\r\n else:\r\n multi = False\r\n relmodel = field.relmodel\r\n related = dict(((obj.id, obj) for obj in\r\n self.make_objects(relmodel._meta, fdata)))\r\n related_data.append((field, related, multi))\r\n for state in data:\r\n instance = make_object(state, self)\r\n for field, rdata, multi in related_data:\r\n if multi:\r\n field.set_cache(instance, rdata.get(str(instance.id)))\r\n else:\r\n rid = getattr(instance, field.attname, None)\r\n if rid is not None:\r\n value = rdata.get(rid)\r\n setattr(instance, field.name, value)\r\n yield instance", "response": "Generator of objects with data\r\nfrom database."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a new backend : class : stdnet. odm. Structure handler.", "response": "def structure(self, instance, client=None):\r\n '''Create a backend :class:`stdnet.odm.Structure` handler.\r\n\r\n :param instance: a :class:`stdnet.odm.Structure`\r\n :param client: Optional client handler.\r\n '''\r\n struct = self.struct_map.get(instance._meta.name)\r\n if struct is None:\r\n raise ModelNotAvailable('\"%s\" is not available for backend '\r\n '\"%s\"' % (instance._meta.name, self))\r\n client = client if client is not None else self.client\r\n return struct(instance, self, client)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Search(ID, mission='k2'):\n # Only K2 supported for now\n assert mission == 'k2', \"Only the K2 mission is supported for now.\"\n print(\"Searching for target %d...\" % ID)\n\n # First check if it is in the database\n season = missions.k2.Season(ID)\n if season in [91, 92, [91, 92]]:\n print(\"Campaign 9 is currently not part of the EVEREST catalog.\")\n return\n elif season == 101:\n print(\"The first half of campaign 10 is not currently part of \" +\n \"the EVEREST catalog.\")\n return\n elif season is not None:\n print(\"Target is in campaign %d of the EVEREST catalog.\" % season)\n return\n\n # Get the kplr object\n star = k2plr_client.k2_star(ID)\n\n # First check if this is a star\n if star.objtype.lower() != \"star\":\n print(\"Target is of type %s, not STAR, \" % star.objtype +\n \"and is therefore not included in the EVEREST catalog.\")\n return\n\n # Let's try to download the pixel data and see what happens\n try:\n tpf = star.get_target_pixel_files()\n except:\n print(\"Unable to download the raw pixel files for this target.\")\n return\n if len(tpf) == 0:\n print(\"Raw pixel files are not available for this target. Looks like \" +\n \"data may not have been collected for it.\")\n return\n\n # Perhaps it's in a campaign we haven't gotten to yet\n if tpf[0].sci_campaign not in missions.k2.SEASONS:\n print(\"Targets for campaign %d are not yet available.\"\n % tpf[0].sci_campaign)\n return\n\n # Let's try to download the K2SFF data\n try:\n k2sff = k2plr.K2SFF(ID)\n except:\n print(\"Error downloading the K2SFF light curve for this target. \" +\n \"Currently, EVEREST uses the K2SFF apertures to perform \" +\n \"photometry. This is likely to change in the next version.\")\n return\n\n # Let's try to get the aperture\n try:\n assert np.count_nonzero(k2sff.apertures[15]), \"Invalid aperture.\"\n except:\n print(\"Unable to retrieve the K2SFF aperture for this target. \" +\n \"Currently, EVEREST uses the K2SFF apertures to perform \" +\n \"photometry. This is likely to change in the next version.\")\n return\n\n # Perhaps the star is *super* saturated and we didn't bother\n # de-trending it?\n if star.kp < 8:\n print(\"Target has Kp = %.1f and is too saturated \" +\n \"for proper de-trending with EVEREST.\")\n return\n\n # I'm out of ideas\n print(\"I'm not sure why this target isn't in the EVEREST catalog.\" +\n \"You can try de-trending it yourself:\")\n print(\"http://faculty.washington.edu/rodluger/everest/pipeline.html\")\n return", "response": "Search for a target in the EVEREST database."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndownloading a given file from MAST.", "response": "def DownloadFile(ID, season=None, mission='k2', cadence='lc',\n filename=None, clobber=False):\n '''\n Download a given :py:mod:`everest` file from MAST.\n\n :param str mission: The mission name. Default `k2`\n :param str cadence: The light curve cadence. Default `lc`\n :param str filename: The name of the file to download. Default \\\n :py:obj:`None`, in which case the default \\\n FITS file is retrieved.\n :param bool clobber: If :py:obj:`True`, download and overwrite \\\n existing files. Default :py:obj:`False`\n\n '''\n\n # Get season\n if season is None:\n season = getattr(missions, mission).Season(ID)\n if hasattr(season, '__len__'):\n raise AttributeError(\n \"Please choose a `season` for this target: %s.\" % season)\n if season is None:\n if getattr(missions, mission).ISTARGET(ID):\n raise ValueError('Target not found in local database. ' +\n 'Run `everest.Search(%d)` for more information.'\n % ID)\n else:\n raise ValueError('Invalid target ID.')\n path = getattr(missions, mission).TargetDirectory(ID, season)\n relpath = getattr(missions, mission).TargetDirectory(\n ID, season, relative=True)\n if filename is None:\n filename = getattr(missions, mission).FITSFile(ID, season, cadence)\n\n # Check if file exists\n if not os.path.exists(path):\n os.makedirs(path)\n elif os.path.exists(os.path.join(path, filename)) and not clobber:\n log.info('Found cached file.')\n return os.path.join(path, filename)\n\n # Get file URL\n log.info('Downloading the file...')\n fitsurl = getattr(missions, mission).FITSUrl(ID, season)\n if not fitsurl.endswith('/'):\n fitsurl += '/'\n\n # Download the data\n r = urllib.request.Request(fitsurl + filename)\n try:\n handler = urllib.request.urlopen(r)\n code = handler.getcode()\n except (urllib.error.HTTPError, urllib.error.URLError):\n code = 0\n if int(code) == 200:\n\n # Read the data\n data = handler.read()\n\n # Atomically save to disk\n f = NamedTemporaryFile(\"wb\", delete=False)\n f.write(data)\n f.flush()\n os.fsync(f.fileno())\n f.close()\n shutil.move(f.name, os.path.join(path, filename))\n\n else:\n\n # Something went wrong!\n log.error(\"Error code {0} for URL '{1}'\".format(\n code, fitsurl + filename))\n\n # If the files can be accessed by `ssh`, let's try that\n # (development version only!)\n if EVEREST_FITS is None:\n raise Exception(\"Unable to locate the file.\")\n\n # Get the url\n inpath = os.path.join(EVEREST_FITS, relpath, filename)\n outpath = os.path.join(path, filename)\n\n # Download the data\n log.info(\"Accessing file via `scp`...\")\n subprocess.call(['scp', inpath, outpath])\n\n # Success?\n if os.path.exists(os.path.join(path, filename)):\n return os.path.join(path, filename)\n else:\n raise Exception(\"Unable to download the file.\" +\n \"Run `everest.Search(%d)` to troubleshoot.\" % ID)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nshows the data validation summary for a given target.", "response": "def DVS(ID, season=None, mission='k2', clobber=False,\n cadence='lc', model='nPLD'):\n '''\n Show the data validation summary (DVS) for a given target.\n\n :param str mission: The mission name. Default `k2`\n :param str cadence: The light curve cadence. Default `lc`\n :param bool clobber: If :py:obj:`True`, download and overwrite \\\n existing files. Default :py:obj:`False`\n\n '''\n\n # Get season\n if season is None:\n season = getattr(missions, mission).Season(ID)\n if hasattr(season, '__len__'):\n raise AttributeError(\n \"Please choose a `season` for this target: %s.\" % season)\n\n # Get file name\n if model == 'nPLD':\n filename = getattr(missions, mission).DVSFile(ID, season, cadence)\n else:\n if cadence == 'sc':\n filename = model + '.sc.pdf'\n else:\n filename = model + '.pdf'\n\n file = DownloadFile(ID, season=season,\n mission=mission,\n filename=filename,\n clobber=clobber)\n\n try:\n if platform.system().lower().startswith('darwin'):\n subprocess.call(['open', file])\n elif os.name == 'nt':\n os.startfile(file)\n elif os.name == 'posix':\n subprocess.call(['xdg-open', file])\n else:\n raise Exception(\"\")\n except:\n log.info(\"Unable to open the pdf. Try opening it manually:\")\n log.info(file)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef compute(self):\n '''\n Re-compute the :py:mod:`everest` model for the given\n value of :py:obj:`lambda`.\n For long cadence `k2` light curves, this should take several\n seconds. For short cadence `k2` light curves, it may take a\n few minutes. Note that this is a simple wrapper around\n :py:func:`everest.Basecamp.compute`.\n\n '''\n\n # If we're doing iterative PLD, get the normalization\n if self.model_name == 'iPLD':\n self._get_norm()\n\n # Compute as usual\n super(Everest, self).compute()\n\n # Make NaN cadences NaNs\n self.flux[self.nanmask] = np.nan", "response": "Compute the current cadence model for the given base cadence class."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_norm(self):\n '''\n Computes the PLD flux normalization array.\n\n ..note :: `iPLD` model **only**.\n\n '''\n\n log.info('Computing the PLD normalization...')\n\n # Loop over all chunks\n mod = [None for b in self.breakpoints]\n for b, brkpt in enumerate(self.breakpoints):\n\n # Unmasked chunk\n c = self.get_chunk(b)\n\n # Masked chunk (original mask plus user transit mask)\n inds = np.array(\n list(set(np.concatenate([self.transitmask,\n self.recmask]))), dtype=int)\n M = np.delete(np.arange(len(self.time)), inds, axis=0)\n if b > 0:\n m = M[(M > self.breakpoints[b - 1] - self.bpad)\n & (M <= self.breakpoints[b] + self.bpad)]\n else:\n m = M[M <= self.breakpoints[b] + self.bpad]\n\n # This block of the masked covariance matrix\n mK = GetCovariance(self.kernel, self.kernel_params,\n self.time[m], self.fraw_err[m])\n\n # Get median\n med = np.nanmedian(self.fraw[m])\n\n # Normalize the flux\n f = self.fraw[m] - med\n\n # The X^2 matrices\n A = np.zeros((len(m), len(m)))\n B = np.zeros((len(c), len(m)))\n\n # Loop over all orders\n for n in range(self.pld_order):\n XM = self.X(n, m)\n XC = self.X(n, c)\n A += self.reclam[b][n] * np.dot(XM, XM.T)\n B += self.reclam[b][n] * np.dot(XC, XM.T)\n del XM, XC\n\n W = np.linalg.solve(mK + A, f)\n mod[b] = np.dot(B, W)\n del A, B, W\n\n # Join the chunks after applying the correct offset\n if len(mod) > 1:\n\n # First chunk\n model = mod[0][:-self.bpad]\n\n # Center chunks\n for m in mod[1:-1]:\n offset = model[-1] - m[self.bpad - 1]\n model = np.concatenate(\n [model, m[self.bpad:-self.bpad] + offset])\n\n # Last chunk\n offset = model[-1] - mod[-1][self.bpad - 1]\n model = np.concatenate([model, mod[-1][self.bpad:] + offset])\n\n else:\n\n model = mod[0]\n\n # Subtract the global median\n model -= np.nanmedian(model)\n\n # Save the norm\n self._norm = self.fraw - model", "response": "Compute the PLD flux normalization array."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads the FITS file and populate the object with its data.", "response": "def load_fits(self):\n '''\n Load the FITS file from disk and populate the\n class instance with its data.\n\n '''\n\n log.info(\"Loading FITS file for %d.\" % (self.ID))\n with pyfits.open(self.fitsfile) as f:\n\n # Params and long cadence data\n self.loaded = True\n self.is_parent = False\n try:\n self.X1N = f[2].data['X1N']\n except KeyError:\n self.X1N = None\n self.aperture = f[3].data\n self.aperture_name = f[1].header['APNAME']\n try:\n self.bkg = f[1].data['BKG']\n except KeyError:\n self.bkg = 0.\n self.bpad = f[1].header['BPAD']\n self.cbv_minstars = []\n self.cbv_num = f[1].header.get('CBVNUM', 1)\n self.cbv_niter = f[1].header['CBVNITER']\n self.cbv_win = f[1].header['CBVWIN']\n self.cbv_order = f[1].header['CBVORD']\n self.cadn = f[1].data['CADN']\n self.cdivs = f[1].header['CDIVS']\n self.cdpp = f[1].header['CDPP']\n self.cdppr = f[1].header['CDPPR']\n self.cdppv = f[1].header['CDPPV']\n self.cdppg = f[1].header['CDPPG']\n self.cv_min = f[1].header['CVMIN']\n self.fpix = f[2].data['FPIX']\n self.pixel_images = [f[4].data['STAMP1'],\n f[4].data['STAMP2'], f[4].data['STAMP3']]\n self.fraw = f[1].data['FRAW']\n self.fraw_err = f[1].data['FRAW_ERR']\n self.giter = f[1].header['GITER']\n self.gmaxf = f[1].header.get('GMAXF', 200)\n self.gp_factor = f[1].header['GPFACTOR']\n try:\n self.hires = f[5].data\n except:\n self.hires = None\n self.kernel_params = np.array([f[1].header['GPWHITE'],\n f[1].header['GPRED'],\n f[1].header['GPTAU']])\n try:\n self.kernel = f[1].header['KERNEL']\n self.kernel_params = np.append(\n self.kernel_params,\n [f[1].header['GPGAMMA'],\n f[1].header['GPPER']])\n except KeyError:\n self.kernel = 'Basic'\n self.pld_order = f[1].header['PLDORDER']\n self.lam_idx = self.pld_order\n self.leps = f[1].header['LEPS']\n self.mag = f[0].header['KEPMAG']\n self.max_pixels = f[1].header['MAXPIX']\n self.model = self.fraw - f[1].data['FLUX']\n self.nearby = []\n for i in range(99):\n try:\n ID = f[1].header['NRBY%02dID' % (i + 1)]\n x = f[1].header['NRBY%02dX' % (i + 1)]\n y = f[1].header['NRBY%02dY' % (i + 1)]\n mag = f[1].header['NRBY%02dM' % (i + 1)]\n x0 = f[1].header['NRBY%02dX0' % (i + 1)]\n y0 = f[1].header['NRBY%02dY0' % (i + 1)]\n self.nearby.append(\n {'ID': ID, 'x': x, 'y': y,\n 'mag': mag, 'x0': x0, 'y0': y0})\n except KeyError:\n break\n self.neighbors = []\n for c in range(99):\n try:\n self.neighbors.append(f[1].header['NEIGH%02d' % (c + 1)])\n except KeyError:\n break\n self.oiter = f[1].header['OITER']\n self.optimize_gp = f[1].header['OPTGP']\n self.osigma = f[1].header['OSIGMA']\n self.planets = []\n for i in range(99):\n try:\n t0 = f[1].header['P%02dT0' % (i + 1)]\n per = f[1].header['P%02dPER' % (i + 1)]\n dur = f[1].header['P%02dDUR' % (i + 1)]\n self.planets.append((t0, per, dur))\n except KeyError:\n break\n self.quality = f[1].data['QUALITY']\n self.saturated = f[1].header['SATUR']\n self.saturation_tolerance = f[1].header['SATTOL']\n self.time = f[1].data['TIME']\n self._norm = np.array(self.fraw)\n\n # Chunk arrays\n self.breakpoints = []\n self.cdpp_arr = []\n self.cdppv_arr = []\n self.cdppr_arr = []\n for c in range(99):\n try:\n self.breakpoints.append(f[1].header['BRKPT%02d' % (c + 1)])\n self.cdpp_arr.append(f[1].header['CDPP%02d' % (c + 1)])\n self.cdppr_arr.append(f[1].header['CDPPR%02d' % (c + 1)])\n self.cdppv_arr.append(f[1].header['CDPPV%02d' % (c + 1)])\n except KeyError:\n break\n self.lam = [[f[1].header['LAMB%02d%02d' % (c + 1, o + 1)]\n for o in range(self.pld_order)]\n for c in range(len(self.breakpoints))]\n if self.model_name == 'iPLD':\n self.reclam = [[f[1].header['RECL%02d%02d' % (c + 1, o + 1)]\n for o in range(self.pld_order)]\n for c in range(len(self.breakpoints))]\n\n # Masks\n self.badmask = np.where(self.quality & 2 ** (QUALITY_BAD - 1))[0]\n self.nanmask = np.where(self.quality & 2 ** (QUALITY_NAN - 1))[0]\n self.outmask = np.where(self.quality & 2 ** (QUALITY_OUT - 1))[0]\n self.recmask = np.where(self.quality & 2 ** (QUALITY_REC - 1))[0]\n self.transitmask = np.where(\n self.quality & 2 ** (QUALITY_TRN - 1))[0]\n\n # CBVs\n self.XCBV = np.empty((len(self.time), 0))\n for i in range(99):\n try:\n self.XCBV = np.hstack(\n [self.XCBV,\n f[1].data['CBV%02d' % (i + 1)].reshape(-1, 1)])\n except KeyError:\n break\n\n # These are not stored in the fits file; we don't need them\n self.saturated_aperture_name = None\n self.apertures = None\n self.Xpos = None\n self.Ypos = None\n self.fpix_err = None\n self.parent_model = None\n self.lambda_arr = None\n self.meta = None\n self._transit_model = None\n self.transit_depth = None"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nplots sample postage stamps for the target with the aperture outline marked as well as a high - res target image.", "response": "def plot_aperture(self, show=True):\n '''\n Plot sample postage stamps for the target with the aperture\n outline marked, as well as a high-res target image (if available).\n\n :param bool show: Show the plot or return the `(fig, ax)` instance? \\\n Default :py:obj:`True`\n\n '''\n\n # Set up the axes\n fig, ax = pl.subplots(2, 2, figsize=(6, 8))\n fig.subplots_adjust(top=0.975, bottom=0.025, left=0.05,\n right=0.95, hspace=0.05, wspace=0.05)\n ax = ax.flatten()\n fig.canvas.set_window_title(\n '%s %d' % (self._mission.IDSTRING, self.ID))\n super(Everest, self).plot_aperture(ax, labelsize=12)\n\n if show:\n pl.show()\n pl.close()\n else:\n return fig, ax"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef plot(self, show=True, plot_raw=True, plot_gp=True,\n plot_bad=True, plot_out=True, plot_cbv=True,\n simple=False):\n '''\n Plots the final de-trended light curve.\n\n :param bool show: Show the plot or return the `(fig, ax)` instance? \\\n Default :py:obj:`True`\n :param bool plot_raw: Show the raw light curve? Default :py:obj:`True`\n :param bool plot_gp: Show the GP model prediction? \\\n Default :py:obj:`True`\n :param bool plot_bad: Show and indicate the bad data points? \\\n Default :py:obj:`True`\n :param bool plot_out: Show and indicate the outliers? \\\n Default :py:obj:`True`\n :param bool plot_cbv: Plot the CBV-corrected light curve? \\\n Default :py:obj:`True`. If :py:obj:`False`, plots the \\\n de-trended but uncorrected light curve.\n\n '''\n\n log.info('Plotting the light curve...')\n\n # Set up axes\n if plot_raw:\n fig, axes = pl.subplots(2, figsize=(13, 9), sharex=True)\n fig.subplots_adjust(hspace=0.1)\n axes = [axes[1], axes[0]]\n if plot_cbv:\n fluxes = [self.fcor, self.fraw]\n else:\n fluxes = [self.flux, self.fraw]\n labels = ['EVEREST Flux', 'Raw Flux']\n else:\n fig, axes = pl.subplots(1, figsize=(13, 6))\n axes = [axes]\n if plot_cbv:\n fluxes = [self.fcor]\n else:\n fluxes = [self.flux]\n labels = ['EVEREST Flux']\n fig.canvas.set_window_title('EVEREST Light curve')\n\n # Set up some stuff\n time = self.time\n badmask = self.badmask\n nanmask = self.nanmask\n outmask = self.outmask\n transitmask = self.transitmask\n fraw_err = self.fraw_err\n breakpoints = self.breakpoints\n if self.cadence == 'sc':\n ms = 2\n else:\n ms = 4\n\n # Get the cdpps\n cdpps = [[self.get_cdpp(self.flux), self.get_cdpp_arr(self.flux)],\n [self.get_cdpp(self.fraw), self.get_cdpp_arr(self.fraw)]]\n self.cdpp = cdpps[0][0]\n self.cdpp_arr = cdpps[0][1]\n\n for n, ax, flux, label, c in zip([0, 1], axes, fluxes, labels, cdpps):\n\n # Initialize CDPP\n cdpp = c[0]\n cdpp_arr = c[1]\n\n # Plot the good data points\n ax.plot(self.apply_mask(time), self.apply_mask(flux),\n ls='none', marker='.', color='k', markersize=ms, alpha=0.5)\n\n # Plot the outliers\n bnmask = np.array(\n list(set(np.concatenate([badmask, nanmask]))), dtype=int)\n bmask = [i for i in self.badmask if i not in self.nanmask]\n\n def O1(x): return x[outmask]\n\n def O2(x): return x[bmask]\n\n def O3(x): return x[transitmask]\n if plot_out:\n ax.plot(O1(time), O1(flux), ls='none', color=\"#777777\",\n marker='.', markersize=ms, alpha=0.5)\n if plot_bad:\n ax.plot(O2(time), O2(flux), 'r.', markersize=ms, alpha=0.25)\n ax.plot(O3(time), O3(flux), 'b.', markersize=ms, alpha=0.25)\n\n # Plot the GP\n if n == 0 and plot_gp and self.cadence != 'sc':\n gp = GP(self.kernel, self.kernel_params)\n gp.compute(self.apply_mask(time), self.apply_mask(fraw_err))\n med = np.nanmedian(self.apply_mask(flux))\n y, _ = gp.predict(self.apply_mask(flux) - med, time)\n y += med\n ax.plot(self.apply_mask(time), self.apply_mask(\n y), 'r-', lw=0.5, alpha=0.5)\n\n # Appearance\n if n == 0:\n ax.set_xlabel('Time (%s)' %\n self._mission.TIMEUNITS, fontsize=18)\n ax.set_ylabel(label, fontsize=18)\n for brkpt in breakpoints[:-1]:\n ax.axvline(time[brkpt], color='r', ls='--', alpha=0.25)\n if len(cdpp_arr) == 2:\n ax.annotate('%.2f ppm' % cdpp_arr[0], xy=(0.02, 0.975),\n xycoords='axes fraction',\n ha='left', va='top', fontsize=12, color='r',\n zorder=99)\n ax.annotate('%.2f ppm' % cdpp_arr[1], xy=(0.98, 0.975),\n xycoords='axes fraction',\n ha='right', va='top', fontsize=12,\n color='r', zorder=99)\n elif len(cdpp_arr) < 6:\n for n in range(len(cdpp_arr)):\n if n > 0:\n x = (self.time[self.breakpoints[n - 1]] - self.time[0]\n ) / (self.time[-1] - self.time[0]) + 0.02\n else:\n x = 0.02\n ax.annotate('%.2f ppm' % cdpp_arr[n], xy=(x, 0.975),\n xycoords='axes fraction',\n ha='left', va='top', fontsize=10,\n zorder=99, color='r')\n else:\n ax.annotate('%.2f ppm' % cdpp, xy=(0.02, 0.975),\n xycoords='axes fraction',\n ha='left', va='top', fontsize=12,\n color='r', zorder=99)\n ax.margins(0.01, 0.1)\n\n # Get y lims that bound 99% of the flux\n f = np.concatenate([np.delete(f, bnmask) for f in fluxes])\n N = int(0.995 * len(f))\n hi, lo = f[np.argsort(f)][[N, -N]]\n pad = (hi - lo) * 0.1\n ylim = (lo - pad, hi + pad)\n ax.set_ylim(ylim)\n ax.get_yaxis().set_major_formatter(Formatter.Flux)\n\n # Indicate off-axis outliers\n for i in np.where(flux < ylim[0])[0]:\n if i in bmask:\n color = \"#ffcccc\"\n if not plot_bad:\n continue\n elif i in outmask:\n color = \"#cccccc\"\n if not plot_out:\n continue\n elif i in nanmask:\n continue\n else:\n color = \"#ccccff\"\n ax.annotate('', xy=(time[i], ylim[0]), xycoords='data',\n xytext=(0, 15), textcoords='offset points',\n arrowprops=dict(arrowstyle=\"-|>\", color=color))\n for i in np.where(flux > ylim[1])[0]:\n if i in bmask:\n color = \"#ffcccc\"\n if not plot_bad:\n continue\n elif i in outmask:\n color = \"#cccccc\"\n if not plot_out:\n continue\n elif i in nanmask:\n continue\n else:\n color = \"#ccccff\"\n ax.annotate('', xy=(time[i], ylim[1]), xycoords='data',\n xytext=(0, -15), textcoords='offset points',\n arrowprops=dict(arrowstyle=\"-|>\", color=color))\n\n # Show total CDPP improvement\n pl.figtext(0.5, 0.94, '%s %d' % (self._mission.IDSTRING, self.ID),\n fontsize=18, ha='center', va='bottom')\n pl.figtext(0.5, 0.905,\n r'$%.2f\\ \\mathrm{ppm} \\rightarrow %.2f\\ \\mathrm{ppm}$' %\n (self.cdppr, self.cdpp), fontsize=14,\n ha='center', va='bottom')\n\n if show:\n pl.show()\n pl.close()\n else:\n if plot_raw:\n return fig, axes\n else:\n return fig, axes[0]", "response": "Plot the final de - trended light curve."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef dvs(self):\n '''\n Shows the data validation summary (DVS) for the target.\n\n '''\n\n DVS(self.ID, season=self.season, mission=self.mission,\n model=self.model_name, clobber=self.clobber)", "response": "Shows the data validation summary for the target."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nplots the light curve for the target de - trended with a given pipeline.", "response": "def plot_pipeline(self, pipeline, *args, **kwargs):\n '''\n Plots the light curve for the target de-trended with a given pipeline.\n\n :param str pipeline: The name of the pipeline (lowercase). Options \\\n are 'everest2', 'everest1', and other mission-specific \\\n pipelines. For `K2`, the available pipelines are 'k2sff' \\\n and 'k2sc'.\n\n Additional :py:obj:`args` and :py:obj:`kwargs` are passed directly to\n the :py:func:`pipelines.plot` function of the mission.\n\n '''\n\n if pipeline != 'everest2':\n return getattr(missions, self.mission).pipelines.plot(self.ID,\n pipeline,\n *args,\n **kwargs)\n\n else:\n\n # We're going to plot the everest 2 light curve like we plot\n # the other pipelines for easy comparison\n plot_raw = kwargs.get('plot_raw', False)\n plot_cbv = kwargs.get('plot_cbv', True)\n show = kwargs.get('show', True)\n\n if plot_raw:\n y = self.fraw\n ylabel = 'Raw Flux'\n elif plot_cbv:\n y = self.fcor\n ylabel = \"EVEREST2 Flux\"\n else:\n y = self.flux\n ylabel = \"EVEREST2 Flux\"\n\n # Remove nans\n bnmask = np.concatenate([self.nanmask, self.badmask])\n time = np.delete(self.time, bnmask)\n flux = np.delete(y, bnmask)\n\n # Plot it\n fig, ax = pl.subplots(1, figsize=(10, 4))\n fig.subplots_adjust(bottom=0.15)\n ax.plot(time, flux, \"k.\", markersize=3, alpha=0.5)\n\n # Axis limits\n N = int(0.995 * len(flux))\n hi, lo = flux[np.argsort(flux)][[N, -N]]\n pad = (hi - lo) * 0.1\n ylim = (lo - pad, hi + pad)\n ax.set_ylim(ylim)\n\n # Plot bad data points\n ax.plot(self.time[self.badmask], y[self.badmask],\n \"r.\", markersize=3, alpha=0.2)\n\n # Show the CDPP\n ax.annotate('%.2f ppm' % self._mission.CDPP(flux),\n xy=(0.98, 0.975), xycoords='axes fraction',\n ha='right', va='top', fontsize=12, color='r',\n zorder=99)\n\n # Appearance\n ax.margins(0, None)\n ax.set_xlabel(\"Time (%s)\" % self._mission.TIMEUNITS, fontsize=16)\n ax.set_ylabel(ylabel, fontsize=16)\n fig.canvas.set_window_title(\"EVEREST2: EPIC %d\" % (self.ID))\n\n if show:\n pl.show()\n pl.close()\n else:\n return fig, ax"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_pipeline(self, *args, **kwargs):\n '''\n Returns the `time` and `flux` arrays for the target obtained by a given\n pipeline.\n\n Options :py:obj:`args` and :py:obj:`kwargs` are passed directly to\n the :py:func:`pipelines.get` function of the mission.\n\n '''\n\n return getattr(missions, self.mission).pipelines.get(self.ID, *args,\n **kwargs)", "response": "Returns the time and flux arrays for the target obtained by a given\n pipeline."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmask all of the transits and eclipses of a given planet.", "response": "def mask_planet(self, t0, period, dur=0.2):\n '''\n Mask all of the transits/eclipses of a given planet/EB. After calling\n this method, you must re-compute the model by calling\n :py:meth:`compute` in order for the mask to take effect.\n\n :param float t0: The time of first transit (same units as light curve)\n :param float period: The period of the planet in days\n :param foat dur: The transit duration in days. Default 0.2\n\n '''\n\n mask = []\n t0 += np.ceil((self.time[0] - dur - t0) / period) * period\n for t in np.arange(t0, self.time[-1] + dur, period):\n mask.extend(np.where(np.abs(self.time - t) < dur / 2.)[0])\n self.transitmask = np.array(\n list(set(np.concatenate([self.transitmask, mask]))))"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nplots the weights for the current light.", "response": "def _plot_weights(self, show=True):\n '''\n .. warning:: Untested!\n\n '''\n\n # Set up the axes\n fig = pl.figure(figsize=(12, 12))\n fig.subplots_adjust(top=0.95, bottom=0.025, left=0.1, right=0.92)\n fig.canvas.set_window_title(\n '%s %d' % (self._mission.IDSTRING, self.ID))\n ax = [pl.subplot2grid((80, 130), (20 * j, 25 * i), colspan=23,\n rowspan=18)\n for j in range(len(self.breakpoints) * 2)\n for i in range(1 + 2 * (self.pld_order - 1))]\n cax = [pl.subplot2grid((80, 130),\n (20 * j, 25 * (1 + 2 * (self.pld_order - 1))),\n colspan=4, rowspan=18)\n for j in range(len(self.breakpoints) * 2)]\n ax = np.array(ax).reshape(2 * len(self.breakpoints), -1)\n cax = np.array(cax)\n\n # Check number of segments\n if len(self.breakpoints) > 3:\n log.error('Cannot currently plot weights for light ' +\n 'curves with more than 3 segments.')\n return\n\n # Loop over all PLD orders and over all chunks\n npix = len(self.fpix[1])\n ap = self.aperture.flatten()\n ncol = 1 + 2 * (len(self.weights[0]) - 1)\n raw_weights = np.zeros(\n (len(self.breakpoints), ncol, self.aperture.shape[0],\n self.aperture.shape[1]), dtype=float)\n scaled_weights = np.zeros(\n (len(self.breakpoints), ncol, self.aperture.shape[0],\n self.aperture.shape[1]), dtype=float)\n\n # Loop over orders\n for o in range(len(self.weights[0])):\n if o == 0:\n oi = 0\n else:\n oi = 1 + 2 * (o - 1)\n\n # Loop over chunks\n for b in range(len(self.weights)):\n\n c = self.get_chunk(b)\n rw_ii = np.zeros(npix)\n rw_ij = np.zeros(npix)\n sw_ii = np.zeros(npix)\n sw_ij = np.zeros(npix)\n X = np.nanmedian(self.X(o, c), axis=0)\n\n # Compute all sets of pixels at this PLD order, then\n # loop over them and assign the weights to the correct pixels\n sets = np.array(list(multichoose(np.arange(npix).T, o + 1)))\n for i, s in enumerate(sets):\n if (o == 0) or (s[0] == s[1]):\n # Not the cross-terms\n j = s[0]\n rw_ii[j] += self.weights[b][o][i]\n sw_ii[j] += X[i] * self.weights[b][o][i]\n else:\n # Cross-terms\n for j in s:\n rw_ij[j] += self.weights[b][o][i]\n sw_ij[j] += X[i] * self.weights[b][o][i]\n\n # Make the array 2D and plot it\n rw = np.zeros_like(ap, dtype=float)\n sw = np.zeros_like(ap, dtype=float)\n n = 0\n for i, a in enumerate(ap):\n if (a & 1):\n rw[i] = rw_ii[n]\n sw[i] = sw_ii[n]\n n += 1\n raw_weights[b][oi] = rw.reshape(*self.aperture.shape)\n scaled_weights[b][oi] = sw.reshape(*self.aperture.shape)\n\n if o > 0:\n # Make the array 2D and plot it\n rw = np.zeros_like(ap, dtype=float)\n sw = np.zeros_like(ap, dtype=float)\n n = 0\n for i, a in enumerate(ap):\n if (a & 1):\n rw[i] = rw_ij[n]\n sw[i] = sw_ij[n]\n n += 1\n raw_weights[b][oi + 1] = rw.reshape(*self.aperture.shape)\n scaled_weights[b][oi +\n 1] = sw.reshape(*self.aperture.shape)\n\n # Plot the images\n log.info('Plotting the PLD weights...')\n rdbu = pl.get_cmap('RdBu_r')\n rdbu.set_bad('k')\n for b in range(len(self.weights)):\n rmax = max([-raw_weights[b][o].min() for o in range(ncol)] +\n [raw_weights[b][o].max() for o in range(ncol)])\n smax = max([-scaled_weights[b][o].min() for o in range(ncol)] +\n [scaled_weights[b][o].max() for o in range(ncol)])\n for o in range(ncol):\n imr = ax[2 * b, o].imshow(raw_weights[b][o], aspect='auto',\n interpolation='nearest', cmap=rdbu,\n origin='lower', vmin=-rmax,\n vmax=rmax)\n ims = ax[2 * b + 1, o].imshow(scaled_weights[b][o],\n aspect='auto',\n interpolation='nearest',\n cmap=rdbu, origin='lower',\n vmin=-smax, vmax=smax)\n\n # Colorbars\n def fmt(x, pos):\n a, b = '{:.0e}'.format(x).split('e')\n b = int(b)\n if float(a) > 0:\n a = r'+' + a\n elif float(a) == 0:\n return ''\n return r'${} \\times 10^{{{}}}$'.format(a, b)\n cbr = pl.colorbar(imr, cax=cax[2 * b], format=FuncFormatter(fmt))\n cbr.ax.tick_params(labelsize=8)\n cbs = pl.colorbar(\n ims, cax=cax[2 * b + 1], format=FuncFormatter(fmt))\n cbs.ax.tick_params(labelsize=8)\n\n # Plot aperture contours\n def PadWithZeros(vector, pad_width, iaxis, kwargs):\n vector[:pad_width[0]] = 0\n vector[-pad_width[1]:] = 0\n return vector\n ny, nx = self.aperture.shape\n contour = np.zeros((ny, nx))\n contour[np.where(self.aperture)] = 1\n contour = np.lib.pad(contour, 1, PadWithZeros)\n highres = zoom(contour, 100, order=0, mode='nearest')\n extent = np.array([-1, nx, -1, ny])\n for axis in ax.flatten():\n axis.contour(highres, levels=[\n 0.5], extent=extent, origin='lower', colors='r',\n linewidths=1)\n\n # Check for saturated columns\n for x in range(self.aperture.shape[0]):\n for y in range(self.aperture.shape[1]):\n if self.aperture[x][y] == AP_SATURATED_PIXEL:\n axis.fill([y - 0.5, y + 0.5, y + 0.5, y - 0.5],\n [x - 0.5, x - 0.5, x + 0.5, x + 0.5],\n fill=False, hatch='xxxxx', color='r', lw=0)\n\n axis.set_xlim(-0.5, nx - 0.5)\n axis.set_ylim(-0.5, ny - 0.5)\n axis.set_xticks([])\n axis.set_yticks([])\n\n # Labels\n titles = [r'$1^{\\mathrm{st}}$',\n r'$2^{\\mathrm{nd}}\\ (i = j)$',\n r'$2^{\\mathrm{nd}}\\ (i \\neq j)$',\n r'$3^{\\mathrm{rd}}\\ (i = j)$',\n r'$3^{\\mathrm{rd}}\\ (i \\neq j)$'] + ['' for i in range(10)]\n for i, axis in enumerate(ax[0]):\n axis.set_title(titles[i], fontsize=12)\n for j in range(len(self.weights)):\n ax[2 * j, 0].text(-0.55, -0.15, r'$%d$' % (j + 1),\n fontsize=16, transform=ax[2 * j, 0].transAxes)\n ax[2 * j, 0].set_ylabel(r'$w_{ij}$', fontsize=18)\n ax[2 * j + 1,\n 0].set_ylabel(r'$\\bar{X}_{ij} \\cdot w_{ij}$', fontsize=18)\n\n if show:\n pl.show()\n pl.close()\n else:\n return fig, ax, cax"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _save_npz(self):\n '''\n Saves all of the de-trending information to disk in an `npz` file\n\n '''\n\n # Save the data\n d = dict(self.__dict__)\n d.pop('_weights', None)\n d.pop('_A', None)\n d.pop('_B', None)\n d.pop('_f', None)\n d.pop('_mK', None)\n d.pop('K', None)\n d.pop('dvs', None)\n d.pop('clobber', None)\n d.pop('clobber_tpf', None)\n d.pop('_mission', None)\n d.pop('debug', None)\n np.savez(os.path.join(self.dir, self.name + '.npz'), **d)", "response": "Saves all of the de - trending information to disk in an npz file\n "} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef optimize(self, piter=3, pmaxf=300, ppert=0.1):\n '''\n Runs :py:obj:`pPLD` on the target in an attempt to further optimize the\n values of the PLD priors. See :py:class:`everest.detrender.pPLD`.\n\n '''\n\n self._save_npz()\n optimized = pPLD(self.ID, piter=piter, pmaxf=pmaxf,\n ppert=ppert, debug=True, clobber=True)\n optimized.publish()\n self.reset()", "response": "Optimize the current set of priors."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef plot_folded(self, t0, period, dur=0.2):\n '''\n Plot the light curve folded on a given `period` and centered at `t0`.\n When plotting folded transits, please mask them using\n :py:meth:`mask_planet` and re-compute the model using\n :py:meth:`compute`.\n\n :param float t0: The time at which to center the plot \\\n (same units as light curve)\n :param float period: The period of the folding operation\n :param float dur: The transit duration in days. Default 0.2\n\n '''\n\n # Mask the planet\n self.mask_planet(t0, period, dur)\n\n # Whiten\n gp = GP(self.kernel, self.kernel_params, white=False)\n gp.compute(self.apply_mask(self.time), self.apply_mask(self.fraw_err))\n med = np.nanmedian(self.apply_mask(self.flux))\n y, _ = gp.predict(self.apply_mask(self.flux) - med, self.time)\n fwhite = (self.flux - y)\n fwhite /= np.nanmedian(fwhite)\n\n # Fold\n tfold = (self.time - t0 - period / 2.) % period - period / 2.\n\n # Crop\n inds = np.where(np.abs(tfold) < 2 * dur)[0]\n x = tfold[inds]\n y = fwhite[inds]\n\n # Plot\n fig, ax = pl.subplots(1, figsize=(9, 5))\n fig.subplots_adjust(bottom=0.125)\n ax.plot(x, y, 'k.', alpha=0.5)\n\n # Get ylims\n yfin = np.delete(y, np.where(np.isnan(y)))\n lo, hi = yfin[np.argsort(yfin)][[3, -3]]\n pad = (hi - lo) * 0.1\n ylim = (lo - pad, hi + pad)\n ax.set_ylim(*ylim)\n\n # Appearance\n ax.set_xlabel(r'Time (days)', fontsize=18)\n ax.set_ylabel(r'Normalized Flux', fontsize=18)\n fig.canvas.set_window_title(\n '%s %d' % (self._mission.IDSTRING, self.ID))\n\n pl.show()", "response": "Plot the light curve folded on a given period and centered at t0."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nplot the light curve de - trended with a join instrumental + transit model overlaid.", "response": "def plot_transit_model(self, show=True, fold=None, ax=None):\n '''\n Plot the light curve de-trended with a join instrumental + transit\n model with the best fit transit model overlaid. The transit model\n should be specified using the :py:obj:`transit_model` attribute\n and should be an instance or list of instances of\n :py:class:`everest.transit.TransitModel`.\n\n :param bool show: Show the plot, or return the `fig, ax` instances? \\\n Default `True`\n :param str fold: The name of the planet/transit model on which to \\\n fold. If only one model is present, can be set to \\\n :py:obj:`True`. Default :py:obj:`False` \\\n (does not fold the data).\n :param ax: A `matplotlib` axis instance to use for plotting. \\\n Default :py:obj:`None`\n\n '''\n\n if self.transit_model is None:\n raise ValueError(\"No transit model provided!\")\n if self.transit_depth is None:\n self.compute()\n if fold is not None:\n if (fold is True and len(self.transit_model) > 1) or \\\n (type(fold) is not str):\n raise Exception(\n \"Kwarg `fold` should be the name of the transit \" +\n \"model on which to fold the data.\")\n if fold is True:\n # We are folding on the first index of `self.transit_model`\n fold = 0\n elif type(fold) is str:\n # Figure out the index of the transit model on which to fold\n fold = np.argmax(\n [fold == tm.name for tm in self.transit_model])\n log.info('Plotting the transit model folded ' +\n 'on transit model index %d...' % fold)\n else:\n log.info('Plotting the transit model...')\n\n # Set up axes\n if ax is None:\n if fold is not None:\n fig, ax = pl.subplots(1, figsize=(8, 5))\n else:\n fig, ax = pl.subplots(1, figsize=(13, 6))\n fig.canvas.set_window_title('EVEREST Light curve')\n else:\n fig = pl.gcf()\n\n # Set up some stuff\n if self.cadence == 'sc':\n ms = 2\n else:\n ms = 4\n\n # Fold?\n if fold is not None:\n times = self.transit_model[fold].params.get('times', None)\n if times is not None:\n time = self.time - \\\n [times[np.argmin(np.abs(ti - times))] for ti in self.time]\n t0 = times[0]\n else:\n t0 = self.transit_model[fold].params.get('t0', 0.)\n period = self.transit_model[fold].params.get('per', 10.)\n time = (self.time - t0 - period / 2.) % period - period / 2.\n dur = 0.01 * \\\n len(np.where(self.transit_model[fold](\n np.linspace(t0 - 0.5, t0 + 0.5, 100)) < 0)[0])\n else:\n time = self.time\n ax.plot(self.apply_mask(time), self.apply_mask(self.flux),\n ls='none', marker='.', color='k', markersize=ms, alpha=0.5)\n ax.plot(time[self.outmask], self.flux[self.outmask],\n ls='none', marker='.', color='k', markersize=ms, alpha=0.5)\n ax.plot(time[self.transitmask], self.flux[self.transitmask],\n ls='none', marker='.', color='k', markersize=ms, alpha=0.5)\n\n # Plot the transit + GP model\n med = np.nanmedian(self.apply_mask(self.flux))\n transit_model = \\\n med * np.sum([depth * tm(self.time)\n for tm, depth in zip(self.transit_model,\n self.transit_depth)], axis=0)\n gp = GP(self.kernel, self.kernel_params, white=False)\n gp.compute(self.apply_mask(self.time), self.apply_mask(self.fraw_err))\n y, _ = gp.predict(self.apply_mask(\n self.flux - transit_model) - med, self.time)\n if fold is not None:\n flux = (self.flux - y) / med\n ax.plot(self.apply_mask(time), self.apply_mask(flux),\n ls='none', marker='.', color='k', markersize=ms, alpha=0.5)\n ax.plot(time[self.outmask], flux[self.outmask], ls='none',\n marker='.', color='k', markersize=ms, alpha=0.5)\n ax.plot(time[self.transitmask], flux[self.transitmask],\n ls='none', marker='.', color='k', markersize=ms, alpha=0.5)\n hires_time = np.linspace(-5 * dur, 5 * dur, 1000)\n hires_transit_model = 1 + \\\n self.transit_depth[fold] * \\\n self.transit_model[fold](hires_time + t0)\n ax.plot(hires_time, hires_transit_model, 'r-', lw=1, alpha=1)\n else:\n flux = self.flux\n y += med\n y += transit_model\n ax.plot(time, y, 'r-', lw=1, alpha=1)\n\n # Plot the bad data points\n bnmask = np.array(\n list(set(np.concatenate([self.badmask, self.nanmask]))), dtype=int)\n bmask = [i for i in self.badmask if i not in self.nanmask]\n ax.plot(time[bmask], flux[bmask], 'r.', markersize=ms, alpha=0.25)\n\n # Appearance\n ax.set_ylabel('EVEREST Flux', fontsize=18)\n ax.margins(0.01, 0.1)\n if fold is not None:\n ax.set_xlabel('Time From Transit Center (days)', fontsize=18)\n ax.set_xlim(-3 * dur, 3 * dur)\n else:\n ax.set_xlabel('Time (%s)' % self._mission.TIMEUNITS, fontsize=18)\n for brkpt in self.breakpoints[:-1]:\n ax.axvline(time[brkpt], color='r', ls='--', alpha=0.25)\n ax.get_yaxis().set_major_formatter(Formatter.Flux)\n\n # Get y lims that bound most of the flux\n if fold is not None:\n lo = np.min(hires_transit_model)\n pad = 1.5 * (1 - lo)\n ylim = (lo - pad, 1 + pad)\n else:\n f = np.delete(flux, bnmask)\n N = int(0.995 * len(f))\n hi, lo = f[np.argsort(f)][[N, -N]]\n pad = (hi - lo) * 0.1\n ylim = (lo - pad, hi + pad)\n ax.set_ylim(ylim)\n\n # Indicate off-axis outliers\n for i in np.where(flux < ylim[0])[0]:\n if i in bmask:\n color = \"#ffcccc\"\n else:\n color = \"#ccccff\"\n ax.annotate('', xy=(time[i], ylim[0]), xycoords='data',\n xytext=(0, 15), textcoords='offset points',\n arrowprops=dict(arrowstyle=\"-|>\", color=color,\n alpha=0.5))\n for i in np.where(flux > ylim[1])[0]:\n if i in bmask:\n color = \"#ffcccc\"\n else:\n color = \"#ccccff\"\n ax.annotate('', xy=(time[i], ylim[1]), xycoords='data',\n xytext=(0, -15), textcoords='offset points',\n arrowprops=dict(arrowstyle=\"-|>\", color=color,\n alpha=0.5))\n\n if show:\n pl.show()\n pl.close()\n else:\n return fig, ax"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninterpolate certain elements in the array y and linearly interpolates over them returning an array y of the same length.", "response": "def Interpolate(time, mask, y):\n '''\n Masks certain elements in the array `y` and linearly\n interpolates over them, returning an array `y'` of the\n same length.\n\n :param array_like time: The time array\n :param array_like mask: The indices to be interpolated over\n :param array_like y: The dependent array\n\n '''\n\n # Ensure `y` doesn't get modified in place\n yy = np.array(y)\n t_ = np.delete(time, mask)\n y_ = np.delete(y, mask, axis=0)\n if len(yy.shape) == 1:\n yy[mask] = np.interp(time[mask], t_, y_)\n elif len(yy.shape) == 2:\n for n in range(yy.shape[1]):\n yy[mask, n] = np.interp(time[mask], t_, y_[:, n])\n else:\n raise Exception(\"Array ``y`` must be either 1- or 2-d.\")\n return yy"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Chunks(l, n, all=False):\n '''\n Returns a generator of consecutive `n`-sized chunks of list `l`.\n If `all` is `True`, returns **all** `n`-sized chunks in `l`\n by iterating over the starting point.\n\n '''\n\n if all:\n jarr = range(0, n - 1)\n else:\n jarr = [0]\n\n for j in jarr:\n for i in range(j, len(l), n):\n if i + 2 * n <= len(l):\n yield l[i:i + n]\n else:\n if not all:\n yield l[i:]\n break", "response": "Returns a generator of consecutive n - sized chunks of list l."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Smooth(x, window_len=100, window='hanning'):\n '''\n Smooth data by convolving on a given timescale.\n\n :param ndarray x: The data array\n :param int window_len: The size of the smoothing window. Default `100`\n :param str window: The window type. Default `hanning`\n\n\n '''\n\n if window_len == 0:\n return np.zeros_like(x)\n s = np.r_[2 * x[0] - x[window_len - 1::-1],\n x, 2 * x[-1] - x[-1:-window_len:-1]]\n if window == 'flat':\n w = np.ones(window_len, 'd')\n else:\n w = eval('np.' + window + '(window_len)')\n y = np.convolve(w / w.sum(), s, mode='same')\n return y[window_len:-window_len + 1]", "response": "Smooth data by convolving on a given timescale."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Scatter(y, win=13, remove_outliers=False):\n '''\n Return the scatter in ppm based on the median running standard deviation\n for a window size of :py:obj:`win` = 13 cadences (for K2, this\n is ~6.5 hours, as in VJ14).\n\n :param ndarray y: The array whose CDPP is to be computed\n :param int win: The window size in cadences. Default `13`\n :param bool remove_outliers: Clip outliers at 5 sigma before computing \\\n the CDPP? Default `False`\n\n '''\n\n if remove_outliers:\n # Remove 5-sigma outliers from data\n # smoothed on a 1 day timescale\n if len(y) >= 50:\n ys = y - Smooth(y, 50)\n else:\n ys = y\n M = np.nanmedian(ys)\n MAD = 1.4826 * np.nanmedian(np.abs(ys - M))\n out = []\n for i, _ in enumerate(y):\n if (ys[i] > M + 5 * MAD) or (ys[i] < M - 5 * MAD):\n out.append(i)\n out = np.array(out, dtype=int)\n y = np.delete(y, out)\n if len(y):\n return 1.e6 * np.nanmedian([np.std(yi) / np.sqrt(win)\n for yi in Chunks(y, win, all=True)])\n else:\n return np.nan", "response": "Computes the scatter in ppm based on the median running standard deviation of the series y."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef SavGol(y, win=49):\n '''\n Subtracts a second order Savitsky-Golay filter with window size `win`\n and returns the result. This acts as a high pass filter.\n\n '''\n\n if len(y) >= win:\n return y - savgol_filter(y, win, 2) + np.nanmedian(y)\n else:\n return y", "response": "Subtracts a second order Savitsky - Golay filter with window size win returns the result."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the number of regressors for npix pixels and PLD order pld_order.", "response": "def NumRegressors(npix, pld_order, cross_terms=True):\n '''\n Return the number of regressors for `npix` pixels\n and PLD order `pld_order`.\n\n :param bool cross_terms: Include pixel cross-terms? Default :py:obj:`True`\n\n '''\n\n res = 0\n for k in range(1, pld_order + 1):\n if cross_terms:\n res += comb(npix + k - 1, k)\n else:\n res += npix\n return int(res)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Downbin(x, newsize, axis=0, operation='mean'):\n '''\n Downbins an array to a smaller size.\n\n :param array_like x: The array to down-bin\n :param int newsize: The new size of the axis along which to down-bin\n :param int axis: The axis to operate on. Default 0\n :param str operation: The operation to perform when down-binning. \\\n Default `mean`\n '''\n\n assert newsize < x.shape[axis], \\\n \"The new size of the array must be smaller than the current size.\"\n oldsize = x.shape[axis]\n newshape = list(x.shape)\n newshape[axis] = newsize\n newshape.insert(axis + 1, oldsize // newsize)\n trim = oldsize % newsize\n if trim:\n xtrim = x[:-trim]\n else:\n xtrim = x\n\n if operation == 'mean':\n xbin = np.nanmean(xtrim.reshape(newshape), axis=axis + 1)\n elif operation == 'sum':\n xbin = np.nansum(xtrim.reshape(newshape), axis=axis + 1)\n elif operation == 'quadsum':\n xbin = np.sqrt(np.nansum(xtrim.reshape(newshape) ** 2, axis=axis + 1))\n elif operation == 'median':\n xbin = np.nanmedian(xtrim.reshape(newshape), axis=axis + 1)\n else:\n raise ValueError(\"`operation` must be either `mean`, \" +\n \"`sum`, `quadsum`, or `median`.\")\n\n return xbin", "response": "Downbin an array - like x to a smaller size."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef register_with_model(self, name, model):\n '''Called during the creation of a the :class:`StdModel`\nclass when :class:`Metaclass` is initialised. It fills\n:attr:`Field.name` and :attr:`Field.model`. This is an internal\nfunction users should never call.'''\n if self.name:\n raise FieldError('Field %s is already registered\\\n with a model' % self)\n self.name = name\n self.attname = self.get_attname()\n self.model = model\n meta = model._meta\n self.meta = meta\n meta.dfields[name] = self\n meta.fields.append(self)\n if not self.primary_key:\n self.add_to_fields()\n else:\n model._meta.pk = self", "response": "Called during the creation of a the Metaclass when the Metaclass is initialised. It fills\n. name and\n. model. This is an internal\nfunction users should never call."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds this : class:`Field` to the fields of the model.", "response": "def add_to_fields(self):\n '''Add this :class:`Field` to the fields of :attr:`model`.'''\n meta = self.model._meta\n meta.scalarfields.append(self)\n if self.index:\n meta.indices.append(self)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_lookup(self, remaining, errorClass=ValueError):\n '''called by the :class:`Query` method when it needs to build\nlookup on fields with additional nested fields. This is the case of\n:class:`ForeignKey` and :class:`JSONField`.\n\n:param remaining: the :ref:`double underscored` fields if this :class:`Field`\n:param errorClass: Optional exception class to use if the *remaining* field\n is not valid.'''\n if remaining:\n raise errorClass('Cannot use nested lookup on field %s' % self)\n return (self.attname, None)", "response": "called by the : class : Query method when it needs to build\nlookup on fields with additional nested fields."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_value(self, instance, *bits):\n '''Retrieve the value :class:`Field` from a :class:`StdModel`\n``instance``.\n\n:param instance: The :class:`StdModel` ``instance`` invoking this function.\n:param bits: Additional information for nested fields which derives from\n the :ref:`double underscore ` notation.\n:return: the value of this :class:`Field` in the ``instance``. can raise\n :class:`AttributeError`.\n\nThis method is used by the :meth:`StdModel.get_attr_value` method when\nretrieving values form a :class:`StdModel` instance.\n'''\n if bits:\n raise AttributeError\n else:\n return getattr(instance, self.attname)", "response": "Retrieve the value of this field from a : class:`StdModel instance."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the value for this field in a instance.", "response": "def set_value(self, instance, value):\n '''Set the ``value`` for this :class:`Field` in a ``instance``\nof a :class:`StdModel`.'''\n setattr(instance, self.attname, self.to_python(value))"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts a list of delimiters to corresponding regular expressions", "response": "def delimiters_to_re(delimiters):\n \"\"\"convert delimiters to corresponding regular expressions\"\"\"\n\n # caching\n delimiters = tuple(delimiters)\n if delimiters in re_delimiters:\n re_tag = re_delimiters[delimiters]\n else:\n open_tag, close_tag = delimiters\n\n # escape\n open_tag = ''.join([c if c.isalnum() else '\\\\' + c for c in open_tag])\n close_tag = ''.join([c if c.isalnum() else '\\\\' + c for c in close_tag])\n\n re_tag = re.compile(open_tag + r'([#^>&{/!=]?)\\s*(.*?)\\s*([}=]?)' + close_tag, re.DOTALL)\n re_delimiters[delimiters] = re_tag\n\n return re_tag"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if the string text [ start end ) is standalone by checking for backwards and blankspaces", "response": "def is_standalone(text, start, end):\n \"\"\"check if the string text[start:end] is standalone by checking forwards\n and backwards for blankspaces\n :text: TODO\n :(start, end): TODO\n :returns: the start of next index after text[start:end]\n\n \"\"\"\n left = False\n start -= 1\n while start >= 0 and text[start] in spaces_not_newline:\n start -= 1\n\n if start < 0 or text[start] == '\\n':\n left = True\n\n right = re_space.match(text, end)\n return (start+1, right.end()) if left and right else None"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncompiling a template into a token tree.", "response": "def compiled(template, delimiters=DEFAULT_DELIMITERS):\n \"\"\"Compile a template into token tree\n\n :template: TODO\n :delimiters: TODO\n :returns: the root token\n\n \"\"\"\n re_tag = delimiters_to_re(delimiters)\n\n # variable to save states\n tokens = []\n index = 0\n sections = []\n tokens_stack = []\n\n # root token\n root = Root('root')\n root.filters = copy.copy(filters)\n\n m = re_tag.search(template, index)\n while m is not None:\n token = None\n last_literal = None\n strip_space = False\n\n if m.start() > index:\n last_literal = Literal('str', template[index:m.start()], root=root)\n tokens.append(last_literal)\n\n # parse token\n prefix, name, suffix = m.groups()\n\n if prefix == '=' and suffix == '=':\n # {{=| |=}} to change delimiters\n delimiters = re.split(r'\\s+', name)\n if len(delimiters) != 2:\n raise SyntaxError('Invalid new delimiter definition: ' + m.group())\n re_tag = delimiters_to_re(delimiters)\n strip_space = True\n\n elif prefix == '{' and suffix == '}':\n # {{{ variable }}}\n token = Variable(name, name, root=root)\n\n elif prefix == '' and suffix == '':\n # {{ name }}\n token = Variable(name, name, root=root)\n token.escape = True\n\n elif suffix != '' and suffix != None:\n raise SyntaxError('Invalid token: ' + m.group())\n\n elif prefix == '&':\n # {{& escaped variable }}\n token = Variable(name, name, root=root)\n\n elif prefix == '!':\n # {{! comment }}\n token = Comment(name, root=root)\n if len(sections) <= 0:\n # considered as standalone only outside sections\n strip_space = True\n\n elif prefix == '>':\n # {{> partial}}\n token = Partial(name, name, root=root)\n strip_space = True\n\n pos = is_standalone(template, m.start(), m.end())\n if pos:\n token.indent = len(template[pos[0]:m.start()])\n\n elif prefix == '#' or prefix == '^':\n # {{# section }} or # {{^ inverted }}\n\n # strip filter\n sec_name = name.split('|')[0].strip()\n token = Section(sec_name, name, root=root) if prefix == '#' else Inverted(name, name, root=root)\n token.delimiter = delimiters\n tokens.append(token)\n\n # save the tokens onto stack\n token = None\n tokens_stack.append(tokens)\n tokens = []\n\n sections.append((sec_name, prefix, m.end()))\n strip_space = True\n\n elif prefix == '/':\n tag_name, sec_type, text_end = sections.pop()\n if tag_name != name:\n raise SyntaxError(\"unclosed tag: '\" + tag_name + \"' Got:\" + m.group())\n\n children = tokens\n tokens = tokens_stack.pop()\n\n tokens[-1].text = template[text_end:m.start()]\n tokens[-1].children = children\n strip_space = True\n\n else:\n raise SyntaxError('Unknown tag: ' + m.group())\n\n if token is not None:\n tokens.append(token)\n\n index = m.end()\n if strip_space:\n pos = is_standalone(template, m.start(), m.end())\n if pos:\n index = pos[1]\n if last_literal: last_literal.value = last_literal.value.rstrip(spaces_not_newline)\n\n m = re_tag.search(template, index)\n\n tokens.append(Literal('str', template[index:]))\n root.children = tokens\n return root"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _escape(self, text):\n ret = EMPTYSTRING if text is None else str(text)\n if self.escape:\n return html_escape(ret)\n else:\n return ret", "response": "Escape text according to self. escape"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _lookup(self, dot_name, contexts):\n\n # process filters\n filters = [x for x in map(lambda x: x.strip(), dot_name.split('|'))]\n dot_name = filters[0]\n filters = filters[1:]\n\n # should support paths like '../../a.b.c/../d', etc.\n if not dot_name.startswith('.'):\n dot_name = './' + dot_name\n\n paths = dot_name.split('/')\n last_path = paths[-1]\n\n # path like '../..' or ./../. etc.\n refer_context = last_path == '' or last_path == '.' or last_path == '..'\n paths = paths if refer_context else paths[:-1]\n\n # count path level\n level = 0\n for path in paths:\n if path == '..':\n level -= 1\n elif path != '.':\n # ../a.b.c/.. in the middle\n level += len(path.strip('.').split('.'))\n\n names = last_path.split('.')\n\n # fetch the correct context\n if refer_context or names[0] == '':\n try:\n value = contexts[level-1]\n except:\n value = None\n else:\n # support {{a.b.c.d.e}} like lookup\n value = lookup(names[0], contexts, level)\n\n # lookup for variables\n if not refer_context:\n for name in names[1:]:\n try:\n # a.num (a.1, a.2) to access list\n index = parse_int(name)\n name = parse_int(name) if isinstance(value, (list, tuple)) else name\n value = value[name]\n except:\n # not found\n value = None\n break;\n\n # apply filters\n for f in filters:\n try:\n func = self.root.filters[f]\n value = func(value)\n except:\n continue\n\n return value", "response": "lookup value for names like a. b. c and handle filters as well"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrender the children tokens", "response": "def _render_children(self, contexts, partials):\n \"\"\"Render the children tokens\"\"\"\n ret = []\n for child in self.children:\n ret.append(child._render(contexts, partials))\n return EMPTYSTRING.join(ret)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrender section of the value", "response": "def _render(self, contexts, partials):\n \"\"\"render section\"\"\"\n val = self._lookup(self.value, contexts)\n if not val:\n # false value\n return EMPTYSTRING\n\n # normally json has types: number/string/list/map\n # but python has more, so we decide that map and string should not iterate\n # by default, other do.\n if hasattr(val, \"__iter__\") and not isinstance(val, (str, dict)):\n # non-empty lists\n ret = []\n\n for item in val:\n contexts.append(item)\n ret.append(self._render_children(contexts, partials))\n contexts.pop()\n\n if len(ret) <= 0:\n # empty lists\n return EMPTYSTRING\n\n return self._escape(''.join(ret))\n elif callable(val):\n # lambdas\n new_template = val(self.text)\n value = inner_render(new_template, contexts, partials, self.delimiter)\n else:\n # context\n contexts.append(val)\n value = self._render_children(contexts, partials)\n contexts.pop()\n\n return self._escape(value)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Setup():\n '''\n Called when the code is installed. Sets up directories and downloads\n the K2 catalog.\n\n '''\n\n if not os.path.exists(os.path.join(EVEREST_DAT, 'k2', 'cbv')):\n os.makedirs(os.path.join(EVEREST_DAT, 'k2', 'cbv'))\n GetK2Stars(clobber=False)", "response": "Called when the code is installed. Sets up directories and downloads\n the K2 catalog."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CDPP(flux, mask=[], cadence='lc'):\n '''\n Compute the proxy 6-hr CDPP metric.\n\n :param array_like flux: The flux array to compute the CDPP for\n :param array_like mask: The indices to be masked\n :param str cadence: The light curve cadence. Default `lc`\n\n '''\n\n # 13 cadences is 6.5 hours\n rmswin = 13\n # Smooth the data on a 2 day timescale\n svgwin = 49\n\n # If short cadence, need to downbin\n if cadence == 'sc':\n newsize = len(flux) // 30\n flux = Downbin(flux, newsize, operation='mean')\n\n flux_savgol = SavGol(np.delete(flux, mask), win=svgwin)\n if len(flux_savgol):\n return Scatter(flux_savgol / np.nanmedian(flux_savgol),\n remove_outliers=True, win=rmswin)\n else:\n return np.nan", "response": "Compute the proxy 6 - hr CDPP metric."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetData(EPIC, season=None, cadence='lc', clobber=False, delete_raw=False,\n aperture_name='k2sff_15', saturated_aperture_name='k2sff_19',\n max_pixels=75, download_only=False, saturation_tolerance=-0.1,\n bad_bits=[1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17],\n get_hires=True,\n get_nearby=True, **kwargs):\n '''\n Returns a :py:obj:`DataContainer` instance with the\n raw data for the target.\n\n :param int EPIC: The EPIC ID number\n :param int season: The observing season (campaign). Default :py:obj:`None`\n :param str cadence: The light curve cadence. Default `lc`\n :param bool clobber: Overwrite existing files? Default :py:obj:`False`\n :param bool delete_raw: Delete the FITS TPF after processing it? \\\n Default :py:obj:`False`\n :param str aperture_name: The name of the aperture to use. Select \\\n `custom` to call :py:func:`GetCustomAperture`. Default `k2sff_15`\n :param str saturated_aperture_name: The name of the aperture to use if \\\n the target is saturated. Default `k2sff_19`\n :param int max_pixels: Maximum number of pixels in the TPF. Default 75\n :param bool download_only: Download raw TPF and return? Default \\\n :py:obj:`False`\n :param float saturation_tolerance: Target is considered saturated \\\n if flux is within this fraction of the pixel well depth. \\\n Default -0.1\n :param array_like bad_bits: Flagged :py:obj`QUALITY` bits to consider \\\n outliers when computing the model. \\\n Default `[1,2,3,4,5,6,7,8,9,11,12,13,14,16,17]`\n :param bool get_hires: Download a high resolution image of the target? \\\n Default :py:obj:`True`\n :param bool get_nearby: Retrieve location of nearby sources? \\\n Default :py:obj:`True`\n\n '''\n\n # Campaign no.\n if season is None:\n campaign = Season(EPIC)\n if hasattr(campaign, '__len__'):\n raise AttributeError(\n \"Please choose a campaign/season for this target: %s.\"\n % campaign)\n else:\n campaign = season\n\n # Is there short cadence data available for this target?\n short_cadence = HasShortCadence(EPIC, season=campaign)\n if cadence == 'sc' and not short_cadence:\n raise ValueError(\"Short cadence data not available for this target.\")\n\n # Local file name\n filename = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % campaign,\n ('%09d' % EPIC)[:4] + '00000', ('%09d' % EPIC)[4:],\n 'data.npz')\n\n # Download?\n if clobber or not os.path.exists(filename):\n\n # Get the TPF\n tpf = os.path.join(KPLR_ROOT, 'data', 'k2', 'target_pixel_files',\n str(EPIC), 'ktwo%09d-c%02d_lpd-targ.fits.gz'\n % (EPIC, campaign))\n sc_tpf = os.path.join(KPLR_ROOT, 'data', 'k2', 'target_pixel_files',\n str(EPIC), 'ktwo%09d-c%02d_spd-targ.fits.gz'\n % (EPIC, campaign))\n if clobber or not os.path.exists(tpf):\n kplr_client.k2_star(EPIC).get_target_pixel_files(fetch=True)\n\n with pyfits.open(tpf) as f:\n qdata = f[1].data\n\n # Get the TPF aperture\n tpf_aperture = (f[2].data & 2) // 2\n\n # Get the enlarged TPF aperture\n tpf_big_aperture = np.array(tpf_aperture)\n for i in range(tpf_big_aperture.shape[0]):\n for j in range(tpf_big_aperture.shape[1]):\n if f[2].data[i][j] == 1:\n for n in [(i - 1, j), (i + 1, j),\n (i, j - 1), (i, j + 1)]:\n if n[0] >= 0 and n[0] < tpf_big_aperture.shape[0]:\n if n[1] >= 0 and n[1] < \\\n tpf_big_aperture.shape[1]:\n if tpf_aperture[n[0]][n[1]] == 1:\n tpf_big_aperture[i][j] = 1\n\n # Is there short cadence data?\n if short_cadence:\n with pyfits.open(sc_tpf) as f:\n sc_qdata = f[1].data\n\n # Get K2SFF apertures\n try:\n k2sff = kplr.K2SFF(EPIC, sci_campaign=campaign)\n k2sff_apertures = k2sff.apertures\n if delete_raw:\n os.remove(k2sff._file)\n except:\n k2sff_apertures = [None for i in range(20)]\n\n # Make a dict of all our apertures\n # We're not getting K2SFF apertures 0-9 any more\n apertures = {'tpf': tpf_aperture, 'tpf_big': tpf_big_aperture}\n for i in range(10, 20):\n apertures.update({'k2sff_%02d' % i: k2sff_apertures[i]})\n\n # Get the header info\n fitsheader = [pyfits.getheader(tpf, 0).cards,\n pyfits.getheader(tpf, 1).cards,\n pyfits.getheader(tpf, 2).cards]\n if short_cadence:\n sc_fitsheader = [pyfits.getheader(sc_tpf, 0).cards,\n pyfits.getheader(sc_tpf, 1).cards,\n pyfits.getheader(sc_tpf, 2).cards]\n else:\n sc_fitsheader = None\n\n # Get a hi res image of the target\n if get_hires:\n hires = GetHiResImage(EPIC)\n else:\n hires = None\n\n # Get nearby sources\n if get_nearby:\n nearby = GetSources(EPIC)\n else:\n nearby = []\n\n # Delete?\n if delete_raw:\n os.remove(tpf)\n if short_cadence:\n os.remove(sc_tpf)\n\n # Get the arrays\n cadn = np.array(qdata.field('CADENCENO'), dtype='int32')\n time = np.array(qdata.field('TIME'), dtype='float64')\n fpix = np.array(qdata.field('FLUX'), dtype='float64')\n fpix_err = np.array(qdata.field('FLUX_ERR'), dtype='float64')\n qual = np.array(qdata.field('QUALITY'), dtype=int)\n\n # Get rid of NaNs in the time array by interpolating\n naninds = np.where(np.isnan(time))\n time = Interpolate(np.arange(0, len(time)), naninds, time)\n\n # Get the motion vectors (if available!)\n pc1 = np.array(qdata.field('POS_CORR1'), dtype='float64')\n pc2 = np.array(qdata.field('POS_CORR2'), dtype='float64')\n if not np.all(np.isnan(pc1)) and not np.all(np.isnan(pc2)):\n pc1 = Interpolate(time, np.where(np.isnan(pc1)), pc1)\n pc2 = Interpolate(time, np.where(np.isnan(pc2)), pc2)\n else:\n pc1 = None\n pc2 = None\n\n # Do the same for short cadence\n if short_cadence:\n sc_cadn = np.array(sc_qdata.field('CADENCENO'), dtype='int32')\n sc_time = np.array(sc_qdata.field('TIME'), dtype='float64')\n sc_fpix = np.array(sc_qdata.field('FLUX'), dtype='float64')\n sc_fpix_err = np.array(sc_qdata.field('FLUX_ERR'), dtype='float64')\n sc_qual = np.array(sc_qdata.field('QUALITY'), dtype=int)\n sc_naninds = np.where(np.isnan(sc_time))\n sc_time = Interpolate(\n np.arange(0, len(sc_time)), sc_naninds, sc_time)\n sc_pc1 = np.array(sc_qdata.field('POS_CORR1'), dtype='float64')\n sc_pc2 = np.array(sc_qdata.field('POS_CORR2'), dtype='float64')\n if not np.all(np.isnan(sc_pc1)) and not np.all(np.isnan(sc_pc2)):\n sc_pc1 = Interpolate(\n sc_time, np.where(np.isnan(sc_pc1)), sc_pc1)\n sc_pc2 = Interpolate(\n sc_time, np.where(np.isnan(sc_pc2)), sc_pc2)\n else:\n sc_pc1 = None\n sc_pc2 = None\n else:\n sc_cadn = None\n sc_time = None\n sc_fpix = None\n sc_fpix_err = None\n sc_qual = None\n sc_pc1 = None\n sc_pc2 = None\n\n # Static pixel images for plotting\n pixel_images = [fpix[0], fpix[len(fpix) // 2], fpix[len(fpix) - 1]]\n\n # Atomically write to disk.\n # http://stackoverflow.com/questions/2333872/\n # atomic-writing-to-file-with-python\n if not os.path.exists(os.path.dirname(filename)):\n os.makedirs(os.path.dirname(filename))\n f = NamedTemporaryFile(\"wb\", delete=False)\n np.savez_compressed(f, cadn=cadn, time=time, fpix=fpix,\n fpix_err=fpix_err,\n qual=qual, apertures=apertures,\n pc1=pc1, pc2=pc2, fitsheader=fitsheader,\n pixel_images=pixel_images, nearby=nearby,\n hires=hires,\n sc_cadn=sc_cadn, sc_time=sc_time, sc_fpix=sc_fpix,\n sc_fpix_err=sc_fpix_err, sc_qual=sc_qual,\n sc_pc1=sc_pc1, sc_pc2=sc_pc2,\n sc_fitsheader=sc_fitsheader)\n f.flush()\n os.fsync(f.fileno())\n f.close()\n shutil.move(f.name, filename)\n\n if download_only:\n return\n\n # Load\n data = np.load(filename)\n apertures = data['apertures'][()]\n pixel_images = data['pixel_images']\n nearby = data['nearby']\n hires = data['hires'][()]\n\n if cadence == 'lc':\n fitsheader = data['fitsheader']\n cadn = data['cadn']\n time = data['time']\n fpix = data['fpix']\n fpix_err = data['fpix_err']\n qual = data['qual']\n pc1 = data['pc1']\n pc2 = data['pc2']\n elif cadence == 'sc':\n fitsheader = data['sc_fitsheader']\n cadn = data['sc_cadn']\n time = data['sc_time']\n fpix = data['sc_fpix']\n fpix_err = data['sc_fpix_err']\n qual = data['sc_qual']\n pc1 = data['sc_pc1']\n pc2 = data['sc_pc2']\n else:\n raise ValueError(\"Invalid value for the cadence.\")\n\n # Select the \"saturated aperture\" to check if the star is saturated\n # If it is, we will use this aperture instead\n if saturated_aperture_name == 'custom':\n saturated_aperture = GetCustomAperture(data)\n else:\n if saturated_aperture_name is None:\n saturated_aperture_name = 'k2sff_19'\n saturated_aperture = apertures[saturated_aperture_name]\n if saturated_aperture is None:\n log.error(\"Invalid aperture selected. Defaulting to `tpf_big`.\")\n saturated_aperture_name = 'tpf_big'\n saturated_aperture = apertures[saturated_aperture_name]\n\n # HACK: Some C05 K2SFF apertures don't match the target pixel file\n # pixel grid size. This is likely because they're defined on the M67\n # superstamp. For now, let's ignore these stars.\n if saturated_aperture.shape != fpix.shape[1:]:\n log.error(\"Aperture size mismatch!\")\n return None\n\n # Compute the saturation flux and the 97.5th percentile\n # flux in each pixel of the saturated aperture. We're going\n # to compare these to decide if the star is saturated.\n satflx = SaturationFlux(EPIC, campaign=campaign) * \\\n (1. + saturation_tolerance)\n f97 = np.zeros((fpix.shape[1], fpix.shape[2]))\n for i in range(fpix.shape[1]):\n for j in range(fpix.shape[2]):\n if saturated_aperture[i, j]:\n # Let's remove NaNs...\n tmp = np.delete(fpix[:, i, j], np.where(\n np.isnan(fpix[:, i, j])))\n # ... and really bad outliers...\n if len(tmp):\n f = SavGol(tmp)\n med = np.nanmedian(f)\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n bad = np.where((f > med + 10. * MAD) |\n (f < med - 10. * MAD))[0]\n np.delete(tmp, bad)\n # ... so we can compute the 97.5th percentile flux\n i97 = int(0.975 * len(tmp))\n tmp = tmp[np.argsort(tmp)[i97]]\n f97[i, j] = tmp\n\n # Check if any of the pixels are actually saturated\n if np.nanmax(f97) <= satflx:\n log.info(\"No saturated columns detected.\")\n saturated = False\n else:\n log.info(\"Saturated pixel(s) found. Switching to aperture `%s`.\" %\n saturated_aperture_name)\n aperture_name = saturated_aperture_name\n saturated = True\n\n # Now grab the aperture we'll actually use\n if aperture_name == 'custom':\n aperture = GetCustomAperture(data)\n else:\n if aperture_name is None:\n aperture_name = 'k2sff_15'\n aperture = apertures[aperture_name]\n if aperture is None:\n log.error(\"Invalid aperture selected. Defaulting to `tpf_big`.\")\n aperture_name = 'tpf_big'\n aperture = apertures[aperture_name]\n\n # HACK: Some C05 K2SFF apertures don't match the target pixel file\n # pixel grid size. This is likely because they're defined on the M67\n # superstamp. For now, let's ignore these stars.\n if aperture.shape != fpix.shape[1:]:\n log.error(\"Aperture size mismatch!\")\n return None\n\n # Now we check if the aperture is too big. Can lead to memory errors...\n # Treat saturated and unsaturated stars differently.\n if saturated:\n\n # Need to check if we have too many pixels *after* collapsing columns.\n # Sort the apertures in decreasing order of pixels, but keep the apert.\n # chosen by the user first.\n aperture_names = np.array(list(apertures.keys()))\n npix_per_aperture = np.array(\n [np.sum(apertures[k]) for k in aperture_names])\n aperture_names = aperture_names[np.argsort(npix_per_aperture)[::-1]]\n aperture_names = np.append([aperture_name], np.delete(\n aperture_names, np.argmax(aperture_names == aperture_name)))\n\n # Loop through them. Pick the first one that satisfies\n # the `max_pixels` constraint\n for aperture_name in aperture_names:\n aperture = apertures[aperture_name]\n aperture[np.isnan(fpix[0])] = 0\n ncol = 0\n apcopy = np.array(aperture)\n for j in range(apcopy.shape[1]):\n if np.any(f97[:, j] > satflx):\n apcopy[:, j] = 0\n ncol += 1\n if np.sum(apcopy) + ncol <= max_pixels:\n break\n if np.sum(apcopy) + ncol > max_pixels:\n log.error(\n \"No apertures available with fewer than %d pixels. Aborting.\"\n % max_pixels)\n return None\n\n # Now, finally, we collapse the saturated columns into single pixels\n # and make the pixel array 2D\n ncol = 0\n fpixnew = []\n ferrnew = []\n\n # HACK: K2SFF sometimes clips the heads/tails of saturated columns\n # That's really bad, since that's where all the information is. Let's\n # artificially extend the aperture by two pixels at the top and bottom\n # of each saturated column. This *could* increase contamination, but\n # it's unlikely since the saturated target is by definition really\n # bright\n ext = 0\n for j in range(aperture.shape[1]):\n if np.any(f97[:, j] > satflx):\n for i in range(aperture.shape[0]):\n if (aperture[i, j] == 0) and \\\n (np.nanmedian(fpix[:, i, j]) > 0):\n if (i + 2 < aperture.shape[0]) and \\\n aperture[i + 2, j] == 1:\n aperture[i, j] = 2\n ext += 1\n elif (i + 1 < aperture.shape[0]) and \\\n aperture[i + 1, j] == 1:\n aperture[i, j] = 2\n ext += 1\n elif (i - 1 >= 0) and aperture[i - 1, j] == 1:\n aperture[i, j] = 2\n ext += 1\n elif (i - 2 >= 0) and aperture[i - 2, j] == 1:\n aperture[i, j] = 2\n ext += 1\n if ext:\n log.info(\"Extended saturated columns by %d pixel(s).\" % ext)\n\n for j in range(aperture.shape[1]):\n if np.any(f97[:, j] > satflx):\n marked = False\n collapsed = np.zeros(len(fpix[:, 0, 0]))\n collapsed_err2 = np.zeros(len(fpix[:, 0, 0]))\n for i in range(aperture.shape[0]):\n if aperture[i, j]:\n if not marked:\n aperture[i, j] = AP_COLLAPSED_PIXEL\n marked = True\n else:\n aperture[i, j] = AP_SATURATED_PIXEL\n collapsed += fpix[:, i, j]\n collapsed_err2 += fpix_err[:, i, j] ** 2\n if np.any(collapsed):\n fpixnew.append(collapsed)\n ferrnew.append(np.sqrt(collapsed_err2))\n ncol += 1\n else:\n for i in range(aperture.shape[0]):\n if aperture[i, j]:\n fpixnew.append(fpix[:, i, j])\n ferrnew.append(fpix_err[:, i, j])\n fpix2D = np.array(fpixnew).T\n fpix_err2D = np.array(ferrnew).T\n log.info(\"Collapsed %d saturated column(s).\" % ncol)\n\n else:\n\n # Check if there are too many pixels\n if np.sum(aperture) > max_pixels:\n\n # This case is simpler: we just pick the largest aperture\n # that's less than or equal to `max_pixels`\n keys = list(apertures.keys())\n npix = np.array([np.sum(apertures[k]) for k in keys])\n aperture_name = keys[np.argmax(npix * (npix <= max_pixels))]\n aperture = apertures[aperture_name]\n aperture[np.isnan(fpix[0])] = 0\n if np.sum(aperture) > max_pixels:\n log.error(\"No apertures available with fewer than \" +\n \"%d pixels. Aborting.\" % max_pixels)\n return None\n log.warn(\n \"Selected aperture is too big. Proceeding with aperture \" +\n \"`%s` instead.\" % aperture_name)\n\n # Make the pixel flux array 2D\n aperture[np.isnan(fpix[0])] = 0\n ap = np.where(aperture & 1)\n fpix2D = np.array([f[ap] for f in fpix], dtype='float64')\n fpix_err2D = np.array([p[ap] for p in fpix_err], dtype='float64')\n\n # Compute the background\n binds = np.where(aperture ^ 1)\n if RemoveBackground(EPIC, campaign=campaign) and (len(binds[0]) > 0):\n bkg = np.nanmedian(np.array([f[binds]\n for f in fpix], dtype='float64'), axis=1)\n # Uncertainty of the median:\n # http://davidmlane.com/hyperstat/A106993.html\n bkg_err = 1.253 * np.nanmedian(np.array([e[binds] for e in fpix_err],\n dtype='float64'), axis=1) \\\n / np.sqrt(len(binds[0]))\n bkg = bkg.reshape(-1, 1)\n bkg_err = bkg_err.reshape(-1, 1)\n else:\n bkg = 0.\n bkg_err = 0.\n\n # Make everything 2D and remove the background\n fpix = fpix2D - bkg\n fpix_err = np.sqrt(fpix_err2D ** 2 + bkg_err ** 2)\n flux = np.sum(fpix, axis=1)\n ferr = np.sqrt(np.sum(fpix_err ** 2, axis=1))\n\n # Get NaN data points\n nanmask = np.where(np.isnan(flux) | (flux == 0))[0]\n\n # Get flagged data points -- we won't train our model on them\n badmask = []\n for b in bad_bits:\n badmask += list(np.where(qual & 2 ** (b - 1))[0])\n\n # Flag >10 sigma outliers -- same thing.\n tmpmask = np.array(list(set(np.concatenate([badmask, nanmask]))))\n t = np.delete(time, tmpmask)\n f = np.delete(flux, tmpmask)\n f = SavGol(f)\n med = np.nanmedian(f)\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n bad = np.where((f > med + 10. * MAD) | (f < med - 10. * MAD))[0]\n badmask.extend([np.argmax(time == t[i]) for i in bad])\n\n # Campaign 2 hack: the first day or two are screwed up\n if campaign == 2:\n badmask.extend(np.where(time < 2061.5)[0])\n\n # TODO: Fix time offsets in first half of\n # Campaign 0. See note in everest 1.0 code\n\n # Finalize the mask\n badmask = np.array(sorted(list(set(badmask))))\n\n # Interpolate the nans\n fpix = Interpolate(time, nanmask, fpix)\n fpix_err = Interpolate(time, nanmask, fpix_err)\n\n # Return\n data = DataContainer()\n data.ID = EPIC\n data.campaign = campaign\n data.cadn = cadn\n data.time = time\n data.fpix = fpix\n data.fpix_err = fpix_err\n data.nanmask = nanmask\n data.badmask = badmask\n data.aperture = aperture\n data.aperture_name = aperture_name\n data.apertures = apertures\n data.quality = qual\n data.Xpos = pc1\n data.Ypos = pc2\n data.meta = fitsheader\n data.mag = fitsheader[0]['KEPMAG'][1]\n data.pixel_images = pixel_images\n data.nearby = nearby\n data.hires = hires\n data.saturated = saturated\n data.bkg = bkg\n\n return data", "response": "Retrieves the raw data for a given aperture."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn neighbors random bright stars on the same module as the EPIC.", "response": "def GetNeighbors(EPIC, season=None, model=None, neighbors=10,\n mag_range=(11., 13.),\n cdpp_range=None, aperture_name='k2sff_15',\n cadence='lc', **kwargs):\n '''\n Return `neighbors` random bright stars on the same module as `EPIC`.\n\n :param int EPIC: The EPIC ID number\n :param str model: The :py:obj:`everest` model name. Only used when \\\n imposing CDPP bounds. Default :py:obj:`None`\n :param int neighbors: Number of neighbors to return. Default 10\n :param str aperture_name: The name of the aperture to use. Select \\\n `custom` to call \\\n :py:func:`GetCustomAperture`. Default `k2sff_15`\n :param str cadence: The light curve cadence. Default `lc`\n :param tuple mag_range: (`low`, `high`) values for the Kepler magnitude. \\\n Default (11, 13)\n :param tuple cdpp_range: (`low`, `high`) values for the de-trended CDPP. \\\n Default :py:obj:`None`\n\n '''\n\n # Zero neighbors?\n if neighbors == 0:\n return []\n\n # Get the IDs\n # Campaign no.\n if season is None:\n campaign = Season(EPIC)\n if hasattr(campaign, '__len__'):\n raise AttributeError(\n \"Please choose a campaign/season for this target: %s.\"\n % campaign)\n else:\n campaign = season\n epics, kepmags, channels, short_cadence = np.array(GetK2Stars()[\n campaign]).T\n short_cadence = np.array(short_cadence, dtype=bool)\n epics = np.array(epics, dtype=int)\n c = GetNeighboringChannels(Channel(EPIC, campaign=season))\n\n # Manage kwargs\n if aperture_name is None:\n aperture_name = 'k2sff_15'\n if mag_range is None:\n mag_lo = -np.inf\n mag_hi = np.inf\n else:\n mag_lo = mag_range[0]\n mag_hi = mag_range[1]\n # K2-specific tweak. The short cadence stars are preferentially\n # really bright ones, so we won't get many neighbors if we\n # stick to the default magnitude range! I'm\n # therefore enforcing a lower magnitude cut-off of 8.\n if cadence == 'sc':\n mag_lo = 8.\n if cdpp_range is None:\n cdpp_lo = -np.inf\n cdpp_hi = np.inf\n else:\n cdpp_lo = cdpp_range[0]\n cdpp_hi = cdpp_range[1]\n targets = []\n\n # First look for nearby targets, then relax the constraint\n # If still no targets, widen magnitude range\n for n in range(3):\n\n if n == 0:\n nearby = True\n elif n == 1:\n nearby = False\n elif n == 2:\n mag_lo -= 1\n mag_hi += 1\n\n # Loop over all stars\n for star, kp, channel, sc in zip(epics, kepmags, channels, short_cadence):\n\n # Preliminary vetting\n if not (((channel in c) if nearby else True) and (kp < mag_hi) \\\n and (kp > mag_lo) and (sc if cadence == 'sc' else True)):\n continue\n\n # Reject if self or if already in list\n if (star == EPIC) or (star in targets):\n continue\n\n # Ensure raw light curve file exists\n if not os.path.exists(\n os.path.join(TargetDirectory(star, campaign), 'data.npz')):\n continue\n\n # Ensure crowding is OK. This is quite conservative, as we\n # need to prevent potential astrophysical false positive\n # contamination from crowded planet-hosting neighbors when\n # doing neighboring PLD.\n contam = False\n data = np.load(os.path.join(\n TargetDirectory(star, campaign), 'data.npz'))\n aperture = data['apertures'][()][aperture_name]\n\n # Check that the aperture exists!\n if aperture is None:\n continue\n\n fpix = data['fpix']\n for source in data['nearby'][()]:\n # Ignore self\n if source['ID'] == star:\n continue\n # Ignore really dim stars\n if source['mag'] < kp - 5:\n continue\n # Compute source position\n x = int(np.round(source['x'] - source['x0']))\n y = int(np.round(source['y'] - source['y0']))\n # If the source is within two pixels of the edge\n # of the target aperture, reject the target\n for j in [x - 2, x - 1, x, x + 1, x + 2]:\n if j < 0:\n # Outside the postage stamp\n continue\n for i in [y - 2, y - 1, y, y + 1, y + 2]:\n if i < 0:\n # Outside the postage stamp\n continue\n try:\n if aperture[i][j]:\n # Oh-oh!\n contam = True\n except IndexError:\n # Out of bounds... carry on!\n pass\n if contam:\n continue\n\n # HACK: This happens for K2SFF M67 targets in C05.\n # Let's skip them\n if aperture.shape != fpix.shape[1:]:\n continue\n\n # Reject if the model is not present\n if model is not None:\n if not os.path.exists(os.path.join(\n TargetDirectory(star, campaign), model + '.npz')):\n continue\n\n # Reject if CDPP out of range\n if cdpp_range is not None:\n cdpp = np.load(os.path.join(TargetDirectory(\n star, campaign), model + '.npz'))['cdpp']\n if (cdpp > cdpp_hi) or (cdpp < cdpp_lo):\n continue\n\n # Passed all the tests!\n targets.append(star)\n\n # Do we have enough? If so, return\n if len(targets) == neighbors:\n random.shuffle(targets)\n return targets\n\n # If we get to this point, we didn't find enough neighbors...\n # Return what we have anyway.\n return targets"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing and plot the CDPP statistics between model and compare_to for all known K2 planets.", "response": "def PlanetStatistics(model='nPLD', compare_to='k2sff', **kwargs):\n '''\n Computes and plots the CDPP statistics comparison between `model` and\n `compare_to` for all known K2 planets.\n\n :param str model: The :py:obj:`everest` model name\n :param str compare_to: The :py:obj:`everest` model name or \\\n other K2 pipeline name\n\n '''\n\n # Load all planet hosts\n f = os.path.join(EVEREST_SRC, 'missions', 'k2', 'tables', 'planets.tsv')\n epic, campaign, kp, _, _, _, _, _, _ = np.loadtxt(\n f, unpack=True, skiprows=2)\n epic = np.array(epic, dtype=int)\n campaign = np.array(campaign, dtype=int)\n cdpp = np.zeros(len(epic))\n saturated = np.zeros(len(epic), dtype=int)\n cdpp_1 = np.zeros(len(epic))\n\n # Get the stats\n for c in set(campaign):\n\n # Everest model\n f = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' % (int(c), model))\n e0, _, _, c0, _, _, _, _, s0 = np.loadtxt(f, unpack=True, skiprows=2)\n for i, e in enumerate(epic):\n if e in e0:\n j = np.argmax(e0 == e)\n cdpp[i] = c0[j]\n saturated[i] = s0[j]\n\n # Comparison model\n f = os.path.join(EVEREST_SRC, 'missions', 'k2', 'tables',\n 'c%02d_%s.cdpp' % (int(c), compare_to.lower()))\n if not os.path.exists(f):\n continue\n if compare_to.lower() in ['everest1', 'k2sff', 'k2sc']:\n e1, c1 = np.loadtxt(f, unpack=True, skiprows=2)\n else:\n e1, _, _, c1, _, _, _, _, _ = np.loadtxt(\n f, unpack=True, skiprows=2)\n for i, e in enumerate(epic):\n if e in e1:\n j = np.argmax(e1 == e)\n cdpp_1[i] = c1[j]\n\n sat = np.where(saturated == 1)\n unsat = np.where(saturated == 0)\n\n # Plot the equivalent of the Aigrain+16 figure\n fig, ax = pl.subplots(1)\n fig.canvas.set_window_title(\n 'K2 Planet Hosts: %s versus %s' % (model, compare_to))\n x = kp\n y = (cdpp - cdpp_1) / cdpp_1\n ax.scatter(x[unsat], y[unsat], color='b', marker='.',\n alpha=0.5, zorder=-1, picker=True)\n ax.scatter(x[sat], y[sat], color='r', marker='.',\n alpha=0.5, zorder=-1, picker=True)\n ax.set_ylim(-1, 1)\n ax.set_xlim(8, 18)\n ax.axhline(0, color='gray', lw=2, zorder=-99, alpha=0.5)\n ax.axhline(0.5, color='gray', ls='--', lw=2, zorder=-99, alpha=0.5)\n ax.axhline(-0.5, color='gray', ls='--', lw=2, zorder=-99, alpha=0.5)\n ax.set_title(r'K2 Planet Hosts', fontsize=18)\n ax.set_ylabel(r'Relative CDPP', fontsize=18)\n ax.set_xlabel('Kepler Magnitude', fontsize=18)\n\n # Pickable points\n Picker = StatsPicker([ax], [kp], [y], epic,\n model=model, compare_to=compare_to)\n fig.canvas.mpl_connect('pick_event', Picker)\n\n # Show\n pl.show()"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ShortCadenceStatistics(campaign=None, clobber=False, model='nPLD',\n plot=True, **kwargs):\n '''\n Computes and plots the CDPP statistics comparison between short cadence\n and long cadence de-trended light curves\n\n :param campaign: The campaign number or list of campaign numbers. \\\n Default is to plot all campaigns\n :param bool clobber: Overwrite existing files? Default :py:obj:`False`\n :param str model: The :py:obj:`everest` model name\n :param bool plot: Default :py:obj:`True`\n\n '''\n\n # Check campaign\n if campaign is None:\n campaign = np.arange(9)\n else:\n campaign = np.atleast_1d(campaign)\n\n # Update model name\n model = '%s.sc' % model\n\n # Compute the statistics\n for camp in campaign:\n sub = np.array(GetK2Campaign(\n camp, cadence='sc', epics_only=True), dtype=int)\n outfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' % (int(camp), model))\n if clobber or not os.path.exists(outfile):\n with open(outfile, 'w') as f:\n print(\"EPIC Kp Raw CDPP \" +\n \"Everest CDPP Saturated\", file=f)\n print(\"--------- ------ --------- \" +\n \"------------ ---------\", file=f)\n all = GetK2Campaign(int(camp), cadence='sc')\n stars = np.array([s[0] for s in all], dtype=int)\n kpmgs = np.array([s[1] for s in all], dtype=float)\n for i, _ in enumerate(stars):\n sys.stdout.write(\n '\\rProcessing target %d/%d...' % (i + 1, len(stars)))\n sys.stdout.flush()\n nf = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % camp,\n ('%09d' % stars[i])[:4] + '00000',\n ('%09d' % stars[i])[4:], model + '.npz')\n try:\n data = np.load(nf)\n print(\"{:>09d} {:>15.3f} {:>15.3f} {:>15.3f} {:>15d}\".format(\n stars[i], kpmgs[i], data['cdppr'][()], data['cdpp'][()],\n int(data['saturated'])), file=f)\n except:\n print(\"{:>09d} {:>15.3f} {:>15.3f} {:>15.3f} {:>15d}\".format(\n stars[i], kpmgs[i], np.nan, np.nan, 0), file=f)\n print(\"\")\n\n if not plot:\n return\n\n # Running lists\n xsat = []\n ysat = []\n xunsat = []\n yunsat = []\n xall = []\n yall = []\n epics = []\n\n # Plot\n for camp in campaign:\n\n # Load all stars\n sub = np.array(GetK2Campaign(\n camp, cadence='sc', epics_only=True), dtype=int)\n outfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' % (int(camp), model))\n epic, kp, cdpp6r, cdpp6, saturated = np.loadtxt(\n outfile, unpack=True, skiprows=2)\n epic = np.array(epic, dtype=int)\n saturated = np.array(saturated, dtype=int)\n\n # Get only stars in this subcamp\n inds = np.array([e in sub for e in epic])\n epic = epic[inds]\n kp = kp[inds]\n # HACK: camp 0 magnitudes are reported only to the nearest tenth,\n # so let's add a little noise to spread them out for nicer plotting\n kp = kp + 0.1 * (0.5 - np.random.random(len(kp)))\n cdpp6r = cdpp6r[inds]\n cdpp6 = cdpp6[inds]\n saturated = saturated[inds]\n sat = np.where(saturated == 1)\n unsat = np.where(saturated == 0)\n if not np.any([not np.isnan(x) for x in cdpp6]):\n continue\n\n # Get the long cadence stats\n compfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' % (int(camp),\n model[:-3]))\n epic_1, _, _, cdpp6_1, _, _, _, _, saturated = np.loadtxt(\n compfile, unpack=True, skiprows=2)\n epic_1 = np.array(epic_1, dtype=int)\n inds = np.array([e in sub for e in epic_1])\n epic_1 = epic_1[inds]\n cdpp6_1 = cdpp6_1[inds]\n cdpp6_1 = sort_like(cdpp6_1, epic, epic_1)\n x = kp\n y = (cdpp6 - cdpp6_1) / cdpp6_1\n\n # Append to running lists\n xsat.extend(x[sat])\n ysat.extend(y[sat])\n xunsat.extend(x[unsat])\n yunsat.extend(y[unsat])\n xall.extend(x)\n yall.extend(y)\n epics.extend(epic)\n\n # Plot the equivalent of the Aigrain+16 figure\n fig, ax = pl.subplots(1)\n fig.canvas.set_window_title('K2 Short Cadence')\n\n ax.scatter(xunsat, yunsat, color='b', marker='.',\n alpha=0.35, zorder=-1, picker=True)\n ax.scatter(xsat, ysat, color='r', marker='.',\n alpha=0.35, zorder=-1, picker=True)\n ax.set_ylim(-1, 1)\n ax.set_xlim(8, 18)\n ax.axhline(0, color='gray', lw=2, zorder=-99, alpha=0.5)\n ax.axhline(0.5, color='gray', ls='--', lw=2, zorder=-99, alpha=0.5)\n ax.axhline(-0.5, color='gray', ls='--', lw=2, zorder=-99, alpha=0.5)\n ax.set_title(r'Short Versus Long Cadence', fontsize=18)\n ax.set_ylabel(r'Relative CDPP', fontsize=18)\n ax.set_xlabel('Kepler Magnitude', fontsize=18)\n\n # Bin the CDPP\n yall = np.array(yall)\n xall = np.array(xall)\n bins = np.arange(7.5, 18.5, 0.5)\n by = np.zeros_like(bins) * np.nan\n for b, bin in enumerate(bins):\n i = np.where((yall > -np.inf) & (yall < np.inf) &\n (xall >= bin - 0.5) & (xall < bin + 0.5))[0]\n if len(i) > 10:\n by[b] = np.median(yall[i])\n ax.plot(bins[:9], by[:9], 'r--', lw=2)\n ax.plot(bins[8:], by[8:], 'k-', lw=2)\n\n # Pickable points\n Picker = StatsPicker([ax], [xall], [yall], epics, model=model[:-3],\n compare_to=model[:-3], cadence='sc',\n campaign=campaign)\n fig.canvas.mpl_connect('pick_event', Picker)\n\n # Show\n pl.show()", "response": "Compute the CDPP statistics between short cadence and long cadence de - trended light curves."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Statistics(season=None, clobber=False, model='nPLD', injection=False,\n compare_to='kepler', plot=True, cadence='lc', planets=False,\n **kwargs):\n '''\n Computes and plots the CDPP statistics comparison between `model`\n and `compare_to` for all long cadence light curves in a given campaign\n\n :param season: The campaign number or list of campaign numbers. \\\n Default is to plot all campaigns\n :param bool clobber: Overwrite existing files? Default :py:obj:`False`\n :param str model: The :py:obj:`everest` model name\n :param str compare_to: The :py:obj:`everest` model name or other \\\n K2 pipeline name\n :param bool plot: Default :py:obj:`True`\n :param bool injection: Statistics for injection tests? Default \\\n :py:obj:`False`\n :param bool planets: Statistics for known K2 planets? \\\n Default :py:obj:`False`\n\n '''\n\n # Multi-mission compatibility\n campaign = season\n\n # Is this short cadence?\n if cadence == 'sc':\n return ShortCadenceStatistics(campaign=campaign, clobber=clobber,\n model=model, plot=plot, **kwargs)\n\n # Check the campaign\n if campaign is None:\n campaign = 0\n\n # Planet hosts only?\n if planets:\n return PlanetStatistics(model=model, compare_to=compare_to, **kwargs)\n\n # Is this an injection run?\n if injection:\n return InjectionStatistics(campaign=campaign, clobber=clobber,\n model=model, plot=plot, **kwargs)\n\n # Compute the statistics\n sub = np.array([s[0] for s in GetK2Campaign(campaign)], dtype=int)\n outfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' % (int(campaign), model))\n if clobber or not os.path.exists(outfile):\n with open(outfile, 'w') as f:\n print(\"EPIC Kp Raw CDPP Everest CDPP\" +\n \" Validation Outliers[1] Outliers[2] \" +\n \"Datapoints Saturated\", file=f)\n print(\"--------- ------ --------- ------------\" +\n \" ---------- ----------- ----------- \" +\n \"---------- ---------\", file=f)\n all = GetK2Campaign(int(campaign))\n stars = np.array([s[0] for s in all], dtype=int)\n kpmgs = np.array([s[1] for s in all], dtype=float)\n for i, _ in enumerate(stars):\n sys.stdout.write('\\rProcessing target %d/%d...' %\n (i + 1, len(stars)))\n sys.stdout.flush()\n nf = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % campaign,\n ('%09d' % stars[i])[:4] + '00000',\n ('%09d' % stars[i])[4:], model + '.npz')\n try:\n data = np.load(nf)\n\n # Remove NaNs and flagged cadences\n flux = np.delete(data['fraw'] - data['model'], np.array(\n list(set(np.concatenate([data['nanmask'],\n data['badmask']])))))\n # Iterative sigma clipping to get 5 sigma outliers\n inds = np.array([], dtype=int)\n m = 1\n while len(inds) < m:\n m = len(inds)\n ff = SavGol(np.delete(flux, inds))\n med = np.nanmedian(ff)\n MAD = 1.4826 * np.nanmedian(np.abs(ff - med))\n inds = np.append(inds, np.where(\n (ff > med + 5. * MAD) | (ff < med - 5. * MAD))[0])\n nout = len(inds)\n ntot = len(flux)\n\n # HACK: Backwards compatibility fix\n try:\n cdpp = data['cdpp'][()]\n except KeyError:\n cdpp = data['cdpp6'][()]\n\n print(\"{:>09d} {:>15.3f} {:>15.3f} {:>15.3f} {:>15.3f} {:>15d} {:>15d} {:>15d} {:>15d}\".format(\n stars[i], kpmgs[i], data['cdppr'][()], cdpp,\n data['cdppv'][()], len(data['outmask']), nout,\n ntot, int(data['saturated'])), file=f)\n except:\n print(\"{:>09d} {:>15.3f} {:>15.3f} {:>15.3f} {:>15.3f} {:>15d} {:>15d} {:>15d} {:>15d}\".format(\n stars[i], kpmgs[i], np.nan, np.nan,\n np.nan, 0, 0, 0, 0), file=f)\n print(\"\")\n\n if plot:\n\n # Load all stars\n epic, kp, cdpp6r, cdpp6, cdpp6v, _, out, tot, saturated = np.loadtxt(\n outfile, unpack=True, skiprows=2)\n epic = np.array(epic, dtype=int)\n out = np.array(out, dtype=int)\n tot = np.array(tot, dtype=int)\n saturated = np.array(saturated, dtype=int)\n\n # Get only stars in this subcampaign\n inds = np.array([e in sub for e in epic])\n epic = epic[inds]\n kp = kp[inds]\n # HACK: Campaign 0 magnitudes are reported only to the nearest tenth,\n # so let's add a little noise to spread them out for nicer plotting\n kp = kp + 0.1 * (0.5 - np.random.random(len(kp)))\n cdpp6r = cdpp6r[inds]\n cdpp6 = cdpp6[inds]\n cdpp6v = cdpp6v[inds]\n out = out[inds]\n tot = tot[inds]\n saturated = saturated[inds]\n sat = np.where(saturated == 1)\n unsat = np.where(saturated == 0)\n if not np.any([not np.isnan(x) for x in cdpp6]):\n raise Exception(\"No targets to plot.\")\n\n # Control transparency\n alpha_kepler = 0.03\n alpha_unsat = min(0.1, 2000. / (1 + len(unsat[0])))\n alpha_sat = min(1., 180. / (1 + len(sat[0])))\n\n # Get the comparison model stats\n if compare_to.lower() == 'everest1':\n epic_1, cdpp6_1 = np.loadtxt(\n os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_everest1.cdpp'\n % int(campaign)), unpack=True)\n cdpp6_1 = sort_like(cdpp6_1, epic, epic_1)\n # Outliers\n epic_1, out_1, tot_1 = np.loadtxt(\n os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_everest1.out'\n % int(campaign)), unpack=True)\n out_1 = sort_like(out_1, epic, epic_1)\n tot_1 = sort_like(tot_1, epic, epic_1)\n elif compare_to.lower() == 'k2sc':\n epic_1, cdpp6_1 = np.loadtxt(\n os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_k2sc.cdpp' %\n int(campaign)), unpack=True)\n cdpp6_1 = sort_like(cdpp6_1, epic, epic_1)\n # Outliers\n epic_1, out_1, tot_1 = np.loadtxt(\n os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_k2sc.out'\n % int(campaign)), unpack=True)\n out_1 = sort_like(out_1, epic, epic_1)\n tot_1 = sort_like(tot_1, epic, epic_1)\n elif compare_to.lower() == 'k2sff':\n epic_1, cdpp6_1 = np.loadtxt(\n os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_k2sff.cdpp'\n % int(campaign)), unpack=True)\n cdpp6_1 = sort_like(cdpp6_1, epic, epic_1)\n # Outliers\n epic_1, out_1, tot_1 = np.loadtxt(\n os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_k2sff.out'\n % int(campaign)), unpack=True)\n out_1 = sort_like(out_1, epic, epic_1)\n tot_1 = sort_like(tot_1, epic, epic_1)\n elif compare_to.lower() == 'kepler':\n kic, kepler_kp, kepler_cdpp6 = np.loadtxt(\n os.path.join(EVEREST_SRC,\n 'missions', 'k2',\n 'tables',\n 'kepler.cdpp'),\n unpack=True)\n else:\n compfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' %\n (int(campaign), compare_to))\n\n epic_1, _, _, cdpp6_1, _, _, out_1, tot_1, saturated = np.loadtxt(\n compfile, unpack=True, skiprows=2)\n epic_1 = np.array(epic_1, dtype=int)\n inds = np.array([e in sub for e in epic_1])\n epic_1 = epic_1[inds]\n cdpp6_1 = cdpp6_1[inds]\n out_1 = out_1[inds]\n tot_1 = tot_1[inds]\n cdpp6_1 = sort_like(cdpp6_1, epic, epic_1)\n out_1 = sort_like(out_1, epic, epic_1)\n tot_1 = sort_like(tot_1, epic, epic_1)\n\n # ------ 1. Plot cdpp vs. mag\n if compare_to.lower() != 'kepler':\n fig = pl.figure(figsize=(16, 5))\n ax = [pl.subplot2grid((120, 120), (0, 0), colspan=35,\n rowspan=120),\n pl.subplot2grid((120, 120), (0, 40),\n colspan=35, rowspan=120),\n pl.subplot2grid((120, 120), (0, 80),\n colspan=35, rowspan=55),\n pl.subplot2grid((120, 120), (65, 80), colspan=35,\n rowspan=55)]\n else:\n fig = pl.figure(figsize=(12, 5))\n ax = [pl.subplot2grid((120, 75), (0, 0), colspan=35, rowspan=120),\n None,\n pl.subplot2grid((120, 75), (0, 40), colspan=35, rowspan=55),\n pl.subplot2grid((120, 75), (65, 40), colspan=35,\n rowspan=55)]\n fig.canvas.set_window_title(\n 'K2 Campaign %s: %s versus %s' % (campaign, model, compare_to))\n fig.subplots_adjust(left=0.05, right=0.95, bottom=0.125, top=0.9)\n bins = np.arange(7.5, 18.5, 0.5)\n if compare_to.lower() != 'kepler':\n ax[0].scatter(kp[unsat], cdpp6_1[unsat], color='y',\n marker='.', alpha=alpha_unsat)\n ax[0].scatter(kp[sat], cdpp6_1[sat], color='y',\n marker='s', alpha=alpha_sat, s=5)\n ax[0].scatter(kp[unsat], cdpp6[unsat], color='b',\n marker='.', alpha=alpha_unsat, picker=True)\n ax[0].scatter(kp[sat], cdpp6[sat], color='b',\n marker='s', alpha=alpha_sat, s=5, picker=True)\n for y, style in zip([cdpp6_1, cdpp6], ['yo', 'bo']):\n by = np.zeros_like(bins) * np.nan\n for b, bin in enumerate(bins):\n i = np.where((y > -np.inf) & (y < np.inf) &\n (kp >= bin - 0.5) & (kp < bin + 0.5))[0]\n if len(i) > 10:\n by[b] = np.median(y[i])\n ax[0].plot(bins, by, style, markeredgecolor='w')\n else:\n ax[0].scatter(kepler_kp, kepler_cdpp6, color='y',\n marker='.', alpha=alpha_kepler)\n ax[0].scatter(kp, cdpp6, color='b', marker='.',\n alpha=alpha_unsat, picker=True)\n for x, y, style in zip([kepler_kp, kp], [kepler_cdpp6, cdpp6],\n ['yo', 'bo']):\n by = np.zeros_like(bins) * np.nan\n for b, bin in enumerate(bins):\n i = np.where((y > -np.inf) & (y < np.inf) &\n (x >= bin - 0.5) & (x < bin + 0.5))[0]\n if len(i) > 10:\n by[b] = np.median(y[i])\n ax[0].plot(bins, by, style, markeredgecolor='w')\n ax[0].set_ylim(-10, 500)\n ax[0].set_xlim(8, 18)\n ax[0].set_xlabel('Kepler Magnitude', fontsize=18)\n ax[0].set_title('CDPP6 (ppm)', fontsize=18)\n\n # ------ 2. Plot the equivalent of the Aigrain+16 figure\n if compare_to.lower() != 'kepler':\n x = kp\n y = (cdpp6 - cdpp6_1) / cdpp6_1\n yv = (cdpp6v - cdpp6_1) / cdpp6_1\n ax[1].scatter(x[unsat], y[unsat], color='b', marker='.',\n alpha=alpha_unsat, zorder=-1, picker=True)\n ax[1].scatter(x[sat], y[sat], color='r', marker='.',\n alpha=alpha_sat, zorder=-1, picker=True)\n ax[1].set_ylim(-1, 1)\n ax[1].set_xlim(8, 18)\n ax[1].axhline(0, color='gray', lw=2, zorder=-99, alpha=0.5)\n ax[1].axhline(0.5, color='gray', ls='--',\n lw=2, zorder=-99, alpha=0.5)\n ax[1].axhline(-0.5, color='gray', ls='--',\n lw=2, zorder=-99, alpha=0.5)\n bins = np.arange(7.5, 18.5, 0.5)\n # Bin the CDPP\n by = np.zeros_like(bins) * np.nan\n for b, bin in enumerate(bins):\n i = np.where((y > -np.inf) & (y < np.inf) &\n (x >= bin - 0.5) & (x < bin + 0.5))[0]\n if len(i) > 10:\n by[b] = np.median(y[i])\n ax[1].plot(bins[:9], by[:9], 'k--', lw=2)\n ax[1].plot(bins[8:], by[8:], 'k-', lw=2)\n ax[1].set_title(r'Relative CDPP', fontsize=18)\n ax[1].set_xlabel('Kepler Magnitude', fontsize=18)\n\n # ------ 3. Plot the outliers\n i = np.argsort(out)\n a = int(0.95 * len(out))\n omax = out[i][a]\n if compare_to.lower() != 'kepler':\n j = np.argsort(out_1)\n b = int(0.95 * len(out_1))\n omax = max(omax, out_1[j][b])\n ax[2].hist(out, 25, range=(0, omax), histtype='step', color='b')\n if compare_to.lower() != 'kepler':\n ax[2].hist(out_1, 25, range=(0, omax), histtype='step', color='y')\n ax[2].margins(0, None)\n ax[2].set_title('Number of Outliers', fontsize=18)\n\n # Plot the total number of data points\n i = np.argsort(tot)\n a = int(0.05 * len(tot))\n b = int(0.95 * len(tot))\n tmin = tot[i][a]\n tmax = tot[i][b]\n if compare_to.lower() != 'kepler':\n j = np.argsort(tot_1)\n c = int(0.05 * len(tot_1))\n d = int(0.95 * len(tot_1))\n tmin = min(tmin, tot_1[j][c])\n tmax = max(tmax, tot_1[j][d])\n ax[3].hist(tot, 25, range=(tmin, tmax), histtype='step', color='b')\n if compare_to.lower() != 'kepler':\n ax[3].hist(tot_1, 25, range=(tmin, tmax),\n histtype='step', color='y')\n ax[3].margins(0, None)\n ax[3].set_xlabel('Number of Data Points', fontsize=18)\n\n # Pickable points\n Picker = StatsPicker([ax[0], ax[1]], [kp, kp], [\n cdpp6, y], epic, model=model,\n compare_to=compare_to, campaign=campaign)\n fig.canvas.mpl_connect('pick_event', Picker)\n\n # Show\n pl.show()", "response": "Compute the CDPP statistics for all long cadence light curves in a given season."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef HasShortCadence(EPIC, season=None):\n '''\n Returns `True` if short cadence data is available for this target.\n\n :param int EPIC: The EPIC ID number\n :param int season: The campaign number. Default :py:obj:`None`\n\n '''\n\n if season is None:\n season = Campaign(EPIC)\n if season is None:\n return None\n stars = GetK2Campaign(season)\n i = np.where([s[0] == EPIC for s in stars])[0]\n if len(i):\n return stars[i[0]][3]\n else:\n return None", "response": "Returns True if short cadence data is available for this target."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef InjectionStatistics(campaign=0, clobber=False, model='nPLD', plot=True,\n show=True, **kwargs):\n '''\n Computes and plots the statistics for injection/recovery tests.\n\n :param int campaign: The campaign number. Default 0\n :param str model: The :py:obj:`everest` model name\n :param bool plot: Default :py:obj:`True`\n :param bool show: Show the plot? Default :py:obj:`True`. \\\n If :py:obj:`False`, returns the `fig, ax` instances.\n :param bool clobber: Overwrite existing files? Default :py:obj:`False`\n\n '''\n\n # Compute the statistics\n stars = GetK2Campaign(campaign, epics_only=True)\n if type(campaign) is int:\n outfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.inj' % (campaign, model))\n else:\n outfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%04.1f_%s.inj' % (campaign, model))\n if clobber or not os.path.exists(outfile):\n with open(outfile, 'w') as f:\n print(\"EPIC Depth UControl URecovered\"+\n \" MControl MRecovered\", file=f)\n print(\"--------- ---------- ---------- ----------\"+\n \" ---------- ----------\", file=f)\n for i, _ in enumerate(stars):\n sys.stdout.write('\\rProcessing target %d/%d...' %\n (i + 1, len(stars)))\n sys.stdout.flush()\n path = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % int(campaign),\n ('%09d' % stars[i])[:4] + '00000',\n ('%09d' % stars[i])[4:])\n\n # Loop over all depths\n for depth in [0.01, 0.001, 0.0001]:\n\n try:\n\n # Unmasked\n data = np.load(os.path.join(\n path, '%s_Inject_U%g.npz' % (model, depth)))\n assert depth == data['inject'][()]['depth'], \"\"\n ucontrol = data['inject'][()]['rec_depth_control']\n urecovered = data['inject'][()]['rec_depth']\n\n # Masked\n data = np.load(os.path.join(\n path, '%s_Inject_M%g.npz' % (model, depth)))\n assert depth == data['inject'][()]['depth'], \"\"\n mcontrol = data['inject'][()]['rec_depth_control']\n mrecovered = data['inject'][()]['rec_depth']\n\n # Log it\n print(\"{:>09d} {:>13.8f} {:>13.8f} {:>13.8f} {:>13.8f} {:>13.8f}\".format(\n stars[i], depth, ucontrol, urecovered, mcontrol,\n mrecovered), file=f)\n\n except:\n pass\n\n print(\"\")\n\n if plot:\n\n # Load the statistics\n try:\n epic, depth, ucontrol, urecovered, mcontrol, mrecovered = \\\n np.loadtxt(outfile, unpack=True, skiprows=2)\n except ValueError:\n raise Exception(\"No targets to plot.\")\n\n # Normalize to the injected depth\n ucontrol /= depth\n urecovered /= depth\n mcontrol /= depth\n mrecovered /= depth\n\n # Set up the plot\n fig, ax = pl.subplots(3, 2, figsize=(9, 12))\n fig.subplots_adjust(hspace=0.29)\n ax[0, 0].set_title(r'Unmasked', fontsize=18)\n ax[0, 1].set_title(r'Masked', fontsize=18)\n ax[0, 0].set_ylabel(\n r'$D_0 = 10^{-2}$', rotation=90, fontsize=18, labelpad=10)\n ax[1, 0].set_ylabel(\n r'$D_0 = 10^{-3}$', rotation=90, fontsize=18, labelpad=10)\n ax[2, 0].set_ylabel(\n r'$D_0 = 10^{-4}$', rotation=90, fontsize=18, labelpad=10)\n\n # Define some useful stuff for plotting\n depths = [1e-2, 1e-3, 1e-4]\n ranges = [(0.75, 1.25), (0.5, 1.5), (0., 2.)]\n nbins = [30, 30, 20]\n ymax = [0.4, 0.25, 0.16]\n xticks = [[0.75, 0.875, 1., 1.125, 1.25], [\n 0.5, 0.75, 1., 1.25, 1.5], [0., 0.5, 1., 1.5, 2.0]]\n\n # Plot\n for i in range(3):\n\n # Indices for this plot\n idx = np.where(depth == depths[i])\n\n for j, control, recovered in zip([0, 1], [ucontrol[idx],\n mcontrol[idx]],\n [urecovered[idx],\n mrecovered[idx]]):\n\n # Control\n ax[i, j].hist(control, bins=nbins[i], range=ranges[i],\n color='r', histtype='step',\n weights=np.ones_like(control) / len(control))\n\n # Recovered\n ax[i, j].hist(recovered, bins=nbins[i], range=ranges[i],\n color='b', histtype='step',\n weights=np.ones_like(recovered) / len(recovered))\n\n # Indicate center\n ax[i, j].axvline(1., color='k', ls='--')\n\n # Indicate the fraction above and below\n if len(recovered):\n au = len(np.where(recovered > ranges[i][1])[\n 0]) / len(recovered)\n al = len(np.where(recovered < ranges[i][0])[\n 0]) / len(recovered)\n ax[i, j].annotate('%.2f' % al, xy=(0.01, 0.93),\n xycoords='axes fraction',\n xytext=(0.1, 0.93), ha='left',\n va='center', color='b',\n arrowprops=dict(arrowstyle=\"->\",\n color='b'))\n ax[i, j].annotate('%.2f' % au, xy=(0.99, 0.93),\n xycoords='axes fraction',\n xytext=(0.9, 0.93), ha='right',\n va='center', color='b',\n arrowprops=dict(arrowstyle=\"->\",\n color='b'))\n if len(control):\n cu = len(np.where(control > ranges[i][1])[\n 0]) / len(control)\n cl = len(np.where(control < ranges[i][0])[\n 0]) / len(control)\n ax[i, j].annotate('%.2f' % cl, xy=(0.01, 0.86),\n xycoords='axes fraction',\n xytext=(0.1, 0.86), ha='left',\n va='center', color='r',\n arrowprops=dict(arrowstyle=\"->\",\n color='r'))\n ax[i, j].annotate('%.2f' % cu, xy=(0.99, 0.86),\n xycoords='axes fraction',\n xytext=(0.9, 0.86), ha='right',\n va='center', color='r',\n arrowprops=dict(arrowstyle=\"->\",\n color='r'))\n\n # Indicate the median\n if len(recovered):\n ax[i, j].annotate('M = %.2f' % np.median(recovered),\n xy=(0.35, 0.5), ha='right',\n xycoords='axes fraction', color='b',\n fontsize=16)\n if len(control):\n ax[i, j].annotate('M = %.2f' % np.median(control),\n xy=(0.65, 0.5), ha='left',\n xycoords='axes fraction',\n color='r', fontsize=16)\n\n # Tweaks\n ax[i, j].set_xticks(xticks[i])\n ax[i, j].set_xlim(xticks[i][0], xticks[i][-1])\n ax[i, j].set_ylim(-0.005, ymax[i])\n ax[i, j].set_xlabel(r'$D/D_0$', fontsize=16)\n\n ax[i, j].get_yaxis().set_major_locator(MaxNLocator(5))\n for tick in ax[i, j].get_xticklabels() + \\\n ax[i, j].get_yticklabels():\n tick.set_fontsize(14)\n\n if show:\n pl.show()\n else:\n return fig, ax", "response": "Compute the injection statistics for the given campaign and model."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef HDUCards(headers, hdu=0):\n '''\n Generates HDU cards for inclusion in the de-trended light curve FITS file.\n Used internally.\n\n '''\n\n if headers is None:\n return []\n\n if hdu == 0:\n # Get info from the TPF Primary HDU Header\n tpf_header = headers[0]\n entries = ['TELESCOP', 'INSTRUME', 'OBJECT', 'KEPLERID', 'CHANNEL',\n 'MODULE', 'OUTPUT', 'CAMPAIGN', 'DATA_REL', 'OBSMODE',\n 'TTABLEID', 'RADESYS', 'RA_OBJ', 'DEC_OBJ', 'EQUINOX',\n 'KEPMAG']\n elif (hdu == 1) or (hdu == 6):\n # Get info from the TPF BinTable HDU Header\n tpf_header = headers[1]\n entries = ['WCSN4P', 'WCAX4P', '1CTY4P', '2CTY4P',\n '1CUN4P', '2CUN4P', '1CRV4P',\n '2CRV4P', '1CDL4P', '2CDL4P', '1CRP4P',\n '2CRP4P', 'WCAX4', '1CTYP4',\n '2CTYP4', '1CRPX4', '2CRPX4', '1CRVL4',\n '2CRVL4', '1CUNI4', '2CUNI4',\n '1CDLT4', '2CDLT4', '11PC4', '12PC4',\n '21PC4', '22PC4', 'WCSN5P',\n 'WCAX5P', '1CTY5P', '2CTY5P', '1CUN5P',\n '2CUN5P', '1CRV5P', '2CRV5P',\n '1CDL5P', '2CDL5P', '1CRP5P', '2CRP5P',\n 'WCAX5', '1CTYP5', '2CTYP5',\n '1CRPX5', '2CRPX5', '1CRVL5', '2CRVL5',\n '1CUNI5', '2CUNI5', '1CDLT5',\n '2CDLT5', '11PC5', '12PC5', '21PC5',\n '22PC5', 'WCSN6P', 'WCAX6P',\n '1CTY6P', '2CTY6P', '1CUN6P', '2CUN6P',\n '1CRV6P', '2CRV6P', '1CDL6P',\n '2CDL6P', '1CRP6P', '2CRP6P', 'WCAX6',\n '1CTYP6', '2CTYP6', '1CRPX6',\n '2CRPX6', '1CRVL6', '2CRVL6', '1CUNI6',\n '2CUNI6', '1CDLT6', '2CDLT6',\n '11PC6', '12PC6', '21PC6', '22PC6',\n 'WCSN7P', 'WCAX7P', '1CTY7P',\n '2CTY7P', '1CUN7P', '2CUN7P', '1CRV7P',\n '2CRV7P', '1CDL7P', '2CDL7P',\n '1CRP7P', '2CRP7P', 'WCAX7', '1CTYP7',\n '2CTYP7', '1CRPX7', '2CRPX7',\n '1CRVL7', '2CRVL7', '1CUNI7', '2CUNI7',\n '1CDLT7', '2CDLT7', '11PC7',\n '12PC7', '21PC7', '22PC7', 'WCSN8P',\n 'WCAX8P', '1CTY8P', '2CTY8P',\n '1CUN8P', '2CUN8P', '1CRV8P', '2CRV8P',\n '1CDL8P', '2CDL8P', '1CRP8P',\n '2CRP8P', 'WCAX8', '1CTYP8', '2CTYP8',\n '1CRPX8', '2CRPX8', '1CRVL8',\n '2CRVL8', '1CUNI8', '2CUNI8', '1CDLT8',\n '2CDLT8', '11PC8', '12PC8',\n '21PC8', '22PC8', 'WCSN9P', 'WCAX9P',\n '1CTY9P', '2CTY9P', '1CUN9P',\n '2CUN9P', '1CRV9P', '2CRV9P', '1CDL9P',\n '2CDL9P', '1CRP9P', '2CRP9P',\n 'WCAX9', '1CTYP9', '2CTYP9', '1CRPX9',\n '2CRPX9', '1CRVL9', '2CRVL9',\n '1CUNI9', '2CUNI9', '1CDLT9', '2CDLT9',\n '11PC9', '12PC9', '21PC9',\n '22PC9', 'INHERIT', 'EXTNAME', 'EXTVER',\n 'TELESCOP', 'INSTRUME',\n 'OBJECT', 'KEPLERID', 'RADESYS', 'RA_OBJ',\n 'DEC_OBJ', 'EQUINOX',\n 'EXPOSURE', 'TIMEREF', 'TASSIGN', 'TIMESYS',\n 'BJDREFI', 'BJDREFF',\n 'TIMEUNIT', 'TELAPSE', 'LIVETIME', 'TSTART',\n 'TSTOP', 'LC_START',\n 'LC_END', 'DEADC', 'TIMEPIXR', 'TIERRELA',\n 'INT_TIME', 'READTIME',\n 'FRAMETIM', 'NUM_FRM', 'TIMEDEL', 'DATE-OBS',\n 'DATE-END', 'BACKAPP',\n 'DEADAPP', 'VIGNAPP', 'GAIN', 'READNOIS',\n 'NREADOUT', 'TIMSLICE',\n 'MEANBLCK', 'LCFXDOFF', 'SCFXDOFF']\n elif (hdu == 3) or (hdu == 4) or (hdu == 5):\n # Get info from the TPF BinTable HDU Header\n tpf_header = headers[2]\n entries = ['TELESCOP', 'INSTRUME', 'OBJECT', 'KEPLERID', 'RADESYS',\n 'RA_OBJ', 'DEC_OBJ', 'EQUINOX', 'WCSAXES', 'CTYPE1',\n 'CTYPE2', 'CRPIX1', 'CRPIX2', 'CRVAL1', 'CRVAL2',\n 'CUNIT1', 'CUNIT2', 'CDELT1', 'CDELT2', 'PC1_1',\n 'PC1_2', 'PC2_1', 'PC2_2', 'WCSNAMEP', 'WCSAXESP',\n 'CTYPE1P', 'CUNIT1P', 'CRPIX1P', 'CRVAL1P', 'CDELT1P',\n 'CTYPE2P', 'CUNIT2P', 'CRPIX2P', 'CRVAL2P', 'CDELT2P',\n 'NPIXSAP', 'NPIXMISS']\n else:\n return []\n\n cards = []\n cards.append(('COMMENT', '************************'))\n cards.append(('COMMENT', '* MISSION INFO *'))\n cards.append(('COMMENT', '************************'))\n for entry in entries:\n try:\n cards.append(tuple(tpf_header[entry]))\n except KeyError:\n pass\n return cards", "response": "Generates HDU cards for inclusion in the de - trended light curve FITS file."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the location of theeverest data on disk for a given target.", "response": "def TargetDirectory(ID, season, relative=False, **kwargs):\n '''\n Returns the location of the :py:mod:`everest` data on disk\n for a given target.\n\n :param ID: The target ID\n :param int season: The target season number\n :param bool relative: Relative path? Default :py:obj:`False`\n\n '''\n\n if season is None:\n return None\n if relative:\n path = ''\n else:\n path = EVEREST_DAT\n return os.path.join(path, 'k2', 'c%02d' % season,\n ('%09d' % ID)[:4] + '00000',\n ('%09d' % ID)[4:])"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the name of the DVS PDF for a given target.", "response": "def DVSFile(ID, season, cadence='lc'):\n '''\n Returns the name of the DVS PDF for a given target.\n\n :param ID: The target ID\n :param int season: The target season number\n :param str cadence: The cadence type. Default `lc`\n\n '''\n\n if cadence == 'sc':\n strcadence = '_sc'\n else:\n strcadence = ''\n return 'hlsp_everest_k2_llc_%d-c%02d_kepler_v%s_dvs%s.pdf' \\\n % (ID, season, EVEREST_MAJOR_MINOR, strcadence)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetTargetCBVs(model):\n '''\n Returns the design matrix of CBVs for the given target.\n\n :param model: An instance of the :py:obj:`everest` model for the target\n\n '''\n\n # Get the info\n season = model.season\n name = model.name\n\n # We use the LC light curves as CBVs; there aren't\n # enough SC light curves to get a good set\n if name.endswith('.sc'):\n name = name[:-3]\n\n model.XCBV = sysrem.GetCBVs(season, model=name,\n niter=model.cbv_niter,\n sv_win=model.cbv_win,\n sv_order=model.cbv_order)", "response": "Returns the design matrix of CBVs for the given target."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef FitCBVs(model):\n '''\n Fits the CBV design matrix to the de-trended flux of a given target. This\n is called internally whenever the user accesses the :py:attr:`fcor`\n attribute.\n\n :param model: An instance of the :py:obj:`everest` model for the target\n\n '''\n\n # Get cbvs?\n if model.XCBV is None:\n GetTargetCBVs(model)\n\n # The number of CBVs to use\n ncbv = model.cbv_num\n\n # Need to treat short and long cadences differently\n if model.cadence == 'lc':\n\n # Loop over all the light curve segments\n m = [None for b in range(len(model.breakpoints))]\n weights = [None for b in range(len(model.breakpoints))]\n for b in range(len(model.breakpoints)):\n\n # Get the indices for this light curve segment\n inds = model.get_chunk(b, pad=False)\n masked_inds = model.get_masked_chunk(b, pad=False)\n\n # Regress\n mX = model.XCBV[masked_inds, :ncbv + 1]\n A = np.dot(mX.T, mX)\n B = np.dot(mX.T, model.flux[masked_inds])\n try:\n weights[b] = np.linalg.solve(A, B)\n except np.linalg.linalg.LinAlgError:\n # Singular matrix\n log.warn('Singular matrix!')\n weights[b] = np.zeros(mX.shape[1])\n\n m[b] = np.dot(model.XCBV[inds, :ncbv + 1], weights[b])\n\n # Vertical alignment\n if b == 0:\n m[b] -= np.nanmedian(m[b])\n else:\n # Match the first finite model point on either side of the\n # break\n # We could consider something more elaborate in the future\n i0 = -1 - np.argmax([np.isfinite(m[b - 1][-i])\n for i in range(1, len(m[b - 1]) - 1)])\n i1 = np.argmax([np.isfinite(m[b][i])\n for i in range(len(m[b]))])\n m[b] += (m[b - 1][i0] - m[b][i1])\n\n # Join model and normalize\n m = np.concatenate(m)\n m -= np.nanmedian(m)\n\n else:\n\n # Interpolate over outliers so we don't have to worry\n # about masking the arrays below\n flux = Interpolate(model.time, model.mask, model.flux)\n\n # Get downbinned light curve\n newsize = len(model.time) // 30\n time = Downbin(model.time, newsize, operation='mean')\n flux = Downbin(flux, newsize, operation='mean')\n\n # Get LC breakpoints\n breakpoints = list(Breakpoints(\n model.ID, season=model.season, cadence='lc'))\n breakpoints += [len(time) - 1]\n\n # Loop over all the light curve segments\n m = [None for b in range(len(breakpoints))]\n weights = [None for b in range(len(breakpoints))]\n for b in range(len(breakpoints)):\n\n # Get the indices for this light curve segment\n M = np.arange(len(time))\n if b > 0:\n inds = M[(M > breakpoints[b - 1]) & (M <= breakpoints[b])]\n else:\n inds = M[M <= breakpoints[b]]\n\n # Regress\n A = np.dot(model.XCBV[inds, :ncbv + 1].T,\n model.XCBV[inds, :ncbv + 1])\n B = np.dot(model.XCBV[inds, :ncbv + 1].T, flux[inds])\n weights[b] = np.linalg.solve(A, B)\n m[b] = np.dot(model.XCBV[inds, :ncbv + 1], weights[b])\n\n # Vertical alignment\n if b == 0:\n m[b] -= np.nanmedian(m[b])\n else:\n # Match the first finite model point on either side of the\n # break\n # We could consider something more elaborate in the future\n i0 = -1 - np.argmax([np.isfinite(m[b - 1][-i])\n for i in range(1, len(m[b - 1]) - 1)])\n i1 = np.argmax([np.isfinite(m[b][i])\n for i in range(len(m[b]))])\n m[b] += (m[b - 1][i0] - m[b][i1])\n\n # Join model and normalize\n m = np.concatenate(m)\n m -= np.nanmedian(m)\n\n # Finally, interpolate back to short cadence\n m = np.interp(model.time, time, m)\n\n return m", "response": "Fits the CBV design matrix to the de - trended flux of a given target."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngenerating the CSV file used in the search database for the documentation.", "response": "def StatsToCSV(campaign, model='nPLD'):\n '''\n Generate the CSV file used in the search database for the documentation.\n '''\n statsfile = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d_%s.cdpp' % (campaign, model))\n csvfile = os.path.join(os.path.dirname(EVEREST_SRC), 'docs',\n 'c%02d.csv' % campaign)\n epic, kp, cdpp6r, cdpp6, _, _, _, _, saturated = \\\n np.loadtxt(statsfile, unpack=True, skiprows=2)\n\n with open(csvfile, 'w') as f:\n print('c%02d' % campaign, file=f)\n for i in range(len(epic)):\n print('%09d,%.3f,%.3f,%.3f,%d' % (epic[i], kp[i],\n cdpp6r[i], cdpp6[i],\n int(saturated[i])),\n file=f)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef remove(self, item):\r\n '''Remove ``item`` for the :class:`zset` it it exists.\r\nIf found it returns the score of the item removed.'''\r\n score = self._dict.pop(item, None)\r\n if score is not None:\r\n self._sl.remove(score)\r\n return score", "response": "Remove an item from the set."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a tuple of ordered fields for this : class : ColumnTS.", "response": "def fields(self):\r\n '''Return a tuple of ordered fields for this :class:`ColumnTS`.'''\r\n key = self.id + ':fields'\r\n encoding = self.client.encoding\r\n return tuple(sorted((f.decode(encoding)\r\n for f in self.client.smembers(key))))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef do_pending_lookups(event, sender, **kwargs):\r\n key = (sender._meta.app_label, sender._meta.name)\r\n for callback in pending_lookups.pop(key, []):\r\n callback(sender)", "response": "Handle any pending relations to the sending model.\r\nSent from class_prepared."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Many2ManyThroughModel(field):\r\n '''Create a Many2Many through model with two foreign key fields and a\r\nCompositeFieldId depending on the two foreign keys.'''\r\n from stdnet.odm import ModelType, StdModel, ForeignKey, CompositeIdField\r\n name_model = field.model._meta.name\r\n name_relmodel = field.relmodel._meta.name\r\n # The two models are the same.\r\n if name_model == name_relmodel:\r\n name_relmodel += '2'\r\n through = field.through\r\n # Create the through model\r\n if through is None:\r\n name = '{0}_{1}'.format(name_model, name_relmodel)\r\n\r\n class Meta:\r\n app_label = field.model._meta.app_label\r\n through = ModelType(name, (StdModel,), {'Meta': Meta})\r\n field.through = through\r\n # The first field\r\n field1 = ForeignKey(field.model,\r\n related_name=field.name,\r\n related_manager_class=makeMany2ManyRelatedManager(\r\n field.relmodel,\r\n name_model,\r\n name_relmodel)\r\n )\r\n field1.register_with_model(name_model, through)\r\n # The second field\r\n field2 = ForeignKey(field.relmodel,\r\n related_name=field.related_name,\r\n related_manager_class=makeMany2ManyRelatedManager(\r\n field.model,\r\n name_relmodel,\r\n name_model)\r\n )\r\n field2.register_with_model(name_relmodel, through)\r\n pk = CompositeIdField(name_model, name_relmodel)\r\n pk.register_with_model('id', through)", "response": "Create a Many2Many through model with two foreign key fields and a\r\nCompositeFieldId depending on the two foreign keys."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef makeMany2ManyRelatedManager(formodel, name_relmodel, name_formodel):\r\n '''formodel is the model which the manager .'''\r\n\r\n class _Many2ManyRelatedManager(Many2ManyRelatedManager):\r\n pass\r\n\r\n _Many2ManyRelatedManager.formodel = formodel\r\n _Many2ManyRelatedManager.name_relmodel = name_relmodel\r\n _Many2ManyRelatedManager.name_formodel = name_formodel\r\n return _Many2ManyRelatedManager", "response": "Creates a new Many2ManyRelatedManager class."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\noverriding this. manager. session so that the manager can retrieve the session from the related_instance.", "response": "def session(self, session=None):\r\n '''Override :meth:`Manager.session` so that this\r\n :class:`RelatedManager` can retrieve the session from the\r\n :attr:`related_instance` if available.\r\n '''\r\n if self.related_instance:\r\n session = self.related_instance.session\r\n # we have a session, we either create a new one return the same session\r\n if session is None:\r\n raise QuerySetError('Related manager can be accessed only from\\\r\n a loaded instance of its related model.')\r\n return session"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds value an instance of Model to the Model.", "response": "def add(self, value, session=None, **kwargs):\r\n '''Add ``value``, an instance of :attr:`formodel` to the\r\n:attr:`through` model. This method can only be accessed by an instance of the\r\nmodel for which this related manager is an attribute.'''\r\n s, instance = self.session_instance('add', value, session, **kwargs)\r\n return s.add(instance)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef remove(self, value, session=None):\r\n '''Remove *value*, an instance of ``self.model`` from the set of\r\nelements contained by the field.'''\r\n s, instance = self.session_instance('remove', value, session)\r\n # update state so that the instance does look persistent\r\n instance.get_state(iid=instance.pkvalue(), action='update')\r\n return s.delete(instance)", "response": "Remove value from the set of\r\nelements contained by the field."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef metaphone_processor(words):\n '''Double metaphone word processor.'''\n for word in words:\n for w in double_metaphone(word):\n if w:\n w = w.strip()\n if w:\n yield w", "response": "Double metaphone word processor."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ndoubles metaphone word processor slightly modified so that when no noMagicWords are returned by the algorithm the original word is returned.", "response": "def tolerant_metaphone_processor(words):\n '''Double metaphone word processor slightly modified so that when no\nwords are returned by the algorithm, the original word is returned.'''\n for word in words:\n r = 0\n for w in double_metaphone(word):\n if w:\n w = w.strip()\n if w:\n r += 1\n yield w\n if not r:\n yield word"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef stemming_processor(words):\n '''Porter Stemmer word processor'''\n stem = PorterStemmer().stem\n for word in words:\n word = stem(word, 0, len(word)-1)\n yield word", "response": "Porter Stemmer word processor"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a new instance of the specified pool.", "response": "def Pool(pool='AnyPool', **kwargs):\n '''\n Chooses between the different pools.\n If ``pool == 'AnyPool'``, chooses based on availability.\n\n '''\n\n if pool == 'MPIPool':\n return MPIPool(**kwargs)\n elif pool == 'MultiPool':\n return MultiPool(**kwargs)\n elif pool == 'SerialPool':\n return SerialPool(**kwargs)\n elif pool == 'AnyPool':\n if MPIPool.enabled():\n return MPIPool(**kwargs)\n elif MultiPool.enabled():\n return MultiPool(**kwargs)\n else:\n return SerialPool(**kwargs)\n else:\n raise ValueError('Invalid pool ``%s``.' % pool)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwaits for a new job to be executed and return it asynchronously.", "response": "def wait(self):\n \"\"\"\n If this isn't the master process, wait for instructions.\n\n \"\"\"\n if self.is_master():\n raise RuntimeError(\"Master node told to await jobs.\")\n\n status = MPI.Status()\n\n while True:\n # Event loop.\n # Sit here and await instructions.\n if self.debug:\n print(\"Worker {0} waiting for task.\".format(self.rank))\n\n # Blocking receive to wait for instructions.\n task = self.comm.recv(source=0, tag=MPI.ANY_TAG, status=status)\n\n if self.debug:\n print(\"Worker {0} got task {1} with tag {2}.\"\n .format(self.rank, type(task), status.tag))\n\n # Check if message is special sentinel signaling end.\n # If so, stop.\n if isinstance(task, _close_pool_message):\n if self.debug:\n print(\"Worker {0} told to quit.\".format(self.rank))\n break\n\n # Check if message is special type containing new function\n # to be applied\n if isinstance(task, _function_wrapper):\n self.function = task.function\n if self.debug:\n print(\"Worker {0} replaced its task function: {1}.\"\n .format(self.rank, self.function))\n continue\n\n # If not a special message, just run the known function on\n # the input and return it asynchronously.\n result = self.function(task)\n if self.debug:\n print(\"Worker {0} sending answer {1} with tag {2}.\"\n .format(self.rank, type(result), status.tag))\n self.comm.isend(result, dest=0, tag=status.tag)\n\n # Kill the process?\n if self.exit_on_end:\n sys.exit()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef map(self, function, tasks):\n ntask = len(tasks)\n\n # If not the master just wait for instructions.\n if not self.is_master():\n self.wait()\n return\n\n if function is not self.function:\n if self.debug:\n print(\"Master replacing pool function with {0}.\"\n .format(function))\n\n self.function = function\n F = _function_wrapper(function)\n\n # Tell all the workers what function to use.\n requests = []\n for i in range(self.size):\n r = self.comm.isend(F, dest=i + 1)\n requests.append(r)\n\n # Wait until all of the workers have responded. See:\n # https://gist.github.com/4176241\n MPI.Request.waitall(requests)\n\n if (not self.loadbalance) or (ntask <= self.size):\n # Do not perform load-balancing - the default load-balancing\n # scheme emcee uses.\n\n # Send all the tasks off and wait for them to be received.\n # Again, see the bug in the above gist.\n requests = []\n for i, task in enumerate(tasks):\n worker = i % self.size + 1\n if self.debug:\n print(\"Sent task {0} to worker {1} with tag {2}.\"\n .format(type(task), worker, i))\n r = self.comm.isend(task, dest=worker, tag=i)\n requests.append(r)\n\n MPI.Request.waitall(requests)\n\n # Now wait for the answers.\n results = []\n for i in range(ntask):\n worker = i % self.size + 1\n if self.debug:\n print(\"Master waiting for worker {0} with tag {1}\"\n .format(worker, i))\n result = self.comm.recv(source=worker, tag=i)\n\n results.append(result)\n\n return results\n\n else:\n # Perform load-balancing. The order of the results are likely to\n # be different from the previous case.\n for i, task in enumerate(tasks[0:self.size]):\n worker = i + 1\n if self.debug:\n print(\"Sent task {0} to worker {1} with tag {2}.\"\n .format(type(task), worker, i))\n # Send out the tasks asynchronously.\n self.comm.isend(task, dest=worker, tag=i)\n\n ntasks_dispatched = self.size\n results = [None] * ntask\n for itask in range(ntask):\n status = MPI.Status()\n # Receive input from workers.\n try:\n result = self.comm.recv(source=MPI.ANY_SOURCE,\n tag=MPI.ANY_TAG, status=status)\n except Exception as e:\n self.close()\n raise e\n\n worker = status.source\n i = status.tag\n results[i] = result\n if self.debug:\n print(\"Master received from worker {0} with tag {1}\"\n .format(worker, i))\n\n # Now send the next task to this idle worker (if there are any\n # left).\n if ntasks_dispatched < ntask:\n task = tasks[ntasks_dispatched]\n i = ntasks_dispatched\n if self.debug:\n print(\"Sent task {0} to worker {1} with tag {2}.\"\n .format(type(task), worker, i))\n # Send out the tasks asynchronously.\n self.comm.isend(task, dest=worker, tag=i)\n ntasks_dispatched += 1\n\n return results", "response": "This function is a wrapper for the function to apply a function to all the values in a list and return the list of results."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsend a close_pool_message to all the members which contains the special _close_pool_message sentinel.", "response": "def close(self):\n \"\"\"\n Just send a message off to all the pool members which contains\n the special :class:`_close_pool_message` sentinel.\n\n \"\"\"\n if self.is_master():\n for i in range(self.size):\n self.comm.isend(_close_pool_message(), dest=i + 1)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the : class : stdnet. BackendStructure.", "response": "def backend(self):\r\n '''Returns the :class:`stdnet.BackendStructure`.\r\n '''\r\n session = self.session\r\n if session is not None:\r\n if self._field:\r\n return session.model(self._field.model).backend\r\n else:\r\n return session.model(self).backend"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the : class : stdnet. BackendStructure.", "response": "def read_backend(self):\r\n '''Returns the :class:`stdnet.BackendStructure`.\r\n '''\r\n session = self.session\r\n if session is not None:\r\n if self._field:\r\n return session.model(self._field.model).read_backend\r\n else:\r\n return session.model(self).read_backend"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef size(self):\r\n '''Number of elements in the :class:`Structure`.'''\r\n if self.cache.cache is None:\r\n return self.read_backend_structure().size()\r\n else:\r\n return len(self.cache.cache)", "response": "Returns the size of the elements in the structure."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_data(self, data, callback=None):\r\n '''Load ``data`` from the :class:`stdnet.BackendDataServer`.'''\r\n return self.backend.execute(\r\n self.value_pickler.load_iterable(data, self.session), callback)", "response": "Load data from the backend data server."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef values(self):\r\n '''Iteratir over values of :class:`PairMixin`.'''\r\n if self.cache.cache is None:\r\n backend = self.read_backend\r\n return backend.execute(backend.structure(self).values(),\r\n self.load_values)\r\n else:\r\n return self.cache.cache.values()", "response": "Iteratir over values of PairMixin."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pair(self, pair):\r\n '''Add a *pair* to the structure.'''\r\n if len(pair) == 1:\r\n # if only one value is passed, the value must implement a\r\n # score function which retrieve the first value of the pair\r\n # (score in zset, timevalue in timeseries, field value in\r\n # hashtable)\r\n return (pair[0].score(), pair[0])\r\n elif len(pair) != 2:\r\n raise TypeError('add expected 2 arguments, got {0}'\r\n .format(len(pair)))\r\n else:\r\n return pair", "response": "Add a pair to the structure."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nremove the specified keys from the key - value container.", "response": "def remove(self, *keys):\r\n '''Remove *keys* from the key-value container.'''\r\n dumps = self.pickler.dumps\r\n self.cache.remove([dumps(v) for v in keys])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef count(self, start, stop):\r\n '''Count the number of elements bewteen *start* and *stop*.'''\r\n s1 = self.pickler.dumps(start)\r\n s2 = self.pickler.dumps(stop)\r\n return self.backend_structure().count(s1, s2)", "response": "Count the number of elements bewteen start and stop."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef irange(self, start=0, end=-1, callback=None, withscores=True,\r\n **options):\r\n '''Return the range by rank between start and end.'''\r\n backend = self.read_backend\r\n res = backend.structure(self).irange(start, end,\r\n withscores=withscores,\r\n **options)\r\n if not callback:\r\n callback = self.load_data if withscores else self.load_values\r\n return backend.execute(res, callback)", "response": "Return the range by rank between start and end."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pop_range(self, start, stop, callback=None, withscores=True):\r\n '''pop a range by score from the :class:`OrderedMixin`'''\r\n s1 = self.pickler.dumps(start)\r\n s2 = self.pickler.dumps(stop)\r\n backend = self.backend\r\n res = backend.structure(self).pop_range(s1, s2, withscores=withscores)\r\n if not callback:\r\n callback = self.load_data if withscores else self.load_values\r\n return backend.execute(res, callback)", "response": "pop a range by score from the OrderedMixin"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ipop_range(self, start=0, stop=-1, callback=None, withscores=True):\r\n '''pop a range from the :class:`OrderedMixin`'''\r\n backend = self.backend\r\n res = backend.structure(self).ipop_range(start, stop,\r\n withscores=withscores)\r\n if not callback:\r\n callback = self.load_data if withscores else self.load_values\r\n return backend.execute(res, callback)", "response": "pop a range from the OrderedMixin"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nappends a copy of value at the end of the Sequence.", "response": "def push_back(self, value):\r\n '''Appends a copy of *value* at the end of the :class:`Sequence`.'''\r\n self.cache.push_back(self.value_pickler.dumps(value))\r\n return self"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nremove the last element from the Sequence.", "response": "def pop_back(self):\r\n '''Remove the last element from the :class:`Sequence`.'''\r\n backend = self.backend\r\n return backend.execute(backend.structure(self).pop_back(),\r\n self.value_pickler.loads)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add(self, value):\r\n '''Add *value* to the set'''\r\n return self.cache.update((self.value_pickler.dumps(value),))", "response": "Add value to the set"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds iterable values to the set", "response": "def update(self, values):\r\n '''Add iterable *values* to the set'''\r\n d = self.value_pickler.dumps\r\n return self.cache.update(tuple((d(v) for v in values)))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef discard(self, value):\r\n '''Remove an element *value* from a set if it is a member.'''\r\n return self.cache.remove((self.value_pickler.dumps(value),))", "response": "Remove an element value from a set if it is a member."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nremoves an iterable of values from the set.", "response": "def difference_update(self, values):\r\n '''Remove an iterable of *values* from the set.'''\r\n d = self.value_pickler.dumps\r\n return self.cache.remove(tuple((d(v) for v in values)))"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nremoves the first element from the list.", "response": "def pop_front(self):\r\n '''Remove the first element from of the list.'''\r\n backend = self.backend\r\n return backend.execute(backend.structure(self).pop_front(),\r\n self.value_pickler.loads)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef block_pop_back(self, timeout=10):\r\n '''Remove the last element from of the list. If no elements are\r\navailable, blocks for at least ``timeout`` seconds.'''\r\n value = yield self.backend_structure().block_pop_back(timeout)\r\n if value is not None:\r\n yield self.value_pickler.loads(value)", "response": "Remove the last element from the list."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef block_pop_front(self, timeout=10):\r\n '''Remove the first element from of the list. If no elements are\r\navailable, blocks for at least ``timeout`` seconds.'''\r\n value = yield self.backend_structure().block_pop_front(timeout)\r\n if value is not None:\r\n yield self.value_pickler.loads(value)", "response": "Remove the first element from the list."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef push_front(self, value):\r\n '''Appends a copy of ``value`` to the beginning of the list.'''\r\n self.cache.push_front(self.value_pickler.dumps(value))", "response": "Appends a copy of value to the beginning of the list."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the rank of a given dte in the timeseries", "response": "def rank(self, dte):\r\n '''The rank of a given *dte* in the timeseries'''\r\n timestamp = self.pickler.dumps(dte)\r\n return self.backend_structure().rank(timestamp)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npop a value at index from the : class:`TS. Return None if no such value is found.", "response": "def ipop(self, index):\r\n '''Pop a value at *index* from the :class:`TS`. Return ``None`` if\r\nindex is not out of bound.'''\r\n backend = self.backend\r\n res = backend.structure(self).ipop(index)\r\n return backend.execute(res,\r\n lambda r: self._load_get_data(r, index, None))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef times(self, start, stop, callback=None, **kwargs):\r\n '''The times between times *start* and *stop*.'''\r\n s1 = self.pickler.dumps(start)\r\n s2 = self.pickler.dumps(stop)\r\n backend = self.read_backend\r\n res = backend.structure(self).times(s1, s2, **kwargs)\r\n return backend.execute(res, callback or self.load_keys)", "response": "The times between times start and stop."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef itimes(self, start=0, stop=-1, callback=None, **kwargs):\r\n '''The times between rank *start* and *stop*.'''\r\n backend = self.read_backend\r\n res = backend.structure(self).itimes(start, stop, **kwargs)\r\n return backend.execute(res, callback or self.load_keys)", "response": "The times between rank start and stop."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_field(self, field):\r\n '''A :class:`Q` performs a series of operations and ultimately\r\ngenerate of set of matched elements ``ids``. If on the other hand, a\r\ndifferent field is required, it can be specified with the :meth:`get_field`\r\nmethod. For example, lets say a model has a field called ``object_id``\r\nwhich contains ids of another model, we could use::\r\n\r\n qs = session.query(MyModel).get_field('object_id')\r\n\r\nto obtain a set containing the values of matched elements ``object_id``\r\nfields.\r\n\r\n:parameter field: the name of the field which will be used to obtained the\r\n matched elements value. Must be an index.\r\n:rtype: a new :class:`Q` instance.\r\n'''\r\n if field != self._get_field:\r\n if field not in self._meta.dfields:\r\n raise QuerySetError('Model \"{0}\" has no field \"{1}\".'\r\n .format(self._meta, field))\r\n q = self._clone()\r\n q.data['get_field'] = field\r\n return q\r\n else:\r\n return self", "response": "A method to obtain a new instance of the same field as the current one."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef filter(self, **kwargs):\r\n '''Create a new :class:`Query` with additional clauses corresponding to\r\n``where`` or ``limit`` in a ``SQL SELECT`` statement.\r\n\r\n:parameter kwargs: dictionary of limiting clauses.\r\n:rtype: a new :class:`Query` instance.\r\n\r\nFor example::\r\n\r\n qs = session.query(MyModel)\r\n result = qs.filter(group = 'planet')\r\n'''\r\n if kwargs:\r\n q = self._clone()\r\n if self.fargs:\r\n kwargs = update_dictionary(self.fargs.copy(), kwargs)\r\n q.fargs = kwargs\r\n return q\r\n else:\r\n return self", "response": "Create a new Query instance with additional clauses corresponding to\r\n where or limit in a SQL SELECT statement."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef exclude(self, **kwargs):\r\n '''Returns a new :class:`Query` with additional clauses corresponding\r\nto ``EXCEPT`` in a ``SQL SELECT`` statement.\r\n\r\n:parameter kwargs: dictionary of limiting clauses.\r\n:rtype: a new :class:`Query` instance.\r\n\r\nUsing an equivalent example to the :meth:`filter` method::\r\n\r\n qs = session.query(MyModel)\r\n result1 = qs.exclude(group = 'planet')\r\n result2 = qs.exclude(group__in = ('planet','stars'))\r\n\r\n'''\r\n if kwargs:\r\n q = self._clone()\r\n if self.eargs:\r\n kwargs = update_dictionary(self.eargs.copy(), kwargs)\r\n q.eargs = kwargs\r\n return q\r\n else:\r\n return self", "response": "Returns a new Query instance with additional clauses corresponding\r\nto EXCEPT in a SQL SELECT statement."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef union(self, *queries):\r\n '''Return a new :class:`Query` obtained form the union of this\r\n:class:`Query` with one or more *queries*.\r\nFor example, lets say we want to have the union\r\nof two queries obtained from the :meth:`filter` method::\r\n\r\n query = session.query(MyModel)\r\n qs = query.filter(field1 = 'bla').union(query.filter(field2 = 'foo'))\r\n'''\r\n q = self._clone()\r\n q.unions += queries\r\n return q", "response": "Return a new Query that obtained form the union of this Query with one or more queries*."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a new Query obtained form the intersection of this Query with one or more queries. Workds the same way as the union of this Query with one or more queries*. Workds the same way as the union of this Query.", "response": "def intersect(self, *queries):\r\n '''Return a new :class:`Query` obtained form the intersection of this\r\n:class:`Query` with one or more *queries*. Workds the same way as\r\nthe :meth:`union` method.'''\r\n q = self._clone()\r\n q.intersections += queries\r\n return q"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsort the query by the given field name.", "response": "def sort_by(self, ordering):\r\n '''Sort the query by the given field\r\n\r\n:parameter ordering: a string indicating the class:`Field` name to sort by.\r\n If prefixed with ``-``, the sorting will be in descending order, otherwise\r\n in ascending order.\r\n:return type: a new :class:`Query` instance.\r\n'''\r\n if ordering:\r\n ordering = self._meta.get_sorting(ordering, QuerySetError)\r\n q = self._clone()\r\n q.data['ordering'] = ordering\r\n return q"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef search(self, text, lookup=None):\r\n '''Search *text* in model. A search engine needs to be installed\r\nfor this function to be available.\r\n\r\n:parameter text: a string to search.\r\n:return type: a new :class:`Query` instance.\r\n'''\r\n q = self._clone()\r\n q.text = (text, lookup)\r\n return q", "response": "Search *text* in model. A search engine needs to be installed\r\nfor this function to be available.\r\n\r\n:parameter text: a string to search.\r\n:return type: a new :class:`Query` instance."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef where(self, code, load_only=None):\r\n '''For :ref:`backend ` supporting scripting, it is possible\r\nto construct complex queries which execute the scripting *code* against\r\neach element in the query. The *coe* should reference an instance of\r\n:attr:`model` by ``this`` keyword.\r\n\r\n:parameter code: a valid expression in the scripting language of the database.\r\n:parameter load_only: Load only the selected fields when performing the query\r\n (this is different from the :meth:`load_only` method which is used when\r\n fetching data from the database). This field is an optimization which is\r\n used by the :ref:`redis backend ` only and can be safely\r\n ignored in most use-cases.\r\n:return: a new :class:`Query`\r\n'''\r\n if code:\r\n q = self._clone()\r\n q.data['where'] = (code, load_only)\r\n return q\r\n else:\r\n return self", "response": "Returns a new Query object with the data set where code = code."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef search_queries(self, q):\r\n '''Return a new :class:`QueryElem` for *q* applying a text search.'''\r\n if self.text:\r\n searchengine = self.session.router.search_engine\r\n if searchengine:\r\n return searchengine.search_model(q, *self.text)\r\n else:\r\n raise QuerySetError('Search not available for %s' % self._meta)\r\n else:\r\n return q", "response": "Return a new : class : QueryElem for *q* applying a text search."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_related(self, related, *related_fields):\r\n '''It returns a new :class:`Query` that automatically\r\nfollows the foreign-key relationship ``related``.\r\n\r\n:parameter related: A field name corresponding to a :class:`ForeignKey`\r\n in :attr:`Query.model`.\r\n:parameter related_fields: optional :class:`Field` names for the ``related``\r\n model to load. If not provided, all fields will be loaded.\r\n\r\nThis function is :ref:`performance boost ` when\r\naccessing the related fields of all (most) objects in your query.\r\n\r\nIf Your model contains more than one foreign key, you can use this function\r\nin a generative way::\r\n\r\n qs = myquery.load_related('rel1').load_related('rel2','field1','field2')\r\n\r\n:rtype: a new :class:`Query`.'''\r\n field = self._get_related_field(related)\r\n if not field:\r\n raise FieldError('\"%s\" is not a related field for \"%s\"' %\r\n (related, self._meta))\r\n q = self._clone()\r\n return q._add_to_load_related(field, *related_fields)", "response": "Returns a new : class:`Query` that automatically follows the foreign - key relationship related."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_only(self, *fields):\r\n '''This is provides a :ref:`performance boost `\r\nin cases when you need to load a subset of fields of your model. The boost\r\nachieved is less than the one obtained when using\r\n:meth:`Query.load_related`, since it does not reduce the number of requests\r\nto the database. However, it can save you lots of bandwidth when excluding\r\ndata intensive fields you don't need.\r\n'''\r\n q = self._clone()\r\n new_fields = []\r\n for field in fields:\r\n if JSPLITTER in field:\r\n bits = field.split(JSPLITTER)\r\n related = self._get_related_field(bits[0])\r\n if related:\r\n q._add_to_load_related(related, JSPLITTER.join(bits[1:]))\r\n continue\r\n new_fields.append(field)\r\n if fields and not new_fields:\r\n # if we added a field to the load_related list and not fields are\r\n # are left we add the primary key so that other firls are not\r\n # loaded.\r\n new_fields.append(self._meta.pkname())\r\n fs = unique_tuple(q.fields, new_fields)\r\n q.data['fields'] = fs if fs else None\r\n return q", "response": "Returns a new instance of this model with only the specified fields."} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a new QuerySet with only the specified fields but not the ones in fields.", "response": "def dont_load(self, *fields):\r\n '''Works like :meth:`load_only` to provides a\r\n:ref:`performance boost ` in cases when you need\r\nto load all fields except a subset specified by *fields*.\r\n'''\r\n q = self._clone()\r\n fs = unique_tuple(q.exclude_fields, fields)\r\n q.exclude_fields = fs if fs else None\r\n return q"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get(self, **kwargs):\r\n '''Return an instance of a model matching the query. A special case is\r\nthe query on ``id`` which provides a direct access to the :attr:`session`\r\ninstances. If the given primary key is present in the session, the object\r\nis returned directly without performing any query.'''\r\n return self.filter(**kwargs).items(\r\n callback=self.model.get_unique_instance)", "response": "Return an instance of a model matching the query."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nbuild the : class:`QueryElement` representing this query.", "response": "def construct(self):\r\n '''Build the :class:`QueryElement` representing this query.'''\r\n if self.__construct is None:\r\n self.__construct = self._construct()\r\n return self.__construct"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nbuild and return the : class : stdnet. utils. async. BackendQuery.", "response": "def backend_query(self, **kwargs):\r\n '''Build and return the :class:`stdnet.utils.async.BackendQuery`.\r\nThis is a lazy method in the sense that it is evaluated once only and its\r\nresult stored for future retrieval.'''\r\n q = self.construct()\r\n return q if isinstance(q, EmptyQuery) else q.backend_query(**kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef aggregate(self, kwargs):\r\n '''Aggregate lookup parameters.'''\r\n meta = self._meta\r\n fields = meta.dfields\r\n field_lookups = {}\r\n for name, value in iteritems(kwargs):\r\n bits = name.split(JSPLITTER)\r\n field_name = bits.pop(0)\r\n if field_name not in fields:\r\n raise QuerySetError('Could not filter on model \"{0}\".\\\r\n Field \"{1}\" does not exist.'.format(meta, field_name))\r\n field = fields[field_name]\r\n attname = field.attname\r\n lookup = None\r\n if bits:\r\n bits = [n.lower() for n in bits]\r\n if bits[-1] == 'in':\r\n bits.pop()\r\n elif bits[-1] in range_lookups:\r\n lookup = bits.pop()\r\n remaining = JSPLITTER.join(bits)\r\n if lookup: # this is a range lookup\r\n attname, nested = field.get_lookup(remaining,\r\n QuerySetError)\r\n lookups = get_lookups(attname, field_lookups)\r\n lookups.append(lookup_value(lookup, (value, nested)))\r\n continue\r\n elif remaining: # Not a range lookup, must be a nested filter\r\n value = field.filter(self.session, remaining, value)\r\n lookups = get_lookups(attname, field_lookups)\r\n # If we are here the field must be an index\r\n if not field.index:\r\n raise QuerySetError(\"%s %s is not an index. Cannot query.\" %\r\n (field.__class__.__name__, field_name))\r\n if not iterable(value):\r\n value = (value,)\r\n for v in value:\r\n if isinstance(v, Q):\r\n v = lookup_value('set', v.construct())\r\n else:\r\n v = lookup_value('value', field.serialise(v, lookup))\r\n lookups.append(v)\r\n #\r\n return [queryset(self, name=name, underlying=field_lookups[name])\r\n for name in sorted(field_lookups)]", "response": "Aggregate the values of the related objects in the model."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef model_iterator(application, include_related=True, exclude=None):\n '''A generator of :class:`StdModel` classes found in *application*.\n\n:parameter application: A python dotted path or an iterable over python\n dotted-paths where models are defined.\n\nOnly models defined in these paths are considered.\n\nFor example::\n\n from stdnet.odm import model_iterator\n\n APPS = ('stdnet.contrib.searchengine',\n 'stdnet.contrib.timeseries')\n\n for model in model_iterator(APPS):\n ...\n\n'''\n if exclude is None:\n exclude = set()\n application = native_str(application)\n if ismodule(application) or isinstance(application, str):\n if ismodule(application):\n mod, application = application, application.__name__\n else:\n try:\n mod = import_module(application)\n except ImportError:\n # the module is not there\n mod = None\n if mod:\n label = application.split('.')[-1]\n try:\n mod_models = import_module('.models', application)\n except ImportError:\n mod_models = mod\n label = getattr(mod_models, 'app_label', label)\n models = set()\n for name in dir(mod_models):\n value = getattr(mod_models, name)\n meta = getattr(value, '_meta', None)\n if isinstance(value, ModelType) and meta:\n for model in models_from_model(\n value, include_related=include_related,\n exclude=exclude):\n if (model._meta.app_label == label\n and model not in models):\n models.add(model)\n yield model\n else:\n for app in application:\n for m in model_iterator(app):\n yield m", "response": "A generator of : class : StdModel classes found in application."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset the search engine for this router.", "response": "def set_search_engine(self, engine):\n '''Set the search ``engine`` for this :class:`Router`.'''\n self._search_engine = engine\n self._search_engine.set_router(self)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef register(self, model, backend=None, read_backend=None,\n include_related=True, **params):\n '''Register a :class:`Model` with this :class:`Router`. If the\nmodel was already registered it does nothing.\n\n:param model: a :class:`Model` class.\n:param backend: a :class:`stdnet.BackendDataServer` or a\n :ref:`connection string `.\n:param read_backend: Optional :class:`stdnet.BackendDataServer` for read\n operations. This is useful when the server has a master/slave\n configuration, where the master accept write and read operations\n and the ``slave`` read only operations.\n:param include_related: ``True`` if related models to ``model`` needs to be\n registered. Default ``True``.\n:param params: Additional parameters for the :func:`getdb` function.\n:return: the number of models registered.\n'''\n backend = backend or self._default_backend\n backend = getdb(backend=backend, **params)\n if read_backend:\n read_backend = getdb(read_backend)\n registered = 0\n if isinstance(model, Structure):\n self._structures[model] = StructureManager(model, backend,\n read_backend, self)\n return model\n for model in models_from_model(model, include_related=include_related):\n if model in self._registered_models:\n continue\n registered += 1\n default_manager = backend.default_manager or Manager\n manager_class = getattr(model, 'manager_class', default_manager)\n manager = manager_class(model, backend, read_backend, self)\n self._registered_models[model] = manager\n if isinstance(model, ModelType):\n attr_name = model._meta.name\n else:\n attr_name = model.__name__.lower()\n if attr_name not in self._registered_names:\n self._registered_names[attr_name] = manager\n if self._install_global:\n model.objects = manager\n if registered:\n return backend", "response": "Register a model with this Router."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef from_uuid(self, uuid, session=None):\n '''Retrieve a :class:`Model` from its universally unique identifier\n``uuid``. If the ``uuid`` does not match any instance an exception will raise.\n'''\n elems = uuid.split('.')\n if len(elems) == 2:\n model = get_model_from_hash(elems[0])\n if not model:\n raise Model.DoesNotExist(\n 'model id \"{0}\" not available'.format(elems[0]))\n if not session or session.router is not self:\n session = self.session()\n return session.query(model).get(id=elems[1])\n raise Model.DoesNotExist('uuid \"{0}\" not recognized'.format(uuid))", "response": "Retrieve a : class:`Model` from its universally unique identifier\nuuid."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef flush(self, exclude=None, include=None, dryrun=False):\n '''Flush :attr:`registered_models`.\n\n :param exclude: optional list of model names to exclude.\n :param include: optional list of model names to include.\n :param dryrun: Doesn't remove anything, simply collect managers\n to flush.\n :return:\n '''\n exclude = exclude or []\n results = []\n for manager in self._registered_models.values():\n m = manager._meta\n if include is not None and not (m.modelkey in include or\n m.app_label in include):\n continue\n if not (m.modelkey in exclude or m.app_label in exclude):\n if dryrun:\n results.append(manager)\n else:\n results.append(manager.flush())\n return results", "response": "Flush all the managers that have the same app_label."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef register_applications(self, applications, models=None, backends=None):\n '''A higher level registration functions for group of models located\non application modules.\nIt uses the :func:`model_iterator` function to iterate\nthrough all :class:`Model` models available in ``applications``\nand register them using the :func:`register` low level method.\n\n:parameter applications: A String or a list of strings representing\n python dotted paths where models are implemented.\n:parameter models: Optional list of models to include. If not provided\n all models found in *applications* will be included.\n:parameter backends: optional dictionary which map a model or an\n application to a backend :ref:`connection string `.\n:rtype: A list of registered :class:`Model`.\n\nFor example::\n\n\n mapper.register_application_models('mylib.myapp')\n mapper.register_application_models(['mylib.myapp', 'another.path'])\n mapper.register_application_models(pythonmodule)\n mapper.register_application_models(['mylib.myapp',pythonmodule])\n\n'''\n return list(self._register_applications(applications, models,\n backends))", "response": "A higher level registration function for group of models located\non application modules."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef execute_script(self, name, keys, *args, **options):\n '''Execute a script.\n\n makes sure all required scripts are loaded.\n '''\n script = get_script(name)\n if not script:\n raise redis.RedisError('No such script \"%s\"' % name)\n address = self.address()\n if address not in all_loaded_scripts:\n all_loaded_scripts[address] = set()\n loaded = all_loaded_scripts[address]\n toload = script.required_scripts.difference(loaded)\n for name in toload:\n s = get_script(name)\n yield self.script_load(s.script)\n loaded.update(toload)\n yield script(self, keys, args, options)", "response": "Execute a script. This is a generator that yields all the required scripts that are loaded."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nregisters a new model with this search engine.", "response": "def register(self, model, related=None):\r\n '''Register a :class:`StdModel` with this search :class:`SearchEngine`.\r\nWhen registering a model, every time an instance is created, it will be\r\nindexed by the search engine.\r\n\r\n:param model: a :class:`StdModel` class.\r\n:param related: a list of related fields to include in the index.\r\n'''\r\n update_model = UpdateSE(self, related)\r\n self.REGISTERED_MODELS[model] = update_model\r\n self.router.post_commit.bind(update_model, model)\r\n self.router.post_delete.bind(update_model, model)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef words_from_text(self, text, for_search=False):\r\n '''Generator of indexable words in *text*.\r\nThis functions loop through the :attr:`word_middleware` attribute\r\nto process the text.\r\n\r\n:param text: string from which to extract words.\r\n:param for_search: flag indicating if the the words will be used for search\r\n or to index the database. This flug is used in conjunction with the\r\n middleware flag *for_search*. If this flag is ``True`` (i.e. we need to\r\n search the database for the words in *text*), only the\r\n middleware functions in :attr:`word_middleware` enabled for searching are\r\n used.\r\n\r\n Default: ``False``.\r\n\r\nreturn a *list* of cleaned words.\r\n'''\r\n if not text:\r\n return []\r\n word_gen = self.split_text(text)\r\n for middleware, fors in self.word_middleware:\r\n if for_search and not fors:\r\n continue\r\n word_gen = middleware(word_gen)\r\n if isgenerator(word_gen):\r\n word_gen = list(word_gen)\r\n return word_gen", "response": "Generator of indexable words in text*."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_word_middleware(self, middleware, for_search=True):\r\n '''Add a *middleware* function to the list of :attr:`word_middleware`,\r\nfor preprocessing words to be indexed.\r\n\r\n:param middleware: a callable receving an iterable over words.\r\n:param for_search: flag indicating if the *middleware* can be used for the\r\n text to search. Default: ``True``.\r\n'''\r\n if hasattr(middleware, '__call__'):\r\n self.word_middleware.append((middleware, for_search))", "response": "Add a middleware function to the list of word_middleware functions that will be called when the word is indexed."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef query(self, model):\r\n '''Return a query for ``model`` when it needs to be indexed.\r\n '''\r\n session = self.router.session()\r\n fields = tuple((f.name for f in model._meta.scalarfields\r\n if f.type == 'text'))\r\n qs = session.query(model).load_only(*fields)\r\n for related in self.get_related_fields(model):\r\n qs = qs.load_related(related)\r\n return qs", "response": "Return a query for model when it needs to be indexed."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a PEP 386 - compliant version number from *version*.", "response": "def get_version(version):\r\n \"Returns a PEP 386-compliant version number from *version*.\"\r\n assert len(version) == 5\r\n assert version[3] in ('alpha', 'beta', 'rc', 'final')\r\n parts = 2 if version[2] == 0 else 3\r\n main = '.'.join(map(str, version[:parts]))\r\n sub = ''\r\n if version[3] == 'alpha' and version[4] == 0:\r\n git_changeset = get_git_changeset()\r\n if git_changeset:\r\n sub = '.dev%s' % git_changeset\r\n elif version[3] != 'final':\r\n mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'c'}\r\n sub = mapping[version[3]] + str(version[4])\r\n return main + sub"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Get_RpRs(d, **kwargs):\n '''\n Returns the value of the planet radius over the stellar radius\n for a given depth :py:obj:`d`, given\n the :py:class:`everest.pysyzygy` transit :py:obj:`kwargs`.\n\n '''\n if ps is None:\n raise Exception(\"Unable to import `pysyzygy`.\")\n\n def Depth(RpRs, **kwargs):\n return 1 - ps.Transit(RpRs=RpRs, **kwargs)([kwargs.get('t0', 0.)])\n\n def DiffSq(r):\n return 1.e10 * (d - Depth(r, **kwargs)) ** 2\n\n return fmin(DiffSq, [np.sqrt(d)], disp=False)", "response": "Returns the value of the planet radius over the stellar radius\n for a given depth."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the value of the stellar density for a given transit duration dur.", "response": "def Get_rhos(dur, **kwargs):\n '''\n Returns the value of the stellar density for a given transit\n duration :py:obj:`dur`, given\n the :py:class:`everest.pysyzygy` transit :py:obj:`kwargs`.\n\n '''\n if ps is None:\n raise Exception(\"Unable to import `pysyzygy`.\")\n\n assert dur >= 0.01 and dur <= 0.5, \"Invalid value for the duration.\"\n\n def Dur(rhos, **kwargs):\n t0 = kwargs.get('t0', 0.)\n time = np.linspace(t0 - 0.5, t0 + 0.5, 1000)\n try:\n t = time[np.where(ps.Transit(rhos=rhos, **kwargs)(time) < 1)]\n except:\n return 0.\n return t[-1] - t[0]\n\n def DiffSq(rhos):\n return (dur - Dur(rhos, **kwargs)) ** 2\n\n return fmin(DiffSq, [0.2], disp=False)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Transit(time, t0=0., dur=0.1, per=3.56789, depth=0.001, **kwargs):\n '''\n A `Mandel-Agol `_\n transit model, but with the depth and the duration as primary\n input variables.\n\n :param numpy.ndarray time: The time array\n :param float t0: The time of first transit in units of \\\n :py:obj:`BJD` - 2454833.\n :param float dur: The transit duration in days. Don't go too crazy on \\\n this one -- very small or very large values will break the \\\n inverter. Default 0.1\n :param float per: The orbital period in days. Default 3.56789\n :param float depth: The fractional transit depth. Default 0.001\n :param dict kwargs: Any additional keyword arguments, passed directly \\\n to :py:func:`pysyzygy.Transit`\n :returns tmod: The transit model evaluated at the same times as the \\\n :py:obj:`time` array\n\n '''\n if ps is None:\n raise Exception(\"Unable to import `pysyzygy`.\")\n\n # Note that rhos can affect RpRs, so we should really do this iteratively,\n # but the effect is pretty negligible!\n RpRs = Get_RpRs(depth, t0=t0, per=per, **kwargs)\n rhos = Get_rhos(dur, t0=t0, per=per, **kwargs)\n return ps.Transit(t0=t0, per=per, RpRs=RpRs, rhos=rhos, **kwargs)(time)", "response": "Returns a transit model evaluated at the specified time."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nevaluate the date intervals from which data is not available. It returns a list of two - dimensional tuples containing start and end dates for the interval.", "response": "def intervals(self, startdate, enddate, parseinterval=None):\r\n '''Given a ``startdate`` and an ``enddate`` dates, evaluate the\r\n date intervals from which data is not available. It return a list\r\n of two-dimensional tuples containing start and end date for the\r\n interval. The list could contain 0, 1 or 2 tuples.'''\r\n return missing_intervals(startdate, enddate, self.data_start,\r\n self.data_end, dateconverter=self.todate,\r\n parseinterval=parseinterval)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the front pair of the structure", "response": "def front(self, *fields):\r\n '''Return the front pair of the structure'''\r\n v, f = tuple(self.irange(0, 0, fields=fields))\r\n if v:\r\n return (v[0], dict(((field, f[field][0]) for field in f)))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef istats(self, start=0, end=-1, fields=None):\r\n '''Perform a multivariate statistic calculation of this\r\n:class:`ColumnTS` from *start* to *end*.\r\n\r\n:param start: Optional index (rank) where to start the analysis.\r\n:param end: Optional index (rank) where to end the analysis.\r\n:param fields: Optional subset of :meth:`fields` to perform analysis on.\r\n If not provided all fields are included in the analysis.\r\n'''\r\n backend = self.read_backend\r\n return backend.execute(\r\n backend.structure(self).istats(start, end, fields), self._stats)", "response": "Perform a multivariate statistic calculation of this\r\n from start to end."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nperforms a multivariate statistic calculation of this Taxonomy object from a start date to an end date.", "response": "def stats(self, start, end, fields=None):\r\n '''Perform a multivariate statistic calculation of this\r\n:class:`ColumnTS` from a *start* date/datetime to an\r\n*end* date/datetime.\r\n\r\n:param start: Start date for analysis.\r\n:param end: End date for analysis.\r\n:param fields: Optional subset of :meth:`fields` to perform analysis on.\r\n If not provided all fields are included in the analysis.\r\n'''\r\n start = self.pickler.dumps(start)\r\n end = self.pickler.dumps(end)\r\n backend = self.read_backend\r\n return backend.execute(\r\n backend.structure(self).stats(start, end, fields), self._stats)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef imulti_stats(self, start=0, end=-1, series=None, fields=None,\r\n stats=None):\r\n '''Perform cross multivariate statistics calculation of\r\nthis :class:`ColumnTS` and other optional *series* from *start*\r\nto *end*.\r\n\r\n:parameter start: the start rank.\r\n:parameter start: the end rank\r\n:parameter field: name of field to perform multivariate statistics.\r\n:parameter series: a list of two elements tuple containing the id of the\r\n a :class:`columnTS` and a field name.\r\n:parameter stats: list of statistics to evaluate.\r\n Default: ['covariance']\r\n'''\r\n stats = stats or self.default_multi_stats\r\n backend = self.read_backend\r\n return backend.execute(\r\n backend.structure(self).imulti_stats(start, end, fields, series,\r\n stats), self._stats)", "response": "Perform cross multivariate statistics calculation of\r\n this object and other optional fields from start to end."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmerge this : class : ColumnTS with several other series.", "response": "def merge(self, *series, **kwargs):\r\n '''Merge this :class:`ColumnTS` with several other *series*.\r\n\r\n:parameters series: a list of tuples where the nth element is a tuple\r\n of the form::\r\n\r\n (wight_n, ts_n1, ts_n2, ..., ts_nMn)\r\n\r\nThe result will be calculated using the formula::\r\n\r\n ts = weight_1*ts_11*ts_12*...*ts_1M1 + weight_2*ts_21*ts_22*...*ts_2M2 +\r\n ...\r\n'''\r\n session = self.session\r\n if not session:\r\n raise SessionNotAvailable('No session available')\r\n self.check_router(session.router, *series)\r\n return self._merge(*series, **kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nmerges series and return the results without storing data in the backend server.", "response": "def merged_series(cls, *series, **kwargs):\r\n '''Merge ``series`` and return the results without storing data\r\nin the backend server.'''\r\n router, backend = cls.check_router(None, *series)\r\n if backend:\r\n target = router.register(cls(), backend)\r\n router.session().add(target)\r\n target._merge(*series, **kwargs)\r\n backend = target.backend\r\n return backend.execute(\r\n backend.structure(target).irange_and_delete(),\r\n target.load_data)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef rank(self, score):\n '''Return the 0-based index (rank) of ``score``. If the score is not\navailable it returns a negative integer which absolute score is the\nleft most closest index with score less than *score*.'''\n node = self.__head\n rank = 0\n for i in range(self.__level-1, -1, -1):\n while node.next[i] and node.next[i].score <= score:\n rank += node.width[i]\n node = node.next[i]\n if node.score == score:\n return rank - 1\n else:\n return -1 - rank", "response": "Return the 0 - based index of score."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a new instance of model from a state tuple.", "response": "def make_object(self, state=None, backend=None):\r\n '''Create a new instance of :attr:`model` from a *state* tuple.'''\r\n model = self.model\r\n obj = model.__new__(model)\r\n self.load_state(obj, state, backend)\r\n return obj"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nperforming validation for the given instance and stores serialized data indexes and errors into local cache. Returns True if the instance is ready to be saved to database.", "response": "def is_valid(self, instance):\r\n '''Perform validation for *instance* and stores serialized data,\r\nindexes and errors into local cache.\r\nReturn ``True`` if the instance is ready to be saved to database.'''\r\n dbdata = instance.dbdata\r\n data = dbdata['cleaned_data'] = {}\r\n errors = dbdata['errors'] = {}\r\n #Loop over scalar fields first\r\n for field, value in instance.fieldvalue_pairs():\r\n name = field.attname\r\n try:\r\n svalue = field.set_get_value(instance, value)\r\n except Exception as e:\r\n errors[name] = str(e)\r\n else:\r\n if (svalue is None or svalue is '') and field.required:\r\n errors[name] = (\"Field '{0}' is required for '{1}'.\"\r\n .format(name, self))\r\n else:\r\n if isinstance(svalue, dict):\r\n data.update(svalue)\r\n elif svalue is not None:\r\n data[name] = svalue\r\n return len(errors) == 0"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a two elements tuple containing a list of field names and a list of field attribute names.", "response": "def backend_fields(self, fields):\r\n '''Return a two elements tuple containing a list\r\nof fields names and a list of field attribute names.'''\r\n dfields = self.dfields\r\n processed = set()\r\n names = []\r\n atts = []\r\n pkname = self.pkname()\r\n for name in fields:\r\n if name == pkname or name in processed:\r\n continue\r\n elif name in dfields:\r\n processed.add(name)\r\n field = dfields[name]\r\n names.append(field.name)\r\n atts.append(field.attname)\r\n else:\r\n bname = name.split(JSPLITTER)[0]\r\n if bname in dfields:\r\n field = dfields[bname]\r\n if field.type in ('json object', 'related object'):\r\n processed.add(name)\r\n names.append(name)\r\n atts.append(name)\r\n return names, atts"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the model metadata in a dictionary", "response": "def as_dict(self):\r\n '''Model metadata in a dictionary'''\r\n pk = self.pk\r\n id_type = 3\r\n if pk.type == 'auto':\r\n id_type = 1\r\n return {'id_name': pk.name,\r\n 'id_type': id_type,\r\n 'sorted': bool(self.ordering),\r\n 'autoincr': self.ordering and self.ordering.auto,\r\n 'multi_fields': [field.name for field in self.multifields],\r\n 'indices': dict(((idx.attname, idx.unique)\r\n for idx in self.indices))}"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the current state for this model.", "response": "def get_state(self, **kwargs):\r\n '''Return the current :class:`ModelState` for this :class:`Model`.\r\nIf ``kwargs`` parameters are passed a new :class:`ModelState` is created,\r\notherwise it returns the cached value.'''\r\n dbdata = self.dbdata\r\n if 'state' not in dbdata or kwargs:\r\n dbdata['state'] = ModelState(self, **kwargs)\r\n return dbdata['state']"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef backend(self, client=None):\r\n '''The :class:`stdnet.BackendDatServer` for this instance.\r\n\r\n It can be ``None``.\r\n '''\r\n session = self.session\r\n if session:\r\n return session.model(self).backend", "response": "The backend for this instance."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a local model class for objects requiring and interface similar to stdnet. BackendDataServer.", "response": "def create_model(name, *attributes, **params):\n '''Create a :class:`Model` class for objects requiring\nand interface similar to :class:`StdModel`. We refers to this type\nof models as :ref:`local models ` since instances of such\nmodels are not persistent on a :class:`stdnet.BackendDataServer`.\n\n:param name: Name of the model class.\n:param attributes: positiona attribute names. These are the only attribute\n available to the model during the default constructor.\n:param params: key-valued parameter to pass to the :class:`ModelMeta`\n constructor.\n:return: a local :class:`Model` class.\n '''\n params['register'] = False\n params['attributes'] = attributes\n kwargs = {'manager_class': params.pop('manager_class', Manager),\n 'Meta': params}\n return ModelType(name, (StdModel,), kwargs)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset cache fields to None. Check : attr : Field. as_cache for information regarding fields which are considered cache.", "response": "def clear_cache_fields(self):\n '''Set cache fields to ``None``. Check :attr:`Field.as_cache`\nfor information regarding fields which are considered cache.'''\n for field in self._meta.scalarfields:\n if field.as_cache:\n setattr(self, field.name, None)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nretrieves the value for the attribute name. The name can be nested following the double underscore notation for example group__name.", "response": "def get_attr_value(self, name):\n '''Retrieve the ``value`` for the attribute ``name``. The ``name``\ncan be nested following the :ref:`double underscore `\nnotation, for example ``group__name``. If the attribute is not available it\nraises :class:`AttributeError`.'''\n if name in self._meta.dfields:\n return self._meta.dfields[name].get_value(self)\n elif not name.startswith('__') and JSPLITTER in name:\n bits = name.split(JSPLITTER)\n fname = bits[0]\n if fname in self._meta.dfields:\n return self._meta.dfields[fname].get_value(self, *bits[1:])\n else:\n return getattr(self, name)\n else:\n return getattr(self, name)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef clone(self, **data):\n '''Utility method for cloning the instance as a new object.\n\n:parameter data: additional which override field data.\n:rtype: a new instance of this class.\n'''\n meta = self._meta\n session = self.session\n pkname = meta.pkname()\n pkvalue = data.pop(pkname, None)\n fields = self.todict(exclude_cache=True)\n fields.update(data)\n fields.pop('__dbdata__', None)\n obj = self._meta.make_object((pkvalue, None, fields))\n obj.session = session\n return obj", "response": "Utility method for cloning the instance as a new object."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a dictionary of serialised scalar field for pickling.", "response": "def todict(self, exclude_cache=False):\n '''Return a dictionary of serialised scalar field for pickling.\nIf the *exclude_cache* flag is ``True``, fields with :attr:`Field.as_cache`\nattribute set to ``True`` will be excluded.'''\n odict = {}\n for field, value in self.fieldvalue_pairs(exclude_cache=exclude_cache):\n value = field.serialise(value)\n if value:\n odict[field.name] = value\n if self._dbdata and 'id' in self._dbdata:\n odict['__dbdata__'] = {'id': self._dbdata['id']}\n return odict"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nloads extra fields to this : class : StdModel.", "response": "def load_fields(self, *fields):\n '''Load extra fields to this :class:`StdModel`.'''\n if self._loadedfields is not None:\n if self.session is None:\n raise SessionNotAvailable('No session available')\n meta = self._meta\n kwargs = {meta.pkname(): self.pkvalue()}\n obj = session.query(self).load_only(fields).get(**kwargs)\n for name in fields:\n field = meta.dfields.get(name)\n if field is not None:\n setattr(self, field.attname,\n getattr(obj, field.attname, None))"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nloads a the related object of the class.", "response": "def load_related_model(self, name, load_only=None, dont_load=None):\n '''Load a the :class:`ForeignKey` field ``name`` if this is part of the\nfields of this model and if the related object is not already loaded.\nIt is used by the lazy loading mechanism of :ref:`one-to-many `\nrelationships.\n\n:parameter name: the :attr:`Field.name` of the :class:`ForeignKey` to load.\n:parameter load_only: Optional parameters which specify the fields to load.\n:parameter dont_load: Optional parameters which specify the fields not to load.\n:return: the related :class:`StdModel` instance.\n'''\n field = self._meta.dfields.get(name)\n if not field:\n raise ValueError('Field \"%s\" not available' % name)\n elif not field.type == 'related object':\n raise ValueError('Field \"%s\" not a foreign key' % name)\n return self._load_related_model(field, load_only, dont_load)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading a : class:`StdModel` from possibly base64encoded data. This method is used to load a : class:`StdModel` from possibly base64encoded data.", "response": "def from_base64_data(cls, **kwargs):\n '''Load a :class:`StdModel` from possibly base64encoded data.\n\nThis method is used to load models from data obtained from the :meth:`tojson`\nmethod.'''\n o = cls()\n meta = cls._meta\n pkname = meta.pkname()\n for name, value in iteritems(kwargs):\n if name == pkname:\n field = meta.pk\n elif name in meta.dfields:\n field = meta.dfields[name]\n else:\n continue\n value = field.to_python(value)\n setattr(o, field.attname, value)\n return o"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DetrendFITS(fitsfile, raw=False, season=None, clobber=False, **kwargs):\n # Get info\n EPIC = pyfits.getheader(fitsfile, 0)['KEPLERID']\n if season is None:\n season = pyfits.getheader(fitsfile, 0)['CAMPAIGN']\n if season is None or season == \"\":\n season = 0\n everestfile = os.path.join(\n everest.missions.k2.TargetDirectory(EPIC, season),\n everest.missions.k2.FITSFile(EPIC, season))\n\n # De-trend?\n if clobber or not os.path.exists(everestfile):\n\n # Get raw data\n data = GetData(fitsfile, EPIC, season, clobber=clobber, **kwargs)\n\n # De-trend\n model = everest.rPLD(EPIC,\n data=data,\n season=season, debug=True,\n clobber=clobber, **kwargs)\n\n # Publish it\n everest.fits.MakeFITS(model)\n shutil.copyfile(os.path.join(model.dir, model.name + '.pdf'),\n os.path.join(model.dir,\n model._mission.DVSFile(model.ID,\n model.season,\n model.cadence)))\n\n # Return an Everest instance\n return everest.Everest(EPIC, season=season)", "response": "De - trend a K2 FITS file using the K2 detector."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nretrieves the raw data for a specific target.", "response": "def GetData(fitsfile, EPIC, campaign, clobber=False,\n saturation_tolerance=-0.1,\n bad_bits=[1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17],\n get_hires=False, get_nearby=False,\n aperture=None, **kwargs):\n '''\n Returns a :py:obj:`DataContainer` instance with the\n raw data for the target.\n\n :param str fitsfile: The full raw target pixel file path\n :param bool clobber: Overwrite existing files? Default :py:obj:`False`\n :param float saturation_tolerance: Target is considered saturated \\\n if flux is within this fraction of the pixel well depth. \\\n Default -0.1\n :param array_like bad_bits: Flagged :py:obj`QUALITY` bits to consider \\\n outliers when computing the model. \\\n Default `[1,2,3,4,5,6,7,8,9,11,12,13,14,16,17]`\n :param bool get_hires: Download a high resolution image of the target? \\\n Default :py:obj:`True`\n :param bool get_nearby: Retrieve location of nearby sources? \\\n Default :py:obj:`True`\n\n '''\n\n # Get the npz file name\n filename = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % campaign,\n ('%09d' % EPIC)[:4] +\n '00000', ('%09d' % EPIC)[4:],\n 'data.npz')\n\n # Create the dir\n if not os.path.exists(os.path.dirname(filename)):\n os.makedirs(os.path.dirname(filename))\n\n # Check for saved data\n if not os.path.exists(filename) or clobber:\n\n log.info(\"Fetching data for target...\")\n\n # Load the tpf\n with pyfits.open(fitsfile) as f:\n qdata = f[1].data\n\n # Get the header info\n fitsheader = [pyfits.getheader(fitsfile, 0).cards,\n pyfits.getheader(fitsfile, 1).cards,\n pyfits.getheader(fitsfile, 2).cards]\n\n # Get a hi res image of the target\n if get_hires:\n try:\n hires = GetHiResImage(EPIC)\n except ValueError:\n hires = None\n else:\n hires = None\n\n # Get nearby sources\n if get_nearby:\n try:\n nearby = GetSources(EPIC)\n except ValueError:\n nearby = []\n else:\n nearby = []\n\n # Get the arrays\n cadn = np.array(qdata.field('CADENCENO'), dtype='int32')\n time = np.array(qdata.field('TIME'), dtype='float64')\n fpix = np.array(qdata.field('FLUX'), dtype='float64')\n fpix_err = np.array(qdata.field('FLUX_ERR'), dtype='float64')\n qual = np.array(qdata.field('QUALITY'), dtype=int)\n\n # Get rid of NaNs in the time array by interpolating\n naninds = np.where(np.isnan(time))\n time = Interpolate(np.arange(0, len(time)), naninds, time)\n\n # Get the motion vectors (if available!)\n pc1 = np.array(qdata.field('POS_CORR1'), dtype='float64')\n pc2 = np.array(qdata.field('POS_CORR2'), dtype='float64')\n if not np.all(np.isnan(pc1)) and not np.all(np.isnan(pc2)):\n pc1 = Interpolate(time, np.where(np.isnan(pc1)), pc1)\n pc2 = Interpolate(time, np.where(np.isnan(pc2)), pc2)\n else:\n pc1 = None\n pc2 = None\n\n # Get the static pixel images for plotting\n pixel_images = [fpix[0], fpix[len(fpix) // 2], fpix[len(fpix) - 1]]\n\n # Get the aperture interactively\n if aperture is None:\n aperture = ApertureSelector(time[::10], fpix[::10],\n title='EPIC %d' % EPIC).aperture\n if np.sum(aperture) == 0:\n raise ValueError(\"Empty aperture!\")\n\n # Atomically write to disk.\n # http://stackoverflow.com/questions/2333872/\n # atomic-writing-to-file-with-python\n if not os.path.exists(os.path.dirname(filename)):\n os.makedirs(os.path.dirname(filename))\n f = NamedTemporaryFile(\"wb\", delete=False)\n np.savez_compressed(f, cadn=cadn, time=time, fpix=fpix,\n fpix_err=fpix_err,\n qual=qual, aperture=aperture,\n pc1=pc1, pc2=pc2, fitsheader=fitsheader,\n pixel_images=pixel_images, nearby=nearby,\n hires=hires)\n f.flush()\n os.fsync(f.fileno())\n f.close()\n shutil.move(f.name, filename)\n\n # Load\n data = np.load(filename)\n aperture = data['aperture'][()]\n pixel_images = data['pixel_images']\n nearby = data['nearby'][()]\n hires = data['hires'][()]\n fitsheader = data['fitsheader']\n cadn = data['cadn']\n time = data['time']\n fpix = data['fpix']\n fpix_err = data['fpix_err']\n qual = data['qual']\n pc1 = data['pc1']\n pc2 = data['pc2']\n\n # Compute the saturation flux and the 97.5th percentile\n # flux in each pixel of the aperture. We're going\n # to compare these to decide if the star is saturated.\n satflx = SaturationFlux(EPIC, campaign=campaign) * \\\n (1. + saturation_tolerance)\n f97 = np.zeros((fpix.shape[1], fpix.shape[2]))\n for i in range(fpix.shape[1]):\n for j in range(fpix.shape[2]):\n if aperture[i, j]:\n # Let's remove NaNs...\n tmp = np.delete(fpix[:, i, j], np.where(\n np.isnan(fpix[:, i, j])))\n # ... and really bad outliers...\n if len(tmp):\n f = SavGol(tmp)\n med = np.nanmedian(f)\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n bad = np.where((f > med + 10. * MAD) |\n (f < med - 10. * MAD))[0]\n np.delete(tmp, bad)\n # ... so we can compute the 97.5th percentile flux\n i97 = int(0.975 * len(tmp))\n tmp = tmp[np.argsort(tmp)[i97]]\n f97[i, j] = tmp\n\n # Check if any of the pixels are actually saturated\n if np.nanmax(f97) <= satflx:\n log.info(\"No saturated columns detected.\")\n saturated = False\n aperture[np.isnan(fpix[0])] = 0\n ap = np.where(aperture & 1)\n fpix2D = np.array([f[ap] for f in fpix], dtype='float64')\n fpix_err2D = np.array([p[ap] for p in fpix_err], dtype='float64')\n else:\n # We need to collapse the saturated columns\n saturated = True\n ncol = 0\n fpixnew = []\n ferrnew = []\n for j in range(aperture.shape[1]):\n if np.any(f97[:, j] > satflx):\n marked = False\n collapsed = np.zeros(len(fpix[:, 0, 0]))\n collapsed_err2 = np.zeros(len(fpix[:, 0, 0]))\n for i in range(aperture.shape[0]):\n if aperture[i, j]:\n if not marked:\n aperture[i, j] = AP_COLLAPSED_PIXEL\n marked = True\n else:\n aperture[i, j] = AP_SATURATED_PIXEL\n collapsed += fpix[:, i, j]\n collapsed_err2 += fpix_err[:, i, j] ** 2\n if np.any(collapsed):\n fpixnew.append(collapsed)\n ferrnew.append(np.sqrt(collapsed_err2))\n ncol += 1\n else:\n for i in range(aperture.shape[0]):\n if aperture[i, j]:\n fpixnew.append(fpix[:, i, j])\n ferrnew.append(fpix_err[:, i, j])\n fpix2D = np.array(fpixnew).T\n fpix_err2D = np.array(ferrnew).T\n log.info(\"Collapsed %d saturated column(s).\" % ncol)\n\n # Compute the background\n binds = np.where(aperture ^ 1)\n if RemoveBackground(EPIC, campaign=campaign) and (len(binds[0]) > 0):\n bkg = np.nanmedian(np.array([f[binds]\n for f in fpix], dtype='float64'), axis=1)\n # Uncertainty of the median:\n # http://davidmlane.com/hyperstat/A106993.html\n bkg_err = 1.253 * np.nanmedian(np.array([e[binds] for e in fpix_err],\n dtype='float64'), axis=1) \\\n / np.sqrt(len(binds[0]))\n bkg = bkg.reshape(-1, 1)\n bkg_err = bkg_err.reshape(-1, 1)\n else:\n bkg = 0.\n bkg_err = 0.\n\n # Make everything 2D and remove the background\n fpix = fpix2D - bkg\n fpix_err = np.sqrt(fpix_err2D ** 2 + bkg_err ** 2)\n flux = np.sum(fpix, axis=1)\n\n # Get NaN data points\n nanmask = np.where(np.isnan(flux) | (flux == 0))[0]\n\n # Get flagged data points -- we won't train our model on them\n badmask = []\n for b in bad_bits:\n badmask += list(np.where(qual & 2 ** (b - 1))[0])\n\n # Flag >10 sigma outliers -- same thing.\n tmpmask = np.array(list(set(np.concatenate([badmask, nanmask]))))\n t = np.delete(time, tmpmask)\n f = np.delete(flux, tmpmask)\n f = SavGol(f)\n med = np.nanmedian(f)\n MAD = 1.4826 * np.nanmedian(np.abs(f - med))\n bad = np.where((f > med + 10. * MAD) | (f < med - 10. * MAD))[0]\n badmask.extend([np.argmax(time == t[i]) for i in bad])\n\n # Campaign 2 hack: the first day or two are screwed up\n if campaign == 2:\n badmask.extend(np.where(time < 2061.5)[0])\n\n # Finalize the mask\n badmask = np.array(sorted(list(set(badmask))))\n\n # Interpolate the nans\n fpix = Interpolate(time, nanmask, fpix)\n fpix_err = Interpolate(time, nanmask, fpix_err)\n\n # Return\n data = DataContainer()\n data.ID = EPIC\n data.campaign = campaign\n data.cadn = cadn\n data.time = time\n data.fpix = fpix\n data.fpix_err = fpix_err\n data.nanmask = nanmask\n data.badmask = badmask\n data.aperture = aperture\n data.aperture_name = 'custom'\n data.apertures = dict(custom=aperture)\n data.quality = qual\n data.Xpos = pc1\n data.Ypos = pc2\n data.meta = fitsheader\n data.mag = fitsheader[0]['KEPMAG'][1]\n if type(data.mag) is pyfits.card.Undefined:\n data.mag = np.nan\n data.pixel_images = pixel_images\n data.nearby = nearby\n data.hires = hires\n data.saturated = saturated\n data.bkg = bkg\n\n return data"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the axis instance where the title will be printed", "response": "def title(self):\n '''\n Returns the axis instance where the title will be printed\n\n '''\n\n return self.title_left(on=False), self.title_center(on=False), \\\n self.title_right(on=False)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef footer(self):\n '''\n Returns the axis instance where the footer will be printed\n\n '''\n\n return self.footer_left(on=False), self.footer_center(on=False), \\\n self.footer_right(on=False)", "response": "Returns the axis instance where the footer will be printed"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the axis instance at the top right of the page", "response": "def top_right(self):\n '''\n Returns the axis instance at the top right of the page,\n where the postage stamp and aperture is displayed\n\n '''\n\n res = self.body_top_right[self.tcount]()\n self.tcount += 1\n return res"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef left(self):\n '''\n Returns the current axis instance on the left side of\n the page where each successive light curve is displayed\n\n '''\n\n res = self.body_left[self.lcount]()\n self.lcount += 1\n return res", "response": "Returns the current axis instance on the left side of\n the page where each successive light curve is displayed"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the current axis instance on the right side of the the page where cross - validation information is displayed.", "response": "def right(self):\n '''\n Returns the current axis instance on the right side of the\n page, where cross-validation information is displayed\n\n '''\n\n res = self.body_right[self.rcount]()\n self.rcount += 1\n return res"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the axis instance where the light curves will be shown", "response": "def body(self):\n '''\n Returns the axis instance where the light curves will be shown\n\n '''\n\n res = self._body[self.bcount]()\n self.bcount += 1\n return res"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef hashmodel(model, library=None):\r\n '''Calculate the Hash id of metaclass ``meta``'''\r\n library = library or 'python-stdnet'\r\n meta = model._meta\r\n sha = hashlib.sha1(to_bytes('{0}({1})'.format(library, meta)))\r\n hash = sha.hexdigest()[:8]\r\n meta.hash = hash\r\n if hash in _model_dict:\r\n raise KeyError('Model \"{0}\" already in hash table.\\\r\n Rename your model or the module containing the model.'.format(meta))\r\n _model_dict[hash] = model", "response": "Calculate the Hash id of metaclass meta."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nbinds a callback for a given sender.", "response": "def bind(self, callback, sender=None):\r\n '''Bind a ``callback`` for a given ``sender``.'''\r\n key = (_make_id(callback), _make_id(sender))\r\n self.callbacks.append((key, callback))"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nfire callbacks from a sender.", "response": "def fire(self, sender=None, **params):\r\n '''Fire callbacks from a ``sender``.'''\r\n keys = (_make_id(None), _make_id(sender))\r\n results = []\r\n for (_, key), callback in self.callbacks:\r\n if key in keys:\r\n results.append(callback(self, sender, **params))\r\n return results"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef execute_command(self, cmnd, *args, **options):\n \"Execute a command and return a parsed response\"\n args, options = self.preprocess_command(cmnd, *args, **options)\n return self.client.execute_command(cmnd, *args, **options)", "response": "Execute a command and return a parsed response"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the 10th - 90th percentile range of array x.", "response": "def _range10_90(x):\n '''\n Returns the 10th-90th percentile range of array :py:obj:`x`.\n\n '''\n\n x = np.delete(x, np.where(np.isnan(x)))\n i = np.argsort(x)\n a = int(0.1 * len(x))\n b = int(0.9 * len(x))\n return x[i][b] - x[i][a]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Campaign(EPIC, **kwargs):\n '''\n Returns the campaign number(s) for a given EPIC target. If target\n is not found, returns :py:obj:`None`.\n\n :param int EPIC: The EPIC number of the target.\n\n '''\n\n campaigns = []\n for campaign, stars in GetK2Stars().items():\n if EPIC in [s[0] for s in stars]:\n campaigns.append(campaign)\n if len(campaigns) == 0:\n return None\n elif len(campaigns) == 1:\n return campaigns[0]\n else:\n return campaigns", "response": "Returns the campaign number for a given EPIC target."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndownload and return a dict of all K2 stars organized by the current K2.", "response": "def GetK2Stars(clobber=False):\n '''\n Download and return a :py:obj:`dict` of all *K2* stars organized by\n campaign. Saves each campaign to a `.stars` file in the\n `everest/missions/k2/tables` directory.\n\n :param bool clobber: If :py:obj:`True`, download and overwrite \\\n existing files. Default :py:obj:`False`\n\n .. note:: The keys of the dictionary returned by this function are the \\\n (integer) numbers of each campaign. Each item in the \\\n :py:obj:`dict` is a list of the targets in the corresponding \\\n campaign, and each item in that list is in turn a list of the \\\n following: **EPIC number** (:py:class:`int`), \\\n **Kp magnitude** (:py:class:`float`), **CCD channel number** \\\n (:py:class:`int`), and **short cadence available** \\\n (:py:class:`bool`).\n\n '''\n\n # Download\n if clobber:\n print(\"Downloading K2 star list...\")\n stars = kplr_client.k2_star_info()\n print(\"Writing star list to disk...\")\n for campaign in stars.keys():\n if not os.path.exists(os.path.join(EVEREST_SRC, 'missions',\n 'k2', 'tables')):\n os.makedirs(os.path.join(\n EVEREST_SRC, 'missions', 'k2', 'tables'))\n with open(os.path.join(EVEREST_SRC, 'missions', 'k2', 'tables',\n 'c%02d.stars' % campaign), 'w') as f:\n for star in stars[campaign]:\n print(\",\".join([str(s) for s in star]), file=f)\n\n # Return\n res = {}\n for campaign in K2_CAMPAIGNS:\n f = os.path.join(EVEREST_SRC, 'missions', 'k2',\n 'tables', 'c%02d.stars' % campaign)\n if os.path.exists(f):\n with open(f, 'r') as file:\n lines = file.readlines()\n if len(lines[0].split(',')) == 4:\n # EPIC number, Kp magnitude, channel number,\n # short cadence available?\n stars = [[int(l.split(',')[0]),\n _float(l.split(',')[1]),\n int(l.split(',')[2]),\n eval(l.split(',')[3])] for l in lines]\n else:\n stars = [[int(l), np.nan, -1, None] for l in lines]\n res.update({campaign: stars})\n\n return res"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nretrieves all stars in a given K2 * campaign.", "response": "def GetK2Campaign(campaign, clobber=False, split=False,\n epics_only=False, cadence='lc'):\n '''\n Return all stars in a given *K2* campaign.\n\n :param campaign: The *K2* campaign number. If this is an :py:class:`int`, \\\n returns all targets in that campaign. If a :py:class:`float` in \\\n the form :py:obj:`X.Y`, runs the :py:obj:`Y^th` decile of campaign \\\n :py:obj:`X`.\n :param bool clobber: If :py:obj:`True`, download and overwrite existing \\\n files. Default :py:obj:`False`\n :param bool split: If :py:obj:`True` and :py:obj:`campaign` is an \\\n :py:class:`int`, returns each of the subcampaigns as a separate \\\n list. Default :py:obj:`False`\n :param bool epics_only: If :py:obj:`True`, returns only the EPIC numbers. \\\n If :py:obj:`False`, returns metadata associated with each target. \\\n Default :py:obj:`False`\n :param str cadence: Long (:py:obj:`lc`) or short (:py:obj:`sc`) cadence? \\\n Default :py:obj:`lc`.\n\n '''\n\n all = GetK2Stars(clobber=clobber)\n if int(campaign) in all.keys():\n all = all[int(campaign)]\n else:\n return []\n\n if cadence == 'sc':\n all = [a for a in all if a[3]]\n\n if epics_only:\n all = [a[0] for a in all]\n if type(campaign) is int or type(campaign) is np.int64:\n if not split:\n return all\n else:\n all_split = list(Chunks(all, len(all) // 10))\n\n # HACK: Sometimes we're left with a few targets\n # dangling at the end. Insert them back evenly\n # into the first few subcampaigns.\n if len(all_split) > 10:\n tmp1 = all_split[:10]\n tmp2 = all_split[10:]\n for n in range(len(tmp2)):\n tmp1[n] = np.append(tmp1[n], tmp2[n])\n all_split = tmp1\n\n res = []\n for subcampaign in range(10):\n res.append(all_split[subcampaign])\n\n return res\n elif type(campaign) is float:\n x, y = divmod(campaign, 1)\n campaign = int(x)\n subcampaign = round(y * 10)\n return list(Chunks(all, len(all) // 10))[subcampaign]\n else:\n raise Exception('Argument `subcampaign` must be an `int` ' +\n 'or a `float` in the form `X.Y`')"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Channel(EPIC, campaign=None):\n '''\n Returns the channel number for a given EPIC target.\n\n '''\n\n if campaign is None:\n campaign = Campaign(EPIC)\n if hasattr(campaign, '__len__'):\n raise AttributeError(\n \"Please choose a campaign/season for this target: %s.\" % campaign)\n try:\n stars = GetK2Stars()[campaign]\n except KeyError:\n # Not sure what else to do here!\n log.warn(\"Unknown channel for target. Defaulting to channel 2.\")\n return 2\n i = np.argmax([s[0] == EPIC for s in stars])\n return stars[i][2]", "response": "Returns the channel number for a given EPIC target."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Module(EPIC, campaign=None):\n '''\n Returns the module number for a given EPIC target.\n\n '''\n\n channel = Channel(EPIC, campaign=campaign)\n nums = {2: 1, 3: 5, 4: 9, 6: 13, 7: 17, 8: 21, 9: 25,\n 10: 29, 11: 33, 12: 37, 13: 41, 14: 45, 15: 49,\n 16: 53, 17: 57, 18: 61, 19: 65, 20: 69, 22: 73,\n 23: 77, 24: 81}\n for c in [channel, channel - 1, channel - 2, channel - 3]:\n if c in nums.values():\n for mod, chan in nums.items():\n if chan == c:\n return mod\n return None", "response": "Returns the module number for a given EPIC target."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Channels(module):\n '''\n Returns the channels contained in the given K2 module.\n\n '''\n\n nums = {2: 1, 3: 5, 4: 9, 6: 13, 7: 17, 8: 21, 9: 25,\n 10: 29, 11: 33, 12: 37, 13: 41, 14: 45, 15: 49,\n 16: 53, 17: 57, 18: 61, 19: 65, 20: 69, 22: 73,\n 23: 77, 24: 81}\n\n if module in nums:\n return [nums[module], nums[module] + 1,\n nums[module] + 2, nums[module] + 3]\n else:\n return None", "response": "Returns the channels contained in the given K2 module."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef KepMag(EPIC, campaign=None):\n '''\n Returns the *Kepler* magnitude for a given EPIC target.\n\n '''\n\n if campaign is None:\n campaign = Campaign(EPIC)\n if hasattr(campaign, '__len__'):\n raise AttributeError(\n \"Please choose a campaign/season for this target: %s.\" % campaign)\n stars = GetK2Stars()[campaign]\n i = np.argmax([s[0] == EPIC for s in stars])\n return stars[i][1]", "response": "Returns the Kepler magnitude for a given EPIC target."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef RemoveBackground(EPIC, campaign=None):\n '''\n Returns :py:obj:`True` or :py:obj:`False`, indicating whether or not\n to remove the background flux for the target. If ``campaign < 3``,\n returns :py:obj:`True`, otherwise returns :py:obj:`False`.\n\n '''\n\n if campaign is None:\n campaign = Campaign(EPIC)\n if hasattr(campaign, '__len__'):\n raise AttributeError(\n \"Please choose a campaign/season for this target: %s.\" % campaign)\n if campaign < 3:\n return True\n else:\n return False", "response": "Removes the background flux for the specified EPIC."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning all channels on the same module as channel.", "response": "def GetNeighboringChannels(channel):\n '''\n Returns all channels on the same module as :py:obj:`channel`.\n\n '''\n\n x = divmod(channel - 1, 4)[1]\n return channel + np.array(range(-x, -x + 4), dtype=int)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting sexadecimal hours to decimal degrees.", "response": "def sex2dec(ra, dec):\n '''\n Convert sexadecimal hours to decimal degrees. Adapted from\n `PyKE `_.\n\n :param float ra: The right ascension\n :param float dec: The declination\n\n :returns: The same values, but in decimal degrees\n\n '''\n\n ra = re.sub('\\s+', '|', ra.strip())\n ra = re.sub(':', '|', ra.strip())\n ra = re.sub(';', '|', ra.strip())\n ra = re.sub(',', '|', ra.strip())\n ra = re.sub('-', '|', ra.strip())\n ra = ra.split('|')\n outra = (float(ra[0]) + float(ra[1]) / 60. + float(ra[2]) / 3600.) * 15.0\n\n dec = re.sub('\\s+', '|', dec.strip())\n dec = re.sub(':', '|', dec.strip())\n dec = re.sub(';', '|', dec.strip())\n dec = re.sub(',', '|', dec.strip())\n dec = dec.split('|')\n\n if float(dec[0]) > 0.0:\n outdec = float(dec[0]) + float(dec[1]) / 60. + float(dec[2]) / 3600.\n else:\n outdec = float(dec[0]) - float(dec[1]) / 60. - float(dec[2]) / 3600.\n\n return outra, outdec"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nretrieve the EPIC coordinates from the TPF and searches MAST for other EPIC targets within the same aperture.", "response": "def GetSources(ID, darcsec=None, stars_only=False):\n '''\n Grabs the EPIC coordinates from the TPF and searches MAST\n for other EPIC targets within the same aperture.\n\n :param int ID: The 9-digit :py:obj:`EPIC` number of the target\n :param float darcsec: The search radius in arcseconds. \\\n Default is four times the largest dimension of the aperture.\n :param bool stars_only: If :py:obj:`True`, only returns objects \\\n explicitly designated as `\"stars\"` in MAST. Default :py:obj:`False`\n :returns: A list of :py:class:`Source` instances containing \\\n other :py:obj:`EPIC` targets within or close to this \\\n target's aperture\n '''\n\n client = kplr.API()\n star = client.k2_star(ID)\n tpf = star.get_target_pixel_files()[0]\n with tpf.open() as f:\n crpix1 = f[2].header['CRPIX1']\n crpix2 = f[2].header['CRPIX2']\n crval1 = f[2].header['CRVAL1']\n crval2 = f[2].header['CRVAL2']\n cdelt1 = f[2].header['CDELT1']\n cdelt2 = f[2].header['CDELT2']\n pc1_1 = f[2].header['PC1_1']\n pc1_2 = f[2].header['PC1_2']\n pc2_1 = f[2].header['PC2_1']\n pc2_2 = f[2].header['PC2_2']\n pc = np.array([[pc1_1, pc1_2], [pc2_1, pc2_2]])\n pc = np.linalg.inv(pc)\n crpix1p = f[2].header['CRPIX1P']\n crpix2p = f[2].header['CRPIX2P']\n crval1p = f[2].header['CRVAL1P']\n crval2p = f[2].header['CRVAL2P']\n cdelt1p = f[2].header['CDELT1P']\n cdelt2p = f[2].header['CDELT2P']\n if darcsec is None:\n darcsec = 4 * max(f[2].data.shape)\n\n epicid, ra, dec, kepmag = MASTRADec(\n star.k2_ra, star.k2_dec, darcsec, stars_only)\n sources = []\n for i, epic in enumerate(epicid):\n dra = (ra[i] - crval1) * np.cos(np.radians(dec[i])) / cdelt1\n ddec = (dec[i] - crval2) / cdelt2\n sx = pc[0, 0] * dra + pc[0, 1] * ddec + crpix1 + crval1p - 1.0\n sy = pc[1, 0] * dra + pc[1, 1] * ddec + crpix2 + crval2p - 1.0\n sources.append(dict(ID=epic, x=sx, y=sy, mag=kepmag[i],\n x0=crval1p, y0=crval2p))\n\n return sources"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nqueries the Palomar Observatory Sky Survey II catalog to obtain a higher resolution optical image of the star with EPIC number ID.", "response": "def GetHiResImage(ID):\n '''\n Queries the Palomar Observatory Sky Survey II catalog to\n obtain a higher resolution optical image of the star with EPIC number\n :py:obj:`ID`.\n\n '''\n\n # Get the TPF info\n client = kplr.API()\n star = client.k2_star(ID)\n k2ra = star.k2_ra\n k2dec = star.k2_dec\n tpf = star.get_target_pixel_files()[0]\n with tpf.open() as f:\n k2wcs = WCS(f[2].header)\n shape = np.array(f[1].data.field('FLUX'), dtype='float64')[0].shape\n\n # Get the POSS URL\n hou = int(k2ra * 24 / 360.)\n min = int(60 * (k2ra * 24 / 360. - hou))\n sec = 60 * (60 * (k2ra * 24 / 360. - hou) - min)\n ra = '%02d+%02d+%.2f' % (hou, min, sec)\n sgn = '' if np.sign(k2dec) >= 0 else '-'\n deg = int(np.abs(k2dec))\n min = int(60 * (np.abs(k2dec) - deg))\n sec = 3600 * (np.abs(k2dec) - deg - min / 60)\n dec = '%s%02d+%02d+%.1f' % (sgn, deg, min, sec)\n url = 'https://archive.stsci.edu/cgi-bin/dss_search?v=poss2ukstu_red&' + \\\n 'r=%s&d=%s&e=J2000&h=3&w=3&f=fits&c=none&fov=NONE&v3=' % (ra, dec)\n\n # Query the server\n r = urllib.request.Request(url)\n handler = urllib.request.urlopen(r)\n code = handler.getcode()\n if int(code) != 200:\n # Unavailable\n return None\n data = handler.read()\n\n # Atomically write to a temp file\n f = NamedTemporaryFile(\"wb\", delete=False)\n f.write(data)\n f.flush()\n os.fsync(f.fileno())\n f.close()\n\n # Now open the POSS fits file\n with pyfits.open(f.name) as ff:\n img = ff[0].data\n\n # Map POSS pixels onto K2 pixels\n xy = np.empty((img.shape[0] * img.shape[1], 2))\n z = np.empty(img.shape[0] * img.shape[1])\n pwcs = WCS(f.name)\n k = 0\n for i in range(img.shape[0]):\n for j in range(img.shape[1]):\n ra, dec = pwcs.all_pix2world(float(j), float(i), 0)\n xy[k] = k2wcs.all_world2pix(ra, dec, 0)\n z[k] = img[i, j]\n k += 1\n\n # Resample\n grid_x, grid_y = np.mgrid[-0.5:shape[1] - 0.5:0.1, -0.5:shape[0] - 0.5:0.1]\n resampled = griddata(xy, z, (grid_x, grid_y), method='cubic')\n\n # Rotate to align with K2 image. Not sure why, but it is necessary\n resampled = np.rot90(resampled)\n\n return resampled"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the Saturation Flux for the target.", "response": "def SaturationFlux(EPIC, campaign=None, **kwargs):\n '''\n Returns the well depth for the target. If any of the target's pixels\n have flux larger than this value, they are likely to be saturated and\n cause charge bleeding. The well depths were obtained from Table 13\n of the Kepler instrument handbook. We assume an exposure time of 6.02s.\n\n '''\n\n channel, well_depth = np.loadtxt(os.path.join(EVEREST_SRC, 'missions',\n 'k2',\n 'tables', 'well_depth.tsv'),\n unpack=True)\n satflx = well_depth[channel == Channel(EPIC, campaign=campaign)][0] / 6.02\n return satflx"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the indices corresponding to a given light curve chunk.", "response": "def GetChunk(time, breakpoints, b, mask=[]):\n '''\n Returns the indices corresponding to a given light curve chunk.\n\n :param int b: The index of the chunk to return\n\n '''\n\n M = np.delete(np.arange(len(time)), mask, axis=0)\n if b > 0:\n res = M[(M > breakpoints[b - 1]) & (M <= breakpoints[b])]\n else:\n res = M[M <= breakpoints[b]]\n return res"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning de - trended light curves for all stars on a given module in a given campaign.", "response": "def GetStars(campaign, module, model='nPLD', **kwargs):\n '''\n Returns de-trended light curves for all stars on a given module in\n a given campaign.\n\n '''\n\n # Get the channel numbers\n channels = Channels(module)\n assert channels is not None, \"No channels available on this module.\"\n\n # Get the EPIC numbers\n all = GetK2Campaign(campaign)\n stars = np.array([s[0] for s in all if s[2] in channels and\n os.path.exists(\n os.path.join(EVEREST_DAT, 'k2', 'c%02d' % int(campaign),\n ('%09d' % s[0])[:4] + '00000',\n ('%09d' % s[0])[4:], model + '.npz'))], dtype=int)\n N = len(stars)\n assert N > 0, \"No light curves found for campaign %d, module %d.\" % (\n campaign, module)\n\n # Loop over all stars and store the fluxes in a list\n fluxes = []\n errors = []\n kpars = []\n\n for n in range(N):\n\n # De-trended light curve file name\n nf = os.path.join(EVEREST_DAT, 'k2', 'c%02d' % int(campaign),\n ('%09d' % stars[n])[:4] + '00000',\n ('%09d' % stars[n])[4:], model + '.npz')\n\n # Get the data\n data = np.load(nf)\n t = data['time']\n if n == 0:\n time = t\n breakpoints = data['breakpoints']\n\n # Get de-trended light curve\n y = data['fraw'] - data['model']\n err = data['fraw_err']\n\n # De-weight outliers and bad timestamps\n m = np.array(list(set(np.concatenate([data['outmask'], data['badmask'],\n data['nanmask'],\n data['transitmask']]))),\n dtype=int)\n\n # Interpolate over the outliers\n y = np.interp(t, np.delete(t, m), np.delete(y, m))\n err = np.interp(t, np.delete(t, m), np.delete(err, m))\n\n # Append to our running lists\n fluxes.append(y)\n errors.append(err)\n kpars.append(data['kernel_params'])\n\n return time, breakpoints, np.array(fluxes), \\\n np.array(errors), np.array(kpars)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\napplying :py:obj:`SysRem` to a given set of light curves. :param array_like time: The time array for all of the light curves :param array_like flux: A 2D array of the fluxes for each of the light \\ curves, shape `(nfluxes, ntime)` :param array_like err: A 2D array of the flux errors for each of the \\ light curves, shape `(nfluxes, ntime)` :param int ncbv: The number of signals to recover. Default 5 :param int niter: The number of :py:obj:`SysRem` iterations to perform. \\ Default 50 :param int sv_win: The Savitsky-Golay filter window size. Default 999 :param int sv_order: The Savitsky-Golay filter order. Default 3", "response": "def SysRem(time, flux, err, ncbv=5, niter=50, sv_win=999,\n sv_order=3, **kwargs):\n '''\n Applies :py:obj:`SysRem` to a given set of light curves.\n\n :param array_like time: The time array for all of the light curves\n :param array_like flux: A 2D array of the fluxes for each of the light \\\n curves, shape `(nfluxes, ntime)`\n :param array_like err: A 2D array of the flux errors for each of the \\\n light curves, shape `(nfluxes, ntime)`\n :param int ncbv: The number of signals to recover. Default 5\n :param int niter: The number of :py:obj:`SysRem` iterations to perform. \\\n Default 50\n :param int sv_win: The Savitsky-Golay filter window size. Default 999\n :param int sv_order: The Savitsky-Golay filter order. Default 3\n\n '''\n\n nflx, tlen = flux.shape\n\n # Get normalized fluxes\n med = np.nanmedian(flux, axis=1).reshape(-1, 1)\n y = flux - med\n\n # Compute the inverse of the variances\n invvar = 1. / err ** 2\n\n # The CBVs for this set of fluxes\n cbvs = np.zeros((ncbv, tlen))\n\n # Recover `ncbv` components\n for n in range(ncbv):\n\n # Initialize the weights and regressors\n c = np.zeros(nflx)\n a = np.ones(tlen)\n f = y * invvar\n\n # Perform `niter` iterations\n for i in range(niter):\n\n # Compute the `c` vector (the weights)\n c = np.dot(f, a) / np.dot(invvar, a ** 2)\n\n # Compute the `a` vector (the regressors)\n a = np.dot(c, f) / np.dot(c ** 2, invvar)\n\n # Remove this component from all light curves\n y -= np.outer(c, a)\n\n # Save this regressor after smoothing it a bit\n if sv_win >= len(a):\n sv_win = len(a) - 1\n if sv_win % 2 == 0:\n sv_win -= 1\n cbvs[n] = savgol_filter(a - np.nanmedian(a), sv_win, sv_order)\n\n return cbvs"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the CBVs for a given campaign.", "response": "def GetCBVs(campaign, model='nPLD', clobber=False, **kwargs):\n '''\n Computes the CBVs for a given campaign.\n\n :param int campaign: The campaign number\n :param str model: The name of the :py:obj:`everest` model. Default `nPLD`\n :param bool clobber: Overwrite existing files? Default `False`\n\n '''\n\n # Initialize logging?\n if len(logging.getLogger().handlers) == 0:\n InitLog(file_name=None, screen_level=logging.DEBUG)\n log.info('Computing CBVs for campaign %d...' % (campaign))\n\n # Output path\n path = os.path.join(EVEREST_DAT, 'k2', 'cbv', 'c%02d' % campaign)\n if not os.path.exists(path):\n os.makedirs(path)\n\n # Get the design matrix\n xfile = os.path.join(path, 'X.npz')\n if clobber or not os.path.exists(xfile):\n\n log.info('Obtaining light curves...')\n time = None\n for module in range(2, 25):\n\n # Get the light curves\n lcfile = os.path.join(path, '%d.npz' % module)\n if clobber or not os.path.exists(lcfile):\n try:\n time, breakpoints, fluxes, errors, kpars = GetStars(\n campaign, module, model=model, **kwargs)\n except AssertionError:\n continue\n np.savez(lcfile, time=time, breakpoints=breakpoints,\n fluxes=fluxes, errors=errors, kpars=kpars)\n\n # Load the light curves\n lcs = np.load(lcfile)\n if time is None:\n time = lcs['time']\n breakpoints = lcs['breakpoints']\n fluxes = lcs['fluxes']\n errors = lcs['errors']\n kpars = lcs['kpars']\n else:\n fluxes = np.vstack([fluxes, lcs['fluxes']])\n errors = np.vstack([errors, lcs['errors']])\n kpars = np.vstack([kpars, lcs['kpars']])\n\n # Compute the design matrix\n log.info('Running SysRem...')\n X = np.ones((len(time), 1 + kwargs.get('ncbv', 5)))\n\n # Loop over the segments\n new_fluxes = np.zeros_like(fluxes)\n for b in range(len(breakpoints)):\n\n # Get the current segment's indices\n inds = GetChunk(time, breakpoints, b)\n\n # Update the error arrays with the white GP component\n for j in range(len(errors)):\n errors[j] = np.sqrt(errors[j] ** 2 + kpars[j][0] ** 2)\n\n # Get de-trended fluxes\n X[inds, 1:] = SysRem(time[inds], fluxes[:, inds],\n errors[:, inds], **kwargs).T\n\n # Save\n np.savez(xfile, X=X, time=time, breakpoints=breakpoints)\n\n else:\n\n # Load from disk\n data = np.load(xfile)\n X = data['X'][()]\n time = data['time'][()]\n breakpoints = data['breakpoints'][()]\n\n # Plot\n plotfile = os.path.join(path, 'X.pdf')\n if clobber or not os.path.exists(plotfile):\n fig, ax = pl.subplots(2, 3, figsize=(12, 8))\n fig.subplots_adjust(left=0.05, right=0.95)\n ax = ax.flatten()\n for axis in ax:\n axis.set_xticks([])\n axis.set_yticks([])\n for b in range(len(breakpoints)):\n inds = GetChunk(time, breakpoints, b)\n for n in range(min(6, X.shape[1])):\n ax[n].plot(time[inds], X[inds, n])\n ax[n].set_title(n, fontsize=14)\n fig.savefig(plotfile, bbox_inches='tight')\n\n return X"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads lua script from the stdnet / lib directory", "response": "def read_lua_file(dotted_module, path=None, context=None):\r\n '''Load lua script from the stdnet/lib/lua directory'''\r\n path = path or DEFAULT_LUA_PATH\r\n bits = dotted_module.split('.')\r\n bits[-1] += '.lua'\r\n name = os.path.join(path, *bits)\r\n with open(name) as f:\r\n data = f.read()\r\n if context:\r\n data = data.format(context)\r\n return data"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nparses the response of Redis s INFO command into a Python dict.", "response": "def parse_info(response):\r\n '''Parse the response of Redis's INFO command into a Python dict.\r\nIn doing so, convert byte data into unicode.'''\r\n info = {}\r\n response = response.decode('utf-8')\r\n\r\n def get_value(value):\r\n if ',' and '=' not in value:\r\n return value\r\n sub_dict = {}\r\n for item in value.split(','):\r\n k, v = item.split('=')\r\n try:\r\n sub_dict[k] = int(v)\r\n except ValueError:\r\n sub_dict[k] = v\r\n return sub_dict\r\n data = info\r\n for line in response.splitlines():\r\n keyvalue = line.split(':')\r\n if len(keyvalue) == 2:\r\n key, value = keyvalue\r\n try:\r\n data[key] = int(value)\r\n except ValueError:\r\n data[key] = get_value(value)\r\n else:\r\n data = {}\r\n info[line[2:]] = data\r\n return info"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing the difference of multiple sorted. The difference of sets specified by keys into a new sorted set in dest.", "response": "def zdiffstore(self, dest, keys, withscores=False):\r\n '''Compute the difference of multiple sorted.\r\n\r\n The difference of sets specified by ``keys`` into a new sorted set\r\n in ``dest``.\r\n '''\r\n keys = (dest,) + tuple(keys)\r\n wscores = 'withscores' if withscores else ''\r\n return self.execute_script('zdiffstore', keys, wscores,\r\n withscores=withscores)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npop a range by rank.", "response": "def zpopbyrank(self, name, start, stop=None, withscores=False, desc=False):\r\n '''Pop a range by rank.\r\n '''\r\n stop = stop if stop is not None else start\r\n return self.execute_script('zpop', (name,), 'rank', start,\r\n stop, int(desc), int(withscores),\r\n withscores=withscores)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the log prior given parameter vector x.", "response": "def lnprior(x):\n \"\"\"Return the log prior given parameter vector `x`.\"\"\"\n per, t0, b = x\n if b < -1 or b > 1:\n return -np.inf\n elif per < 7 or per > 10:\n return -np.inf\n elif t0 < 1978 or t0 > 1979:\n return -np.inf\n else:\n return 0."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the log likelihood given parameter vector x.", "response": "def lnlike(x, star):\n \"\"\"Return the log likelihood given parameter vector `x`.\"\"\"\n ll = lnprior(x)\n if np.isinf(ll):\n return ll, (np.nan, np.nan)\n per, t0, b = x\n model = TransitModel('b', per=per, t0=t0, b=b, rhos=10.)(star.time)\n like, d, vard = star.lnlike(model, full_output=True)\n ll += like\n return ll, (d,)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef check_user(self, username, email):\n '''username and email (if provided) must be unique.'''\n users = self.router.user\n avail = yield users.filter(username=username).count()\n if avail:\n raise FieldError('Username %s not available' % username)\n if email:\n avail = yield users.filter(email=email).count()\n if avail:\n raise FieldError('Email %s not available' % email)", "response": "Check that username and email is unique."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchanges the query so that only instances for which ``group has roles with permission on operations are returned.", "response": "def permitted_query(self, query, group, operations):\n '''Change the ``query`` so that only instances for which\n``group`` has roles with permission on ``operations`` are returned.'''\n session = query.session\n models = session.router\n user = group.user\n if user.is_superuser: # super-users have all permissions\n return query\n roles = group.roles.query()\n roles = group.roles.query() # query on all roles for group\n # The throgh model for Role/Permission relationship\n throgh_model = models.role.permissions.model\n models[throgh_model].filter(role=roles,\n permission__model_type=query.model,\n permission__operations=operations)\n\n # query on all relevant permissions\n permissions = router.permission.filter(model_type=query.model,\n level=operations)\n\n owner_query = query.filter(user=user)\n # all roles for the query model with appropriate permission level\n roles = models.role.filter(model_type=query.model, level__ge=level)\n # Now we need groups which have these roles\n groups = Role.groups.throughquery(\n session).filter(role=roles).get_field('group')\n # I need to know if user is in any of these groups\n if user.groups.filter(id=groups).count():\n # it is, lets get the model with permissions less\n # or equal permission level\n permitted = models.instancerole.filter(\n role=roles).get_field('object_id')\n return owner_query.union(model.objects.filter(id=permitted))\n else:\n return owner_query"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a new role owned by this subject", "response": "def create_role(self, name):\n '''Create a new :class:`Role` owned by this :class:`Subject`'''\n models = self.session.router\n return models.role.new(name=name, owner=self)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef assign(self, role):\n '''Assign :class:`Role` ``role`` to this :class:`Subject`. If this\n:class:`Subject` is the :attr:`Role.owner`, this method does nothing.'''\n if role.owner_id != self.id:\n return self.roles.add(role)", "response": "Assign a role to this object."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef has_permissions(self, object, group, operations):\n '''Check if this :class:`Subject` has permissions for ``operations``\non an ``object``. It returns the number of valid permissions.'''\n if self.is_superuser:\n return 1\n else:\n models = self.session.router\n # valid permissions\n query = models.permission.for_object(object, operation=operations)\n objects = models[models.role.permissions.model]\n return objects.filter(role=self.role.query(),\n permission=query).count()", "response": "Check if this subject has permissions for operations\non an object. It returns the number of valid permissions."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a new : class:`Permission for resource to perform an operation. The resource can be either an object or a model. The operation can be either a string or a function. The resource can be either an object or a model.", "response": "def add_permission(self, resource, operation):\n '''Add a new :class:`Permission` for ``resource`` to perform an\n``operation``. The resource can be either an object or a model.'''\n if isclass(resource):\n model_type = resource\n pk = ''\n else:\n model_type = resource.__class__\n pk = resource.pkvalue()\n p = Permission(model_type=model_type, object_pk=pk,\n operation=operation)\n session = self.session\n if session.transaction:\n session.add(p)\n self.permissions.add(p)\n return p\n else:\n with session.begin() as t:\n t.add(p)\n self.permissions.add(p)\n return t.add_callback(lambda r: p)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef init_app(self, app, session=None, parameters=None):\n\n if parameters is not None and not isinstance(parameters, ParamsBuilder):\n raise InvalidUsage(\"parameters should be a pysnow.ParamsBuilder object, not %r\" % type(parameters).__name__)\n\n self._session = session\n self._parameters = parameters\n\n app.config.setdefault('SNOW_INSTANCE', None)\n app.config.setdefault('SNOW_HOST', None)\n app.config.setdefault('SNOW_USER', None)\n app.config.setdefault('SNOW_PASSWORD', None)\n app.config.setdefault('SNOW_OAUTH_CLIENT_ID', None)\n app.config.setdefault('SNOW_OAUTH_CLIENT_SECRET', None)\n app.config.setdefault('SNOW_USE_SSL', True)\n\n if app.config['SNOW_OAUTH_CLIENT_ID'] and app.config['SNOW_OAUTH_CLIENT_SECRET']:\n self._client_type_oauth = True\n elif self._session or (app.config['SNOW_USER'] and app.config['SNOW_PASSWORD']):\n self._client_type_basic = True\n else:\n raise ConfigError(\"You must supply user credentials, a session or OAuth credentials to use flask-snow\")", "response": "Initializes the snow extension with the given app."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a new instance of the currently active Snowball client and its associated resource.", "response": "def connection(self):\n \"\"\"Snow connection instance, stores a `pysnow.Client` instance and `pysnow.Resource` instances\n\n Creates a new :class:`pysnow.Client` object if it doesn't exist in the app slice of the context stack\n\n :returns: :class:`pysnow.Client` object\n \"\"\"\n\n ctx = stack.top.app\n if ctx is not None:\n if not hasattr(ctx, 'snow'):\n if self._client_type_oauth:\n if not self._token_updater:\n warnings.warn(\"No token updater has been set. Token refreshes will be ignored.\")\n\n client = self._get_oauth_client()\n else:\n client = self._get_basic_client()\n\n if self._parameters:\n # Set parameters passed on app init\n client.parameters = self._parameters\n\n ctx.snow = client\n\n return ctx.snow"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef usage():\n\n global options\n l = len(options['long'])\n options['shortlist'] = [s for s in options['short'] if s is not \":\"]\n\n print(\"python -m behave2cucumber [-h] [-d level|--debug=level]\")\n for i in range(l):\n print(\" -{0}|--{1:20} {2}\".format(options['shortlist'][i], options['long'][i], options['descriptions'][i]))", "response": "Print out a usage message"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef main(argv):\n global options\n\n opts = None\n try:\n opts, args = getopt.getopt(argv, options['short'], options['long'])\n except getopt.GetoptError:\n usage()\n exit(2)\n\n for opt, arg in opts:\n if opt in (\"-h\", \"--help\"):\n usage()\n exit()\n elif opt in (\"-d\", \"--debug\"):\n try:\n arg = int(arg)\n log.debug(\"Debug level received: \" + str(arg))\n except ValueError:\n log.warning(\"Invalid log level: \" + arg)\n continue\n\n if 0 <= arg <= 5:\n log.setLevel(60 - (arg*10))\n log.critical(\"Log level changed to: \" + str(logging.getLevelName(60 - (arg*10))))\n else:\n log.warning(\"Invalid log level: \" + str(arg))\n\n infile = None\n outfile = None\n remove_background = False\n duration_format = False\n deduplicate = False\n\n for opt, arg in opts:\n if opt in (\"-i\", \"--infile\"):\n log.info(\"Input File: \" + arg)\n infile = arg\n if opt in (\"-o\", \"--outfile\"):\n log.info(\"Output File: \" + arg)\n outfile = arg\n if opt in (\"-r\", \"--remove-background\"):\n log.info(\"Remove Background: Enabled\")\n remove_background = True\n if opt in (\"-f\", \"--format-duration\"):\n log.info(\"Format Duration: Enabled\")\n duration_format = True\n if opt in (\"-D\", \"--deduplicate\"):\n log.info(\"Deduplicate: Enabled\")\n deduplicate = True\n\n if infile is None:\n log.critical(\"No input JSON provided.\")\n usage()\n exit(3)\n\n with open(infile) as f:\n cucumber_output = convert(json.load(f),\n remove_background=remove_background,\n duration_format=duration_format,\n deduplicate=deduplicate)\n\n if outfile is not None:\n with open(outfile, 'w') as f:\n json.dump(cucumber_output, f, indent=4, separators=(',', ': '))\n else:\n pprint(cucumber_output)", "response": "Main entry point for the cucumber command line interface."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef direction(theta, phi):\n '''Return the direction vector of a cylinder defined\n by the spherical coordinates theta and phi.\n '''\n return np.array([np.cos(phi) * np.sin(theta), np.sin(phi) * np.sin(theta),\n np.cos(theta)])", "response": "Return the direction vector of a cylinder defined by the spherical coordinates theta and phi."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef projection_matrix(w):\n '''Return the projection matrix of a direction w.'''\n return np.identity(3) - np.dot(np.reshape(w, (3,1)), np.reshape(w, (1, 3)))", "response": "Return the projection matrix of a direction w."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef skew_matrix(w):\n '''Return the skew matrix of a direction w.'''\n return np.array([[0, -w[2], w[1]],\n [w[2], 0, -w[0]],\n [-w[1], w[0], 0]])", "response": "Return the skew matrix of a direction w."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the matrix A from a list of Y vectors.", "response": "def calc_A(Ys):\n '''Return the matrix A from a list of Y vectors.'''\n return sum(np.dot(np.reshape(Y, (3,1)), np.reshape(Y, (1, 3)))\n for Y in Ys)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef calc_A_hat(A, S):\n '''Return the A_hat matrix of A given the skew matrix S'''\n return np.dot(S, np.dot(A, np.transpose(S)))", "response": "Return the A_hat matrix of A given the skew matrix S"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef preprocess_data(Xs_raw):\n '''Translate the center of mass (COM) of the data to the origin.\n Return the prossed data and the shift of the COM'''\n n = len(Xs_raw)\n Xs_raw_mean = sum(X for X in Xs_raw) / n\n\n return [X - Xs_raw_mean for X in Xs_raw], Xs_raw_mean", "response": "Translate the center of mass of the data to the origin."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef G(w, Xs):\n '''Calculate the G function given a cylinder direction w and a\n list of data points Xs to be fitted.'''\n n = len(Xs)\n P = projection_matrix(w)\n Ys = [np.dot(P, X) for X in Xs]\n A = calc_A(Ys)\n A_hat = calc_A_hat(A, skew_matrix(w))\n\n \n u = sum(np.dot(Y, Y) for Y in Ys) / n\n v = np.dot(A_hat, sum(np.dot(Y, Y) * Y for Y in Ys)) / np.trace(np.dot(A_hat, A))\n\n return sum((np.dot(Y, Y) - u - 2 * np.dot(Y, v)) ** 2 for Y in Ys)", "response": "Calculate the G function given a cylinder direction w and a\n list of data points Xs to be fitted."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the cylinder center given the cylinder direction and a list of data points.", "response": "def C(w, Xs):\n '''Calculate the cylinder center given the cylinder direction and \n a list of data points.\n '''\n n = len(Xs)\n P = projection_matrix(w)\n Ys = [np.dot(P, X) for X in Xs]\n A = calc_A(Ys)\n A_hat = calc_A_hat(A, skew_matrix(w))\n\n return np.dot(A_hat, sum(np.dot(Y, Y) * Y for Y in Ys)) / np.trace(np.dot(A_hat, A))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef r(w, Xs):\n '''Calculate the radius given the cylinder direction and a list\n of data points.\n '''\n n = len(Xs)\n P = projection_matrix(w)\n c = C(w, Xs)\n\n return np.sqrt(sum(np.dot(c - X, np.dot(P, c - X)) for X in Xs) / n)", "response": "Calculate the radius given the cylinder direction and a list of data points."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fit(data, guess_angles=None):\n '''Fit a list of data points to a cylinder surface. The algorithm implemented\n here is from David Eberly's paper \"Fitting 3D Data with a Cylinder\" from \n https://www.geometrictools.com/Documentation/CylinderFitting.pdf\n\n Arguments:\n data - A list of 3D data points to be fitted.\n guess_angles[0] - Guess of the theta angle of the axis direction\n guess_angles[1] - Guess of the phi angle of the axis direction\n \n Return:\n Direction of the cylinder axis\n A point on the cylinder axis\n Radius of the cylinder\n Fitting error (G function)\n '''\n Xs, t = preprocess_data(data) \n\n # Set the start points\n\n start_points = [(0, 0), (np.pi / 2, 0), (np.pi / 2, np.pi / 2)]\n if guess_angles:\n start_points = guess_angles\n\n # Fit the cylinder from different start points \n\n best_fit = None\n best_score = float('inf')\n\n for sp in start_points:\n fitted = minimize(lambda x : G(direction(x[0], x[1]), Xs),\n sp, method='Powell', tol=1e-6)\n\n if fitted.fun < best_score:\n best_score = fitted.fun\n best_fit = fitted\n\n w = direction(best_fit.x[0], best_fit.x[1])\n\n return w, C(w, Xs) + t, r(w, Xs), best_fit.fun", "response": "Fit a list of data points to a cylinder surface."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npublishes a message to a topic.", "response": "def send(self, topic, value=None, timeout=60, key=None, partition=None, timestamp_ms=None):\n \"\"\"Publish a message to a topic.\n\n - ``topic`` (str): topic where the message will be published\n - ``value``: message value. Must be type bytes, or be serializable to bytes via configured value_serializer.\n If value is None, key is required and message acts as a `delete`.\n - ``timeout``\n - ``key``: a key to associate with the message. Can be used to determine which partition\n to send the message to. If partition is None (and producer's partitioner config is left as default),\n then messages with the same key will be delivered to the same partition (but if key is None,\n partition is chosen randomly). Must be type bytes, or be serializable to bytes via configured key_serializer.\n - ``partition`` (int): optionally specify a partition.\n If not set, the partition will be selected using the configured `partitioner`.\n - ``timestamp_ms`` (int): epoch milliseconds (from Jan 1 1970 UTC) to use as the message timestamp.\n Defaults to current time.\n \"\"\"\n future = self.producer.send(topic, value=value, key=key, partition=partition, timestamp_ms=timestamp_ms)\n future.get(timeout=timeout)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef connect_consumer(\n self,\n bootstrap_servers='127.0.0.1:9092',\n client_id='Robot',\n group_id=None,\n auto_offset_reset='latest',\n enable_auto_commit=True,\n **kwargs\n ):\n \"\"\"Connect kafka consumer.\n \n Keyword Arguments:\n - ``bootstrap_servers``: 'host[:port]' string (or list of 'host[:port]'\n strings) that the consumer should contact to bootstrap initial\n cluster metadata. This does not have to be the full node list.\n It just needs to have at least one broker that will respond to a\n Metadata API Request. Default: `127.0.0.1:9092`.\n - ``client_id`` (str): a name for this client. This string is passed in\n each request to servers and can be used to identify specific\n server-side log entries that correspond to this client. Also\n submitted to GroupCoordinator for logging with respect to\n consumer group administration. Default: `Robot`.\n - ``group_id`` (str or None): name of the consumer group to join for dynamic\n partition assignment (if enabled), and to use for fetching and\n committing offsets. If None, auto-partition assignment (via\n group coordinator) and offset commits are disabled.\n Default: `None`.\n - ``auto_offset_reset`` (str): A policy for resetting offsets on\n OffsetOutOfRange errors: `earliest` will move to the oldest\n available message, `latest` will move to the most recent. Any\n other value will raise the exception. Default: `latest`.\n - ``enable_auto_commit`` (bool): If true the consumer's offset will be\n periodically committed in the background. Default: `True`.\n \n Note:\n Configuration parameters are described in more detail at\n http://kafka-python.readthedocs.io/en/master/apidoc/KafkaConsumer.html\n \"\"\"\n\n self.consumer = KafkaConsumer(\n bootstrap_servers=bootstrap_servers,\n auto_offset_reset=auto_offset_reset,\n client_id=client_id,\n group_id=group_id,\n enable_auto_commit=enable_auto_commit,\n **kwargs\n )", "response": "Connects to kafka consumer."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nassigns a list of TopicPartitions to this consumer.", "response": "def assign_to_topic_partition(self, topic_partition=None):\n \"\"\"Assign a list of TopicPartitions to this consumer.\n \n - ``partitions`` (list of `TopicPartition`): Assignment for this instance.\n \"\"\"\n\n if isinstance(topic_partition, TopicPartition):\n topic_partition = [topic_partition]\n if not self._is_assigned(topic_partition):\n self.consumer.assign(topic_partition)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsubscribing to a list of topics or a regex pattern.", "response": "def subscribe_topic(self, topics=[], pattern=None):\n \"\"\"Subscribe to a list of topics, or a topic regex pattern.\n \n - ``topics`` (list): List of topics for subscription.\n - ``pattern`` (str): Pattern to match available topics. You must provide either topics or pattern,\n but not both.\n \"\"\"\n\n if not isinstance(topics, list):\n topics = [topics]\n self.consumer.subscribe(topics, pattern=pattern)"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_position(self, topic_partition=None):\n\n if isinstance(topic_partition, TopicPartition):\n return self.consumer.position(topic_partition)\n else:\n raise TypeError(\"topic_partition must be of type TopicPartition, create it with Create TopicPartition keyword.\")", "response": "Returns the offset of the next record that will be fetched."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef seek(self, offset, topic_partition=None):\n\n if isinstance(topic_partition, TopicPartition):\n self.consumer.seek(topic_partition, offset=offset)\n else:\n raise TypeError(\"topic_partition must be of type TopicPartition, create it with Create TopicPartition keyword.\")", "response": "Seeks the consumer to a specific offset for a topic partition."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef seek_to_beginning(self, topic_partition=None):\n\n if isinstance(topic_partition, TopicPartition):\n self.consumer.seek_to_beginning(topic_partition)\n else:\n raise TypeError(\"topic_partition must be of type TopicPartition, create it with Create TopicPartition keyword.\")", "response": "Seek to the oldest available offset for partitions."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef seek_to_end(self, topic_partition=None):\n\n if isinstance(topic_partition, TopicPartition):\n self.consumer.seek_to_end(topic_partition)\n else:\n raise TypeError(\"topic_partition must be of type TopicPartition, create it with Create TopicPartition keyword.\")", "response": "Seek to the most recent available offset for partitions."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_number_of_messages_in_topics(self, topics):\n\n if not isinstance(topics, list):\n topics = [topics]\n\n number_of_messages = 0\n for t in topics:\n part = self.get_kafka_partitions_for_topic(topic=t)\n Partitions = map(lambda p: TopicPartition(topic=t, partition=p), part)\n number_of_messages += self.get_number_of_messages_in_topicpartition(Partitions)\n\n return number_of_messages", "response": "Retrun number of messages in topics."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_number_of_messages_in_topicpartition(self, topic_partition=None):\n\n if isinstance(topic_partition, TopicPartition):\n topic_partition = [topic_partition]\n\n number_of_messages = 0\n assignment = self.consumer.assignment()\n\n self.consumer.unsubscribe()\n for Partition in topic_partition:\n if not isinstance(Partition, TopicPartition):\n raise TypeError(\"topic_partition must be of type TopicPartition, create it with Create TopicPartition keyword.\")\n\n self.assign_to_topic_partition(Partition)\n\n self.consumer.seek_to_end(Partition)\n end = self.consumer.position(Partition)\n self.consumer.seek_to_beginning(Partition)\n start = self.consumer.position(Partition)\n number_of_messages += end-start\n\n self.consumer.unsubscribe()\n self.consumer.assign(assignment)\n return number_of_messages", "response": "Returns the number of messages in a topic partition."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef poll(self, timeout_ms=0, max_records=None):\n\n messages = self.consumer.poll(timeout_ms=timeout_ms, max_records=max_records)\n\n result = []\n for _, msg in messages.items():\n for item in msg:\n result.append(item)\n return result", "response": "Fetch data from assigned topics / partitions."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nvalidates an email with the given key", "response": "def get(self, request, key):\n \"\"\"Validate an email with the given key\"\"\"\n\n try:\n email_val = EmailAddressValidation.objects.get(validation_key=key)\n except EmailAddressValidation.DoesNotExist:\n messages.error(request, _('The email address you are trying to '\n 'verify either has already been verified'\n ' or does not exist.'))\n return redirect('/')\n\n try:\n email = EmailAddress.objects.get(address=email_val.address)\n except EmailAddress.DoesNotExist:\n email = EmailAddress(address=email_val.address)\n\n if email.user and email.user.is_active:\n messages.error(request, _('The email address you are trying to '\n 'verify is already an active email '\n 'address.'))\n email_val.delete()\n return redirect('/')\n\n email.user = email_val.user\n email.save()\n email_val.delete()\n\n user = User.objects.get(username=email.user.username)\n user.is_active = True\n user.save()\n\n messages.success(request, _('Email address verified!'))\n return redirect('user_profile', username=email_val.user.username)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nremoving an email address validated or not.", "response": "def delete(self, request, key):\n \"\"\"Remove an email address, validated or not.\"\"\"\n\n request.DELETE = http.QueryDict(request.body)\n email_addr = request.DELETE.get('email')\n user_id = request.DELETE.get('user')\n\n if not email_addr:\n return http.HttpResponseBadRequest()\n\n try:\n email = EmailAddressValidation.objects.get(address=email_addr,\n user_id=user_id)\n except EmailAddressValidation.DoesNotExist:\n pass\n else:\n email.delete()\n return http.HttpResponse(status=204)\n\n try:\n email = EmailAddress.objects.get(address=email_addr,\n user_id=user_id)\n except EmailAddress.DoesNotExist:\n raise http.Http404\n\n email.user = None\n email.save()\n return http.HttpResponse(status=204)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating an email address as primary address.", "response": "def update(self, request, key):\n \"\"\"Set an email address as primary address.\"\"\"\n\n request.UPDATE = http.QueryDict(request.body)\n\n email_addr = request.UPDATE.get('email')\n user_id = request.UPDATE.get('user')\n if not email_addr:\n return http.HttpResponseBadRequest()\n\n try:\n email = EmailAddress.objects.get(address=email_addr,\n user_id=user_id)\n except EmailAddress.DoesNotExist:\n raise http.Http404\n\n email.user.email = email_addr\n email.user.save()\n return http.HttpResponse(status=204)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck if a logged user is trying to access the register page. If so redirect him to his profile.", "response": "def is_logged(self, user):\n \"\"\"Check if a logged user is trying to access the register page.\n If so, redirect him/her to his/her profile\"\"\"\n\n response = None\n if user.is_authenticated():\n if not user.needs_update:\n response = redirect('user_profile', username=user.username)\n\n return response"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the environment setting or return exception", "response": "def get_env_setting(setting):\n \"\"\" Get the environment setting or return exception \"\"\"\n try:\n return os.environ[setting]\n except KeyError:\n error_msg = \"Set the %s env variable\" % setting\n raise ImproperlyConfigured(error_msg)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef validate_social_account(account, url):\n\n request = urllib2.Request(urlparse.urljoin(url, account))\n request.get_method = lambda: 'HEAD'\n\n try:\n response = urllib2.urlopen(request)\n except urllib2.HTTPError:\n return False\n\n return response.code == 200", "response": "Verifies if a social account is valid."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates the RMSD of fitting.", "response": "def fitting_rmsd(w_fit, C_fit, r_fit, Xs):\n '''Calculate the RMSD of fitting.'''\n return np.sqrt(sum((geometry.point_line_distance(p, C_fit, w_fit) - r_fit) ** 2\n for p in Xs) / len(Xs))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconnect to kafka - ``bootstrap_servers``: default 127.0.0.1:9092 - ``client_id``: default: Robot", "response": "def connect_to_kafka(self, bootstrap_servers='127.0.0.1:9092',\n auto_offset_reset='latest',\n client_id='Robot',\n **kwargs\n ):\n \"\"\"Connect to kafka\n - ``bootstrap_servers``: default 127.0.0.1:9092\n - ``client_id``: default: Robot\n \"\"\"\n\n self.connect_consumer(\n bootstrap_servers=bootstrap_servers,\n auto_offset_reset=auto_offset_reset,\n client_id=client_id,\n **kwargs\n )\n self.connect_producer(bootstrap_servers=bootstrap_servers, client_id=client_id)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef drop_connection(self, name, database=None):\n request_executor = self._store.get_request_executor(database)\n command = DropSubscriptionConnectionCommand(name)\n request_executor.execute(command)", "response": "Drop a subscription connection to the server"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef execute_from_command_line(argv=None):\n os.environ.setdefault(\"DJANGO_SETTINGS_MODULE\", \"colab.settings\")\n from django.conf import settings\n\n if not hasattr(settings, 'SECRET_KEY') and 'initconfig' in sys.argv:\n command = initconfig.Command()\n command.handle()\n else:\n utility = ManagementUtility(argv)\n utility.execute()", "response": "A simple method that runs a ManagementUtility."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nnormalizes a vector based on its 2 norm.", "response": "def normalize(v):\n '''Normalize a vector based on its 2 norm.'''\n if 0 == np.linalg.norm(v):\n return v\n return v / np.linalg.norm(v)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef rotation_matrix_from_axis_and_angle(u, theta):\n '''Calculate a rotation matrix from an axis and an angle.'''\n \n x = u[0]\n y = u[1]\n z = u[2]\n s = np.sin(theta)\n c = np.cos(theta)\n \n return np.array([[c + x**2 * (1 - c), x * y * (1 - c) - z * s, x * z * (1 - c) + y * s],\n [y * x * (1 - c) + z * s, c + y**2 * (1 - c), y * z * (1 - c) - x * s ],\n [z * x * (1 - c) - y * s, z * y * (1 - c) + x * s, c + z**2 * (1 - c) ]])", "response": "Calculate a rotation matrix from an axis and an angle."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef point_line_distance(p, l_p, l_v):\n '''Calculate the distance between a point and a line defined\n by a point and a direction vector.\n '''\n l_v = normalize(l_v)\n u = p - l_p\n return np.linalg.norm(u - np.dot(u, l_v) * l_v)", "response": "Calculate the distance between a point and a line defined by a point and a direction vector."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef raw_query(self, query, query_parameters=None):\n self.assert_no_raw_query()\n\n if len(self._where_tokens) != 0 or len(self._select_tokens) != 0 or len(\n self._order_by_tokens) != 0 or len(self._group_by_tokens) != 0:\n raise InvalidOperationException(\n \"You can only use raw_query on a new query, without applying any operations \"\n \"(such as where, select, order_by, group_by, etc)\")\n\n if query_parameters:\n self.query_parameters = query_parameters\n self._query = query\n return self", "response": "This method is used to get all the document that equal to the query."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef where(self, exact=False, **kwargs):\n for field_name in kwargs:\n if isinstance(kwargs[field_name], list):\n self.where_in(field_name, kwargs[field_name], exact)\n else:\n self.where_equals(field_name, kwargs[field_name], exact)\n return self", "response": "This method will return all the document that equal to the value within kwargs with the specific key\n "} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef search(self, field_name, search_terms, operator=QueryOperator.OR):\n\n if field_name is None:\n raise ValueError(\"None field_name is invalid\")\n\n field_name = Query.escape_if_needed(field_name)\n self._add_operator_if_needed()\n self.negate_if_needed(field_name)\n\n self.last_equality = {field_name: \"(\" + search_terms + \")\" if ' ' in search_terms else search_terms}\n token = _Token(field_name=field_name, token=\"search\", value=self.add_query_parameter(search_terms),\n search_operator=operator)\n token.write = self.rql_where_write(token)\n self._where_tokens.append(token)\n return self", "response": "For more complex text searching\n "} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd a where clause to get all the documents that ends with the value in the giving field_name", "response": "def where_ends_with(self, field_name, value):\n \"\"\"\n To get all the document that ends with the value in the giving field_name\n\n @param str field_name:The field name in the index you want to query.\n @param str value: The value will be the fields value you want to query\n \"\"\"\n if field_name is None:\n raise ValueError(\"None field_name is invalid\")\n\n field_name = Query.escape_if_needed(field_name)\n self._add_operator_if_needed()\n self.negate_if_needed(field_name)\n\n self.last_equality = {field_name: value}\n token = _Token(field_name=field_name, token=\"endsWith\", value=self.add_query_parameter(value))\n token.write = self.rql_where_write(token)\n self._where_tokens.append(token)\n\n return self"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef where_in(self, field_name, values, exact=False):\n field_name = Query.escape_if_needed(field_name)\n self._add_operator_if_needed()\n self.negate_if_needed(field_name)\n\n token = _Token(field_name=field_name, value=self.add_query_parameter(list(Utils.unpack_iterable(values))),\n token=\"in\", exact=exact)\n token.write = self.rql_where_write(token)\n self._where_tokens.append(token)\n\n return self", "response": "Check that the field_name has one of the specified values."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to_facets(self, facets, start=0, page_size=None):\n\n if len(facets) == 0:\n raise ValueError(\"Facets must contain at least one entry\", \"facets\")\n str_query = self.__str__()\n facet_query = FacetQuery(str_query, None, facets, start, page_size, query_parameters=self.query_parameters,\n wait_for_non_stale_results=self.wait_for_non_stale_results,\n wait_for_non_stale_results_timeout=self.timeout, cutoff_etag=self.cutoff_etag)\n\n command = GetFacetsCommand(query=facet_query)\n return self.session.requests_executor.execute(command)", "response": "Query the facets results for this entry"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef show_G_distribution(data):\n '''Show the distribution of the G function.'''\n Xs, t = fitting.preprocess_data(data) \n\n Theta, Phi = np.meshgrid(np.linspace(0, np.pi, 50), np.linspace(0, 2 * np.pi, 50))\n G = []\n\n for i in range(len(Theta)):\n G.append([])\n for j in range(len(Theta[i])):\n w = fitting.direction(Theta[i][j], Phi[i][j])\n G[-1].append(fitting.G(w, Xs))\n\n plt.imshow(G, extent=[0, np.pi, 0, 2 * np.pi], origin='lower')\n plt.show()", "response": "Show the distribution of the G function."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef show_fit(w_fit, C_fit, r_fit, Xs):\n '''Plot the fitting given the fitted axis direction, the fitted\n center, the fitted radius and the data points.\n '''\n\n fig = plt.figure()\n ax = fig.gca(projection='3d')\n \n # Plot the data points\n \n ax.scatter([X[0] for X in Xs], [X[1] for X in Xs], [X[2] for X in Xs])\n \n # Get the transformation matrix\n\n theta = np.arccos(np.dot(w_fit, np.array([0, 0, 1])))\n phi = np.arctan2(w_fit[1], w_fit[0])\n\n M = np.dot(rotation_matrix_from_axis_and_angle(np.array([0, 0, 1]), phi),\n rotation_matrix_from_axis_and_angle(np.array([0, 1, 0]), theta))\n\n # Plot the cylinder surface\n \n delta = np.linspace(-np.pi, np.pi, 20)\n z = np.linspace(-10, 10, 20)\n\n Delta, Z = np.meshgrid(delta, z)\n X = r_fit * np.cos(Delta)\n Y = r_fit * np.sin(Delta)\n\n for i in range(len(X)):\n for j in range(len(X[i])):\n p = np.dot(M, np.array([X[i][j], Y[i][j], Z[i][j]])) + C_fit\n\n X[i][j] = p[0]\n Y[i][j] = p[1]\n Z[i][j] = p[2]\n\n ax.plot_surface(X, Y, Z, alpha=0.2)\n\n # Plot the center and direction\n\n ax.quiver(C_fit[0], C_fit[1], C_fit[2], \n r_fit * w_fit[0], r_fit * w_fit[1], r_fit * w_fit[2], color='red')\n\n\n plt.show()", "response": "Plot the fitting given the fitted axis direction center radius and the data points."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_window(self, highlight_locations):\n\n if len(self.text_block) <= self.max_length:\n return (0, self.max_length)\n\n num_chars_before = getattr(\n settings,\n 'HIGHLIGHT_NUM_CHARS_BEFORE_MATCH',\n 0\n )\n\n best_start, best_end = super(ColabHighlighter, self).find_window(\n highlight_locations\n )\n if best_start <= num_chars_before:\n best_end -= best_start\n best_start = 0\n else:\n best_start -= num_chars_before\n best_end -= num_chars_before\n\n return (best_start, best_end)", "response": "Finds the window that contains the text."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __set(self, key, real_value, coded_value):\n morse_set = self.get(key, StringMorsel())\n morse_set.set(key, real_value, coded_value)\n dict.__setitem__(self, key, morse_set)", "response": "Private method for setting a cookie s value"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef login(self):\n response = self.session.get(self.base_url + '/login_sid.lua', timeout=10)\n xml = ET.fromstring(response.text)\n if xml.find('SID').text == \"0000000000000000\":\n challenge = xml.find('Challenge').text\n url = self.base_url + \"/login_sid.lua\"\n response = self.session.get(url, params={\n \"username\": self.username,\n \"response\": self.calculate_response(challenge, self.password),\n }, timeout=10)\n xml = ET.fromstring(response.text)\n sid = xml.find('SID').text\n if xml.find('SID').text == \"0000000000000000\":\n blocktime = int(xml.find('BlockTime').text)\n exc = Exception(\"Login failed, please wait {} seconds\".format(\n blocktime\n ))\n exc.blocktime = blocktime\n raise exc\n self.sid = sid\n return sid", "response": "Try to login and set the internal session id."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the response for the challenge - response authentication", "response": "def calculate_response(self, challenge, password):\n \"\"\"Calculate response for the challenge-response authentication\"\"\"\n to_hash = (challenge + \"-\" + password).encode(\"UTF-16LE\")\n hashed = hashlib.md5(to_hash).hexdigest()\n return \"{0}-{1}\".format(challenge, hashed)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of Actor objects for querying SmartHome devices.", "response": "def get_actors(self):\n \"\"\"\n Returns a list of Actor objects for querying SmartHome devices.\n\n This is currently the only working method for getting temperature data.\n \"\"\"\n devices = self.homeautoswitch(\"getdevicelistinfos\")\n xml = ET.fromstring(devices)\n\n actors = []\n for device in xml.findall('device'):\n actors.append(Actor(fritzbox=self, device=device))\n\n return actors"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_actor_by_ain(self, ain):\n for actor in self.get_actors():\n if actor.actor_id == ain:\n return actor", "response": "Return a actor identified by it s ain."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef homeautoswitch(self, cmd, ain=None, param=None):\n assert self.sid, \"Not logged in\"\n params = {\n 'switchcmd': cmd,\n 'sid': self.sid,\n }\n if param is not None:\n params['param'] = param\n if ain:\n params['ain'] = ain\n\n url = self.base_url + '/webservices/homeautoswitch.lua'\n response = self.session.get(url, params=params, timeout=10)\n response.raise_for_status()\n return response.text.strip().encode('utf-8')", "response": "Call a switch method."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_switch_actors(self):\n actors = {}\n for ain in self.homeautoswitch(\"getswitchlist\").split(','):\n actors[ain] = {\n 'name': self.homeautoswitch(\"getswitchname\", ain),\n 'state': bool(self.homeautoswitch(\"getswitchstate\", ain)),\n 'present': bool(self.homeautoswitch(\"getswitchpresent\", ain)),\n 'power': self.homeautoswitch(\"getswitchpower\", ain),\n 'energy': self.homeautoswitch(\"getswitchenergy\", ain),\n 'temperature': self.homeautoswitch(\"getswitchtemperature\", ain),\n }\n return actors", "response": "Get information about all actors in the current environment"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a list of devices.", "response": "def get_devices(self):\n \"\"\"\n Return a list of devices.\n Deprecated, use get_actors instead.\n \"\"\"\n url = self.base_url + '/net/home_auto_query.lua'\n response = self.session.get(url, params={\n 'sid': self.sid,\n 'command': 'AllOutletStates',\n 'xhr': 0,\n }, timeout=15)\n response.raise_for_status()\n data = response.json()\n count = int(data[\"Outlet_count\"])\n devices = []\n for i in range(1, count + 1):\n device = Device(\n int(data[\"DeviceID_{0}\".format(i)]),\n int(data[\"DeviceConnectState_{0}\".format(i)]),\n int(data[\"DeviceSwitchState_{0}\".format(i)])\n )\n devices.append(device)\n return devices"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget all energy consumption data for the device.", "response": "def get_consumption(self, deviceid, timerange=\"10\"):\n \"\"\"\n Return all available energy consumption data for the device.\n You need to divice watt_values by 100 and volt_values by 1000\n to get the \"real\" values.\n\n :return: dict\n \"\"\"\n tranges = (\"10\", \"24h\", \"month\", \"year\")\n if timerange not in tranges:\n raise ValueError(\n \"Unknown timerange. Possible values are: {0}\".format(tranges)\n )\n\n url = self.base_url + \"/net/home_auto_query.lua\"\n response = self.session.get(url, params={\n 'sid': self.sid,\n 'command': 'EnergyStats_{0}'.format(timerange),\n 'id': deviceid,\n 'xhr': 0,\n }, timeout=15)\n response.raise_for_status()\n\n data = response.json()\n result = {}\n\n # Single result values\n values_map = {\n 'MM_Value_Amp': 'mm_value_amp',\n 'MM_Value_Power': 'mm_value_power',\n 'MM_Value_Volt': 'mm_value_volt',\n\n 'EnStats_average_value': 'enstats_average_value',\n 'EnStats_max_value': 'enstats_max_value',\n 'EnStats_min_value': 'enstats_min_value',\n 'EnStats_timer_type': 'enstats_timer_type',\n\n 'sum_Day': 'sum_day',\n 'sum_Month': 'sum_month',\n 'sum_Year': 'sum_year',\n }\n for avm_key, py_key in values_map.items():\n result[py_key] = int(data[avm_key])\n\n # Stats counts\n count = int(data[\"EnStats_count\"])\n watt_values = [None for i in range(count)]\n volt_values = [None for i in range(count)]\n for i in range(1, count + 1):\n watt_values[i - 1] = int(data[\"EnStats_watt_value_{}\".format(i)])\n volt_values[i - 1] = int(data[\"EnStats_volt_value_{}\".format(i)])\n\n result['watt_values'] = watt_values\n result['volt_values'] = volt_values\n\n return result"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the system logs since the last reboot.", "response": "def get_logs(self):\n \"\"\"\n Return the system logs since the last reboot.\n \"\"\"\n assert BeautifulSoup, \"Please install bs4 to use this method\"\n\n url = self.base_url + \"/system/syslog.lua\"\n response = self.session.get(url, params={\n 'sid': self.sid,\n 'stylemode': 'print',\n }, timeout=15)\n response.raise_for_status()\n\n entries = []\n tree = BeautifulSoup(response.text)\n rows = tree.find('table').find_all('tr')\n for row in rows:\n columns = row.find_all(\"td\")\n date = columns[0].string\n time = columns[1].string\n message = columns[2].find(\"a\").string\n\n merged = \"{} {} {}\".format(date, time, message.encode(\"UTF-8\"))\n msg_hash = hashlib.md5(merged).hexdigest()\n entries.append(LogEntry(date, time, message, msg_hash))\n return entries"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn True if the Hawk nonce has already been seen.", "response": "def seen_nonce(id, nonce, timestamp):\n \"\"\"\n Returns True if the Hawk nonce has been seen already.\n \"\"\"\n key = '{id}:{n}:{ts}'.format(id=id, n=nonce, ts=timestamp)\n if cache.get(key):\n log.warning('replay attack? already processed nonce {k}'\n .format(k=key))\n return True\n else:\n log.debug('caching nonce {k}'.format(k=key))\n cache.set(key, True,\n # We only need the nonce until the message itself expires.\n # This also adds a little bit of padding.\n timeout=getattr(settings, 'HAWK_MESSAGE_EXPIRATION',\n default_message_expiration) + 5)\n return False"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef actors(context):\n fritz = context.obj\n fritz.login()\n\n for actor in fritz.get_actors():\n click.echo(\"{} ({} {}; AIN {} )\".format(\n actor.name,\n actor.manufacturer,\n actor.productname,\n actor.actor_id,\n ))\n if actor.has_temperature:\n click.echo(\"Temp: act {} target {}; battery (low): {}\".format(\n actor.temperature,\n actor.target_temperature,\n actor.battery_low,\n ))\n\n click.echo(\"Temp (via get): act {} target {}\".format(\n actor.get_temperature(),\n actor.get_target_temperature(),\n ))", "response": "Display a list of actors"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef energy(context, features):\n fritz = context.obj\n fritz.login()\n\n for actor in fritz.get_actors():\n if actor.temperature is not None:\n click.echo(\"{} ({}): {:.2f} Watt current, {:.3f} wH total, {:.2f} \u00b0C\".format(\n actor.name.encode('utf-8'),\n actor.actor_id,\n (actor.get_power() or 0.0) / 1000,\n (actor.get_energy() or 0.0) / 100,\n actor.temperature\n ))\n else:\n click.echo(\"{} ({}): {:.2f} Watt current, {:.3f} wH total, offline\".format(\n actor.name.encode('utf-8'),\n actor.actor_id,\n (actor.get_power() or 0.0) / 1000,\n (actor.get_energy() or 0.0) / 100\n ))\n if features:\n click.echo(\" Features: PowerMeter: {}, Temperatur: {}, Switch: {}\".format(\n actor.has_powermeter, actor.has_temperature, actor.has_switch\n ))", "response": "Display energy stats of all actors"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconnect to a carbon server and display statistics of all actors and actors in a single node.", "response": "def graphite(context, server, port, interval, prefix):\n \"\"\"Display energy stats of all actors\"\"\"\n fritz = context.obj\n fritz.login()\n sid_ttl = time.time() + 600\n\n # Find actors and create carbon keys\n click.echo(\" * Requesting actors list\")\n simple_chars = re.compile('[^A-Za-z0-9]+')\n actors = fritz.get_actors()\n keys = {}\n for actor in actors:\n keys[actor.name] = \"{}.{}\".format(\n prefix,\n simple_chars.sub('_', actor.name)\n )\n\n # Connect to carbon\n click.echo(\" * Trying to connect to carbon\")\n timeout = 2\n sock = socket.socket()\n sock.settimeout(timeout)\n try:\n sock.connect((server, port))\n except socket.timeout:\n raise Exception(\"Took over {} second(s) to connect to {}\".format(\n timeout, server\n ))\n except Exception as error:\n raise Exception(\"unknown exception while connecting to {} - {}\".format(\n server, error\n ))\n\n def send(key, value):\n \"\"\"Send a key-value-pair to carbon\"\"\"\n now = int(time.time())\n payload = \"{} {} {}\\n\".format(key, value, now)\n sock.sendall(payload)\n\n while True:\n if time.time() > sid_ttl:\n click.echo(\" * Requesting new SID\")\n fritz.login()\n sid_ttl = time.time() + 600\n\n click.echo(\" * Requesting statistics\")\n for actor in actors:\n power = actor.get_power()\n total = actor.get_energy()\n click.echo(\" -> {}: {:.2f} Watt current, {:.3f} wH total\".format(\n actor.name, power / 1000, total / 100\n ))\n\n send(keys[actor.name] + '.current', power)\n send(keys[actor.name] + '.total', total)\n\n time.sleep(interval)"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef switch_on(context, ain):\n context.obj.login()\n actor = context.obj.get_actor_by_ain(ain)\n if actor:\n click.echo(\"Switching {} on\".format(actor.name))\n actor.switch_on()\n else:\n click.echo(\"Actor not found: {}\".format(ain))", "response": "Switch an actor s power to ON"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nswitch the state of an actor", "response": "def switch_state(context, ain):\n \"\"\"Get an actor's power state\"\"\"\n context.obj.login()\n actor = context.obj.get_actor_by_ain(ain)\n if actor:\n click.echo(\"State for {} is: {}\".format(ain,'ON' if actor.get_state() else 'OFF'))\n else:\n click.echo(\"Actor not found: {}\".format(ain))"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef switch_toggle(context, ain):\n context.obj.login()\n actor = context.obj.get_actor_by_ain(ain)\n if actor:\n if actor.get_state():\n actor.switch_off()\n click.echo(\"State for {} is now OFF\".format(ain))\n else:\n actor.switch_on()\n click.echo(\"State for {} is now ON\".format(ain))\n else:\n click.echo(\"Actor not found: {}\".format(ain))", "response": "Toggle an actor s power state"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nshows system logs since last reboot", "response": "def logs(context, format):\n \"\"\"Show system logs since last reboot\"\"\"\n fritz = context.obj\n fritz.login()\n\n messages = fritz.get_logs()\n if format == \"plain\":\n for msg in messages:\n merged = \"{} {} {}\".format(msg.date, msg.time, msg.message.encode(\"UTF-8\"))\n click.echo(merged)\n\n if format == \"json\":\n entries = [msg._asdict() for msg in messages]\n click.echo(json.dumps({\n \"entries\": entries,\n }))"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the current power usage in milliWatts.", "response": "def get_power(self):\n \"\"\"\n Returns the current power usage in milliWatts.\n Attention: Returns None if the value can't be queried or is unknown.\n \"\"\"\n value = self.box.homeautoswitch(\"getswitchpower\", self.actor_id)\n return int(value) if value.isdigit() else None"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_energy(self):\n value = self.box.homeautoswitch(\"getswitchenergy\", self.actor_id)\n return int(value) if value.isdigit() else None", "response": "Returns the consumed energy since the start of the statistics in Wh."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_temperature(self):\n #raise NotImplementedError(\"This should work according to the AVM docs, but don't...\")\n value = self.box.homeautoswitch(\"gettemperature\", self.actor_id)\n if value.isdigit():\n self.temperature = float(value)/10\n else:\n self.temperature = None\n return self.temperature", "response": "Returns the current environment temperature."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the actual target temperature.", "response": "def get_target_temperature(self):\n \"\"\"\n Returns the actual target temperature.\n Attention: Returns None if the value can't be queried or is unknown.\n \"\"\"\n value = self.box.homeautoswitch(\"gethkrtsoll\", self.actor_id)\n self.target_temperature = self.__get_temp(value)\n return self.target_temperature"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_temperature(self, temp):\n\n # Temperature is send to fritz.box a little weird\n param = 16 + ( ( temp - 8 ) * 2 )\n if param < 16:\n param = 253\n logger.info(\"Actor \" + self.name + \": Temperature control set to off\")\n elif param >= 56:\n param = 254\n logger.info(\"Actor \" + self.name + \": Temperature control set to on\")\n else:\n logger.info(\"Actor \" + self.name + \": Temperature control set to \" + str(temp))\n\n return self.box.homeautoswitch(\"sethkrtsoll\", self.actor_id, param)", "response": "Sets the temperature in celcius\n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_openshift_base_uri(self):\n deprecated_key = \"openshift_uri\"\n key = \"openshift_url\"\n val = self._get_value(deprecated_key, self.conf_section, deprecated_key)\n if val is not None:\n warnings.warn(\"%r is deprecated, use %r instead\" % (deprecated_key, key))\n return val\n return self._get_value(key, self.conf_section, key)", "response": "Get the OpenShift base URI from the configuration file."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_builder_openshift_url(self):\n key = \"builder_openshift_url\"\n url = self._get_deprecated(key, self.conf_section, key)\n if url is None:\n logger.warning(\"%r not found, falling back to get_openshift_base_uri()\", key)\n url = self.get_openshift_base_uri()\n return url", "response": "url of OpenShift where builder will connect"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef generate_nodeselector_dict(self, nodeselector_str):\n nodeselector = {}\n if nodeselector_str and nodeselector_str != 'none':\n constraints = [x.strip() for x in nodeselector_str.split(',')]\n raw_nodeselector = dict([constraint.split('=', 1) for constraint in constraints])\n nodeselector = dict([k.strip(), v.strip()] for (k, v) in raw_nodeselector.items())\n\n return nodeselector", "response": "helper method for generating nodeselector dict"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_platform_node_selector(self, platform):\n nodeselector = {}\n if platform:\n nodeselector_str = self._get_value(\"node_selector.\" + platform, self.conf_section,\n \"node_selector.\" + platform)\n nodeselector = self.generate_nodeselector_dict(nodeselector_str)\n\n return nodeselector", "response": "search the configuration for entries of the form node_selector. platform\n "} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load(self):\n\n self._validate()\n self._logger.logging_load()\n self.encoding = get_file_encoding(self.source, self.encoding)\n\n if six.PY3:\n self._csv_reader = csv.reader(\n io.open(self.source, \"r\", encoding=self.encoding),\n delimiter=self.delimiter,\n quotechar=self.quotechar,\n strict=True,\n skipinitialspace=True,\n )\n else:\n self._csv_reader = csv.reader(\n _utf_8_encoder(io.open(self.source, \"r\", encoding=self.encoding)),\n delimiter=self.delimiter,\n quotechar=self.quotechar,\n strict=True,\n skipinitialspace=True,\n )\n\n formatter = CsvTableFormatter(self._to_data_matrix())\n formatter.accept(self)\n\n return formatter.to_table_data()", "response": "Loads the table data from a CSV file."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads the table data from a CSV file.", "response": "def load(self):\n \"\"\"\n Extract tabular data as |TableData| instances from a CSV text object.\n |load_source_desc_text|\n\n :return:\n Loaded table data.\n |load_table_name_desc|\n\n =================== ========================================\n Format specifier Value after the replacement\n =================== ========================================\n ``%(filename)s`` ``\"\"``\n ``%(format_name)s`` ``\"csv\"``\n ``%(format_id)s`` |format_id_desc|\n ``%(global_id)s`` |global_id|\n =================== ========================================\n :rtype: |TableData| iterator\n :raises pytablereader.DataError:\n If the CSV data is invalid.\n\n .. seealso::\n :py:func:`csv.reader`\n \"\"\"\n\n self._validate()\n self._logger.logging_load()\n\n self._csv_reader = csv.reader(\n six.StringIO(self.source.strip()),\n delimiter=self.delimiter,\n quotechar=self.quotechar,\n strict=True,\n skipinitialspace=True,\n )\n formatter = CsvTableFormatter(self._to_data_matrix())\n formatter.accept(self)\n\n return formatter.to_table_data()"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets parameters according to specification", "response": "def set_params(self, **kwargs):\n \"\"\"\n set parameters according to specification\n\n these parameters are accepted:\n\n :param pulp_secret: str, resource name of pulp secret\n :param koji_target: str, koji tag with packages used to build the image\n :param kojiroot: str, URL from which koji packages are fetched\n :param kojihub: str, URL of the koji hub\n :param koji_certs_secret: str, resource name of secret that holds the koji certificates\n :param koji_task_id: int, Koji Task that created this build config\n :param flatpak: if we should build a Flatpak OCI Image\n :param filesystem_koji_task_id: int, Koji Task that created the base filesystem\n :param pulp_registry: str, name of pulp registry in dockpulp.conf\n :param sources_command: str, command used to fetch dist-git sources\n :param architecture: str, architecture we are building for\n :param vendor: str, vendor name\n :param build_host: str, host the build will run on or None for auto\n :param authoritative_registry: str, the docker registry authoritative for this image\n :param distribution_scope: str, distribution scope for this image\n (private, authoritative-source-only, restricted, public)\n :param use_auth: bool, use auth from atomic-reactor?\n :param platform_node_selector: dict, a nodeselector for a specific platform\n :param platform_descriptors: dict, platforms and their archiectures and enable_v1 settings\n :param scratch_build_node_selector: dict, a nodeselector for scratch builds\n :param explicit_build_node_selector: dict, a nodeselector for explicit builds\n :param auto_build_node_selector: dict, a nodeselector for auto builds\n :param isolated_build_node_selector: dict, a nodeselector for isolated builds\n :param is_auto: bool, indicates if build is auto build\n :param parent_images_digests: dict, mapping image names with tags to platform specific\n digests, example:\n {'registry.fedorahosted.org/fedora:29': {\n x86_64': 'registry.fedorahosted.org/fedora@sha256:....'}\n }\n \"\"\"\n\n # Here we cater to the koji \"scratch\" build type, this will disable\n # all plugins that might cause importing of data to koji\n self.scratch = kwargs.pop('scratch', False)\n # When true, it indicates build was automatically started by\n # OpenShift via a trigger, for instance ImageChangeTrigger\n self.is_auto = kwargs.pop('is_auto', False)\n # An isolated build is meant to patch a certain release and not\n # update transient tags in container registry\n self.isolated = kwargs.pop('isolated', False)\n\n self.validate_build_variation()\n\n self.base_image = kwargs.get('base_image')\n self.platform_node_selector = kwargs.get('platform_node_selector', {})\n self.platform_descriptors = kwargs.get('platform_descriptors', {})\n self.scratch_build_node_selector = kwargs.get('scratch_build_node_selector', {})\n self.explicit_build_node_selector = kwargs.get('explicit_build_node_selector', {})\n self.auto_build_node_selector = kwargs.get('auto_build_node_selector', {})\n self.isolated_build_node_selector = kwargs.get('isolated_build_node_selector', {})\n\n logger.debug(\"setting params '%s' for %s\", kwargs, self.spec)\n self.spec.set_params(**kwargs)\n self.osbs_api = kwargs.pop('osbs_api')"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef has_ist_trigger(self):\n triggers = self.template['spec'].get('triggers', [])\n if not triggers:\n return False\n for trigger in triggers:\n if trigger['type'] == 'ImageChange' and \\\n trigger['imageChange']['from']['kind'] == 'ImageStreamTag':\n return True\n return False", "response": "Return True if this BuildConfig has ImageStreamTag trigger."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_secret_for_plugin(self, secret, plugin=None, mount_path=None):\n has_plugin_conf = False\n if plugin is not None:\n has_plugin_conf = self.dj.dock_json_has_plugin_conf(plugin[0],\n plugin[1])\n if 'secrets' in self.template['spec']['strategy']['customStrategy']:\n if not plugin or has_plugin_conf:\n\n custom = self.template['spec']['strategy']['customStrategy']\n if mount_path:\n secret_path = mount_path\n else:\n secret_path = os.path.join(SECRETS_PATH, secret)\n\n logger.info(\"Configuring %s secret at %s\", secret, secret_path)\n existing = [secret_mount for secret_mount in custom['secrets']\n if secret_mount['secretSource']['name'] == secret]\n if existing:\n logger.debug(\"secret %s already set\", secret)\n else:\n custom['secrets'].append({\n 'secretSource': {\n 'name': secret,\n },\n 'mountPath': secret_path,\n })\n\n # there's no need to set args if no plugin secret specified\n # this is used in tag_and_push plugin, as it sets secret path\n # for each registry separately\n if plugin and plugin[2] is not None:\n self.dj.dock_json_set_arg(*(plugin + (secret_path,)))\n else:\n logger.debug(\"not setting secret for unused plugin %s\",\n plugin[1])", "response": "Sets secret for a specific plugin"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_secrets(self, secrets):\n secret_set = False\n for (plugin, secret) in secrets.items():\n if not isinstance(plugin, tuple) or len(plugin) != 3:\n raise ValueError('got \"%s\" as secrets key, need 3-tuple' % plugin)\n if secret is not None:\n if isinstance(secret, list):\n for secret_item in secret:\n self.set_secret_for_plugin(secret_item, plugin=plugin)\n else:\n self.set_secret_for_plugin(secret, plugin=plugin)\n secret_set = True\n\n if not secret_set:\n # remove references to secret if no secret was set\n if 'secrets' in self.template['spec']['strategy']['customStrategy']:\n del self.template['spec']['strategy']['customStrategy']['secrets']", "response": "Sets the secrets for the current instance of the class."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef remove_tag_and_push_registries(tag_and_push_registries, version):\n registries = [uri\n for uri, regdict in tag_and_push_registries.items()\n if regdict['version'] == version]\n for registry in registries:\n logger.info(\"removing %s registry: %s\", version, registry)\n del tag_and_push_registries[registry]", "response": "Remove matching entries from tag_and_push_registries."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadjust the internal state of the registry API versions based on the supported registry API versions.", "response": "def adjust_for_registry_api_versions(self):\n \"\"\"\n Enable/disable plugins depending on supported registry API versions\n \"\"\"\n versions = self.spec.registry_api_versions.value\n\n if 'v2' not in versions:\n raise OsbsValidationException('v1-only docker registry API is not supported')\n\n try:\n push_conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',\n 'tag_and_push')\n tag_and_push_registries = push_conf['args']['registries']\n except (KeyError, IndexError):\n tag_and_push_registries = {}\n\n if 'v1' not in versions:\n # Remove v1-only plugins\n for phase, name in [('postbuild_plugins', 'pulp_push')]:\n logger.info(\"removing v1-only plugin: %s\", name)\n self.dj.remove_plugin(phase, name)\n\n # remove extra tag_and_push config\n self.remove_tag_and_push_registries(tag_and_push_registries, 'v1')\n\n # Remove 'version' from tag_and_push plugin config as it's no\n # longer needed\n for regdict in tag_and_push_registries.values():\n if 'version' in regdict:\n del regdict['version']"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nremove trigger - related plugins when needed", "response": "def adjust_for_triggers(self):\n \"\"\"Remove trigger-related plugins when needed\n\n If there are no triggers defined, it's assumed the\n feature is disabled and all trigger-related plugins\n are removed.\n\n If there are triggers defined, and this is a custom\n base image, some trigger-related plugins do not apply.\n\n Additionally, this method ensures that custom base\n images never have triggers since triggering a base\n image rebuild is not a valid scenario.\n \"\"\"\n triggers = self.template['spec'].get('triggers', [])\n\n remove_plugins = [\n (\"prebuild_plugins\", \"check_and_set_rebuild\"),\n (\"prebuild_plugins\", \"stop_autorebuild_if_disabled\"),\n ]\n\n should_remove = False\n if triggers and (self.is_custom_base_image() or self.is_from_scratch_image()):\n if self.is_custom_base_image():\n msg = \"removing %s from request because custom base image\"\n elif self.is_from_scratch_image():\n msg = 'removing %s from request because FROM scratch image'\n del self.template['spec']['triggers']\n should_remove = True\n\n elif not triggers:\n msg = \"removing %s from request because there are no triggers\"\n should_remove = True\n\n if should_remove:\n for when, which in remove_plugins:\n logger.info(msg, which)\n self.dj.remove_plugin(when, which)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadjusts the metadata for scratch builds.", "response": "def adjust_for_scratch(self):\n \"\"\"\n Remove certain plugins in order to handle the \"scratch build\"\n scenario. Scratch builds must not affect subsequent builds,\n and should not be imported into Koji.\n \"\"\"\n if self.scratch:\n self.template['spec'].pop('triggers', None)\n\n remove_plugins = [\n (\"prebuild_plugins\", \"koji_parent\"),\n (\"postbuild_plugins\", \"compress\"), # required only to make an archive for Koji\n (\"postbuild_plugins\", \"pulp_pull\"), # required only to make an archive for Koji\n (\"postbuild_plugins\", \"koji_upload\"),\n (\"postbuild_plugins\", \"fetch_worker_metadata\"),\n (\"postbuild_plugins\", \"compare_components\"),\n (\"postbuild_plugins\", \"import_image\"),\n (\"exit_plugins\", \"koji_promote\"),\n (\"exit_plugins\", \"koji_import\"),\n (\"exit_plugins\", \"koji_tag_build\"),\n (\"exit_plugins\", \"remove_worker_metadata\"),\n (\"exit_plugins\", \"import_image\"),\n ]\n\n if not self.has_tag_suffixes_placeholder():\n remove_plugins.append((\"postbuild_plugins\", \"tag_from_config\"))\n\n for when, which in remove_plugins:\n logger.info(\"removing %s from scratch build request\",\n which)\n self.dj.remove_plugin(when, which)\n\n if self.dj.dock_json_has_plugin_conf('postbuild_plugins',\n 'tag_by_labels'):\n self.dj.dock_json_set_arg('postbuild_plugins', 'tag_by_labels',\n 'unique_tag_only', True)\n\n self.set_label('scratch', 'true')"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef adjust_for_custom_base_image(self):\n plugins = []\n if self.is_custom_base_image():\n # Plugins irrelevant to building base images.\n plugins.append((\"prebuild_plugins\", \"pull_base_image\"))\n plugins.append((\"prebuild_plugins\", \"koji_parent\"))\n plugins.append((\"prebuild_plugins\", \"inject_parent_image\"))\n msg = \"removing %s from custom image build request\"\n else:\n # Plugins not needed for building non base images.\n plugins.append((\"prebuild_plugins\", \"add_filesystem\"))\n msg = \"removing %s from non custom image build request\"\n\n for when, which in plugins:\n logger.info(msg, which)\n self.dj.remove_plugin(when, which)", "response": "Disable plugins to handle builds depending on whether this is a custom base image."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef render_koji(self):\n phase = 'prebuild_plugins'\n plugin = 'koji'\n if not self.dj.dock_json_has_plugin_conf(phase, plugin):\n return\n\n if self.spec.yum_repourls.value:\n logger.info(\"removing koji from request \"\n \"because there is yum repo specified\")\n self.dj.remove_plugin(phase, plugin)\n elif not (self.spec.koji_target.value and\n self.spec.kojiroot.value and\n self.spec.kojihub.value):\n logger.info(\"removing koji from request as not specified\")\n self.dj.remove_plugin(phase, plugin)\n else:\n self.dj.dock_json_set_arg(phase, plugin,\n \"target\", self.spec.koji_target.value)\n self.dj.dock_json_set_arg(phase, plugin,\n \"root\", self.spec.kojiroot.value)\n self.dj.dock_json_set_arg(phase, plugin,\n \"hub\", self.spec.kojihub.value)\n if self.spec.proxy.value:\n self.dj.dock_json_set_arg(phase, plugin,\n \"proxy\", self.spec.proxy.value)", "response": "Render koji from yum repo if yum repo is specified."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef render_bump_release(self):\n phase = 'prebuild_plugins'\n plugin = 'bump_release'\n if not self.dj.dock_json_has_plugin_conf(phase, plugin):\n return\n\n if self.spec.release.value:\n logger.info('removing %s from request as release already specified',\n plugin)\n self.dj.remove_plugin(phase, plugin)\n return\n\n hub = self.spec.kojihub.value\n if not hub:\n logger.info('removing %s from request as koji hub not specified',\n plugin)\n self.dj.remove_plugin(phase, plugin)\n return\n\n self.dj.dock_json_set_arg(phase, plugin, 'hub', hub)\n\n # For flatpak, we want a name-version-release of\n # --., where the . makes\n # sure that the build is unique in Koji\n if self.spec.flatpak.value:\n self.dj.dock_json_set_arg(phase, plugin, 'append', True)", "response": "If the bump_release plugin is present, configure it"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef render_pulp_pull(self):\n # pulp_pull is a multi-phase plugin\n phases = ('postbuild_plugins', 'exit_plugins')\n plugin = 'pulp_pull'\n for phase in phases:\n if not self.dj.dock_json_has_plugin_conf(phase, plugin):\n continue\n\n pulp_registry = self.spec.pulp_registry.value\n if not pulp_registry:\n logger.info(\"removing %s from request, requires pulp_registry\", pulp_registry)\n self.dj.remove_plugin(phase, plugin)\n continue\n\n if not self.spec.kojihub.value:\n logger.info('Removing %s because no kojihub was specified', plugin)\n self.dj.remove_plugin(phase, plugin)\n continue\n\n if self.spec.prefer_schema1_digest.value is not None:\n self.dj.dock_json_set_arg(phase, 'pulp_pull',\n 'expect_v2schema2',\n not self.spec.prefer_schema1_digest.value)", "response": "Render the pulp pull request."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrendering the pulp sync plugin.", "response": "def render_pulp_sync(self):\n \"\"\"\n If a pulp registry is specified, use the pulp plugin as well as the\n delete_from_registry to delete the image after sync\n \"\"\"\n if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',\n 'pulp_sync'):\n return\n\n pulp_registry = self.spec.pulp_registry.value\n\n # Find which registry to use\n docker_registry = None\n registry_secret = None\n registries = zip_longest(self.spec.registry_uris.value,\n self.spec.registry_secrets.value)\n for registry, secret in registries:\n if registry.version == 'v2':\n # First specified v2 registry is the one we'll tell pulp\n # to sync from. Keep the http prefix -- pulp wants it.\n docker_registry = registry.uri\n registry_secret = secret\n logger.info(\"using docker v2 registry %s for pulp_sync\",\n docker_registry)\n break\n\n if pulp_registry and docker_registry:\n self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',\n 'pulp_registry_name', pulp_registry)\n\n self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',\n 'docker_registry', docker_registry)\n\n if registry_secret:\n self.set_secret_for_plugin(registry_secret,\n plugin=('postbuild_plugins',\n 'pulp_sync',\n 'registry_secret_path'))\n\n # Verify we have a pulp secret\n if self.spec.pulp_secret.value is None:\n raise OsbsValidationException(\"Pulp registry specified \"\n \"but no auth config\")\n\n source_registry = self.spec.source_registry_uri.value\n perform_delete = (source_registry is None or\n source_registry.docker_uri != registry.docker_uri)\n if perform_delete:\n push_conf = self.dj.dock_json_get_plugin_conf('exit_plugins',\n 'delete_from_registry')\n args = push_conf.setdefault('args', {})\n delete_registries = args.setdefault('registries', {})\n placeholder = '{{REGISTRY_URI}}'\n\n # use passed in params like 'insecure' if available\n if placeholder in delete_registries:\n regdict = delete_registries[placeholder].copy()\n del delete_registries[placeholder]\n else:\n regdict = {}\n\n if registry_secret:\n regdict['secret'] = \\\n os.path.join(SECRETS_PATH, registry_secret)\n # tag_and_push configured the registry secret, no neet to set it again\n\n delete_registries[docker_registry] = regdict\n\n self.dj.dock_json_set_arg('exit_plugins', 'delete_from_registry',\n 'registries', delete_registries)\n else:\n logger.info(\"removing delete_from_registry from request, \"\n \"source and target registry are identical\")\n self.dj.remove_plugin(\"exit_plugins\", \"delete_from_registry\")\n else:\n # If no pulp registry is specified, don't run the pulp plugin\n logger.info(\"removing pulp_sync+delete_from_registry from request, \"\n \"requires pulp_registry and a v2 registry\")\n self.dj.remove_plugin(\"postbuild_plugins\", \"pulp_sync\")\n self.dj.remove_plugin(\"exit_plugins\", \"delete_from_registry\")"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrendering the pulp_tag plugin.", "response": "def render_pulp_tag(self):\n \"\"\"\n Configure the pulp_tag plugin.\n \"\"\"\n if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',\n 'pulp_tag'):\n return\n\n pulp_registry = self.spec.pulp_registry.value\n if pulp_registry:\n self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_tag',\n 'pulp_registry_name', pulp_registry)\n\n # Verify we have either a secret or username/password\n if self.spec.pulp_secret.value is None:\n conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',\n 'pulp_tag')\n args = conf.get('args', {})\n if 'username' not in args:\n raise OsbsValidationException(\"Pulp registry specified \"\n \"but no auth config\")\n else:\n # If no pulp registry is specified, don't run the pulp plugin\n logger.info(\"removing pulp_tag from request, \"\n \"requires pulp_registry\")\n self.dj.remove_plugin(\"postbuild_plugins\", \"pulp_tag\")"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrender the group_manifests plugin.", "response": "def render_group_manifests(self):\n \"\"\"\n Configure the group_manifests plugin. Group is always set to false for now.\n \"\"\"\n if not self.dj.dock_json_has_plugin_conf('postbuild_plugins', 'group_manifests'):\n return\n\n push_conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',\n 'group_manifests')\n args = push_conf.setdefault('args', {})\n # modify registries in place\n registries = args.setdefault('registries', {})\n placeholder = '{{REGISTRY_URI}}'\n\n if placeholder in registries:\n for registry, secret in zip_longest(self.spec.registry_uris.value,\n self.spec.registry_secrets.value):\n if not registry.uri:\n continue\n regdict = registries[placeholder].copy()\n regdict['version'] = registry.version\n if secret:\n regdict['secret'] = os.path.join(SECRETS_PATH, secret)\n registries[registry.docker_uri] = regdict\n del registries[placeholder]\n\n self.dj.dock_json_set_arg('postbuild_plugins', 'group_manifests',\n 'group', self.spec.group_manifests.value)\n goarch = {}\n for platform, architecture in self.platform_descriptors.items():\n goarch[platform] = architecture['architecture']\n self.dj.dock_json_set_arg('postbuild_plugins', 'group_manifests',\n 'goarch', goarch)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef render_import_image(self, use_auth=None):\n # import_image is a multi-phase plugin\n phases = ('postbuild_plugins', 'exit_plugins')\n plugin = 'import_image'\n\n for phase in phases:\n if self.spec.imagestream_name.value is None or self.spec.imagestream_url.value is None:\n logger.info(\"removing %s from request, \"\n \"registry or repo url is not defined\", plugin)\n self.dj.remove_plugin(phase, plugin)\n continue\n\n if self.dj.dock_json_has_plugin_conf(phase, plugin):\n self.dj.dock_json_set_arg(phase, plugin, 'imagestream',\n self.spec.imagestream_name.value)\n self.dj.dock_json_set_arg(phase, plugin, 'docker_image_repo',\n self.spec.imagestream_url.value)\n self.dj.dock_json_set_arg(phase, plugin, 'url',\n self.spec.builder_openshift_url.value)\n self.dj.dock_json_set_arg(phase, plugin, 'build_json_dir',\n self.spec.builder_build_json_dir.value)\n\n use_auth = self.spec.use_auth.value\n if use_auth is not None:\n self.dj.dock_json_set_arg(phase, plugin, 'use_auth', use_auth)\n\n if self.spec.imagestream_insecure_registry.value:\n self.dj.dock_json_set_arg(phase, plugin, 'insecure_registry', True)", "response": "Configure the import_image plugin"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nrenders site - specific customizations.", "response": "def render_customizations(self):\n \"\"\"\n Customize prod_inner for site specific customizations\n \"\"\"\n\n disable_plugins = self.customize_conf.get('disable_plugins', [])\n if not disable_plugins:\n logger.debug(\"No site-specific plugins to disable\")\n else:\n for plugin_dict in disable_plugins:\n try:\n self.dj.remove_plugin(\n plugin_dict['plugin_type'],\n plugin_dict['plugin_name']\n )\n logger.debug(\n \"site-specific plugin disabled -> Type:{} Name:{}\".format(\n plugin_dict['plugin_type'],\n plugin_dict['plugin_name']\n )\n )\n except KeyError:\n # Malformed config\n logger.debug(\"Invalid custom configuration found for disable_plugins\")\n\n enable_plugins = self.customize_conf.get('enable_plugins', [])\n if not enable_plugins:\n logger.debug(\"No site-specific plugins to enable\")\n else:\n for plugin_dict in enable_plugins:\n try:\n self.dj.add_plugin(\n plugin_dict['plugin_type'],\n plugin_dict['plugin_name'],\n plugin_dict['plugin_args']\n )\n logger.debug(\n \"site-specific plugin enabled -> Type:{} Name:{} Args: {}\".format(\n plugin_dict['plugin_type'],\n plugin_dict['plugin_name'],\n plugin_dict['plugin_args']\n )\n )\n except KeyError:\n # Malformed config\n logger.debug(\"Invalid custom configuration found for enable_plugins\")"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the Build / BuildConfig object name", "response": "def render_name(self, name, image_tag, platform):\n \"\"\"Sets the Build/BuildConfig object name\"\"\"\n\n if self.scratch or self.isolated:\n name = image_tag\n # Platform name may contain characters not allowed by OpenShift.\n if platform:\n platform_suffix = '-{}'.format(platform)\n if name.endswith(platform_suffix):\n name = name[:-len(platform_suffix)]\n\n _, salt, timestamp = name.rsplit('-', 2)\n\n if self.scratch:\n name = 'scratch-{}-{}'.format(salt, timestamp)\n elif self.isolated:\n name = 'isolated-{}-{}'.format(salt, timestamp)\n\n # !IMPORTANT! can't be too long: https://github.com/openshift/origin/issues/733\n self.template['metadata']['name'] = name"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_terminal_size():\n try:\n rows, columns = subprocess.check_output(['stty', 'size']).split()\n except subprocess.CalledProcessError:\n # not attached to terminal\n logger.info(\"not attached to terminal\")\n return 0, 0\n logger.debug(\"console size is %s %s\", rows, columns)\n return int(rows), int(columns)", "response": "get size of console"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget size of longest value in specific column", "response": "def _longest_val_in_column(self, col):\n \"\"\"\n get size of longest value in specific column\n\n :param col: str, column name\n :return int\n \"\"\"\n try:\n # +2 is for implicit separator\n return max([len(x[col]) for x in self.table if x[col]]) + 2\n except KeyError:\n logger.error(\"there is no column %r\", col)\n raise"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize all values based on input .", "response": "def _init(self):\n \"\"\"\n initialize all values based on provided input\n\n :return: None\n \"\"\"\n self.col_count = len(self.col_list)\n # list of lengths of longest entries in columns\n self.col_longest = self.get_all_longest_col_lengths()\n self.data_length = sum(self.col_longest.values())\n\n if self.terminal_width > 0:\n # free space is space which should be equeally distributed for all columns\n # self.terminal_width -- terminal is our canvas\n # - self.data_length -- substract length of content (the actual data)\n # - self.col_count + 1 -- table lines are not part of free space, their width is\n # (number of columns - 1)\n self.total_free_space = (self.terminal_width - self.data_length) - self.col_count + 1\n if self.total_free_space <= 0:\n self.total_free_space = None\n else:\n self.default_column_space = self.total_free_space // self.col_count\n self.default_column_space_remainder = self.total_free_space % self.col_count\n logger.debug(\"total free space: %d, column space: %d, remainder: %d, columns: %d\",\n self.total_free_space, self.default_column_space,\n self.default_column_space_remainder, self.col_count)\n else:\n self.total_free_space = None"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _count_sizes(self):\n format_list = []\n header_sepa_format_list = []\n # actual widths of columns\n self.col_widths = {}\n\n for col in self.col_list:\n col_length = self.col_longest[col]\n col_width = col_length + self._separate()\n # -2 is for implicit separator -- spaces around\n format_list.append(\" {%s:%d} \" % (col, col_width - 2))\n header_sepa_format_list.append(\"{%s:%d}\" % (col, col_width))\n self.col_widths[col] = col_width\n\n logger.debug(\"column widths %s\", self.col_widths)\n\n self.format_str = \"|\".join(format_list)\n\n self.header_format_str = \"+\".join(header_sepa_format_list)\n self.header_data = {}\n for k in self.col_widths:\n self.header_data[k] = \"-\" * self.col_widths[k]", "response": "Counts the number of columns needed to display whole table."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_all_longest_col_lengths(self):\n response = {}\n for col in self.col_list:\n response[col] = self._longest_val_in_column(col)\n return response", "response": "iterate over all columns and get their longest values"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _separate(self):\n if self.total_free_space is None:\n return 0\n else:\n sepa = self.default_column_space\n # we need to distribute remainders\n if self.default_column_space_remainder > 0:\n sepa += 1\n self.default_column_space_remainder -= 1\n logger.debug(\"remainder: %d, separator: %d\",\n self.default_column_space_remainder, sepa)\n return sepa", "response": "get a width of separator for current column"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef render(self):\n print(self.format_str.format(**self.header), file=sys.stderr)\n print(self.header_format_str.format(**self.header_data), file=sys.stderr)\n for row in self.data:\n print(self.format_str.format(**row))", "response": "print provided table\n\n :return: None"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _validate_source_data(self):\n\n try:\n jsonschema.validate(self._buffer, self._schema)\n except jsonschema.ValidationError as e:\n raise ValidationError(e)", "response": "Validate the source data."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef to_table_data(self):\n\n self._validate_source_data()\n self._loader.inc_table_count()\n\n yield TableData(\n self._make_table_name(),\n [\"key\", \"value\"],\n [record for record in self._buffer.items()],\n dp_extractor=self._loader.dp_extractor,\n type_hints=self._extract_type_hints(),\n )", "response": "Yields a generator that yields TableData objects."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nyields a generator that yields TableData objects.", "response": "def to_table_data(self):\n \"\"\"\n :raises ValueError:\n :raises pytablereader.error.ValidationError:\n \"\"\"\n\n self._validate_source_data()\n\n for table_key, json_records in six.iteritems(self._buffer):\n attr_name_set = set()\n for json_record in json_records:\n attr_name_set = attr_name_set.union(six.viewkeys(json_record))\n headers = sorted(attr_name_set)\n\n self._loader.inc_table_count()\n self._table_key = table_key\n\n yield TableData(\n self._make_table_name(),\n headers,\n json_records,\n dp_extractor=self._loader.dp_extractor,\n type_hints=self._extract_type_hints(headers),\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_table_data(self):\n\n self._validate_source_data()\n\n for table_key, json_records in six.iteritems(self._buffer):\n headers = sorted(six.viewkeys(json_records))\n\n self._loader.inc_table_count()\n self._table_key = table_key\n\n yield TableData(\n self._make_table_name(),\n headers,\n zip(*[json_records.get(header) for header in headers]),\n dp_extractor=self._loader.dp_extractor,\n type_hints=self._extract_type_hints(headers),\n )", "response": "Yields a generator that yields TableData objects."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef to_table_data(self):\n\n self._validate_source_data()\n\n for table_key, json_records in six.iteritems(self._buffer):\n self._loader.inc_table_count()\n self._table_key = table_key\n\n yield TableData(\n self._make_table_name(),\n [\"key\", \"value\"],\n [record for record in json_records.items()],\n dp_extractor=self._loader.dp_extractor,\n type_hints=self._extract_type_hints(),\n )", "response": "Yields a generator that yields TableData objects."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_params(self, **kwargs):\n\n # Here we cater to the koji \"scratch\" build type, this will disable\n # all plugins that might cause importing of data to koji\n self.scratch = kwargs.get('scratch')\n # When true, it indicates build was automatically started by\n # OpenShift via a trigger, for instance ImageChangeTrigger\n self.is_auto = kwargs.pop('is_auto', False)\n # An isolated build is meant to patch a certain release and not\n # update transient tags in container registry\n self.isolated = kwargs.get('isolated')\n\n self.osbs_api = kwargs.pop('osbs_api', None)\n self.validate_build_variation()\n\n self.base_image = kwargs.get('base_image')\n self.platform_node_selector = kwargs.get('platform_node_selector', {})\n self.scratch_build_node_selector = kwargs.get('scratch_build_node_selector', {})\n self.explicit_build_node_selector = kwargs.get('explicit_build_node_selector', {})\n self.auto_build_node_selector = kwargs.get('auto_build_node_selector', {})\n self.isolated_build_node_selector = kwargs.get('isolated_build_node_selector', {})\n\n logger.debug(\"now setting params '%s' for user_params\", kwargs)\n self.user_params.set_params(**kwargs)\n\n self.source_registry = None\n self.organization = None", "response": "set parameters in the user parameters\n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset data from reactor config", "response": "def set_data_from_reactor_config(self):\n \"\"\"\n Sets data from reactor config\n \"\"\"\n reactor_config_override = self.user_params.reactor_config_override.value\n reactor_config_map = self.user_params.reactor_config_map.value\n data = None\n\n if reactor_config_override:\n data = reactor_config_override\n elif reactor_config_map:\n config_map = self.osbs_api.get_config_map(reactor_config_map)\n data = config_map.get_data_by_key('config.yaml')\n\n if not data:\n if self.user_params.flatpak.value:\n raise OsbsValidationException(\"flatpak_base_image must be provided\")\n else:\n return\n\n source_registry_key = 'source_registry'\n registry_organization_key = 'registries_organization'\n req_secrets_key = 'required_secrets'\n token_secrets_key = 'worker_token_secrets'\n flatpak_key = 'flatpak'\n flatpak_base_image_key = 'base_image'\n\n if source_registry_key in data:\n self.source_registry = data[source_registry_key]\n if registry_organization_key in data:\n self.organization = data[registry_organization_key]\n\n if self.user_params.flatpak.value:\n flatpack_base_image = data.get(flatpak_key, {}).get(flatpak_base_image_key, None)\n if flatpack_base_image:\n self.base_image = flatpack_base_image\n self.user_params.base_image.value = flatpack_base_image\n else:\n raise OsbsValidationException(\"flatpak_base_image must be provided\")\n\n required_secrets = data.get(req_secrets_key, [])\n token_secrets = data.get(token_secrets_key, [])\n self._set_required_secrets(required_secrets, token_secrets)"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _set_required_secrets(self, required_secrets, token_secrets):\n if self.user_params.build_type.value == BUILD_TYPE_ORCHESTRATOR:\n required_secrets += token_secrets\n\n if not required_secrets:\n return\n\n secrets = self.template['spec']['strategy']['customStrategy'].setdefault('secrets', [])\n existing = set(secret_mount['secretSource']['name'] for secret_mount in secrets)\n required_secrets = set(required_secrets)\n\n already_set = required_secrets.intersection(existing)\n if already_set:\n logger.debug(\"secrets %s are already set\", already_set)\n\n for secret in required_secrets - existing:\n secret_path = os.path.join(SECRETS_PATH, secret)\n logger.info(\"Configuring %s secret at %s\", secret, secret_path)\n\n secrets.append({\n 'secretSource': {\n 'name': secret,\n },\n 'mountPath': secret_path,\n })", "response": "Sets the required secrets for the current user."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef remove_plugin(self, phase, name, reason=None):\n for p in self.template[phase]:\n if p.get('name') == name:\n self.template[phase].remove(p)\n if reason:\n logger.info('Removing {}:{}, {}'.format(phase, name, reason))\n break", "response": "Remove a plugin from the config"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_plugin(self, phase, name, args, reason=None):\n plugin_modified = False\n\n for plugin in self.template[phase]:\n if plugin['name'] == name:\n plugin['args'] = args\n plugin_modified = True\n\n if not plugin_modified:\n self.template[phase].append({\"name\": name, \"args\": args})\n if reason:\n logger.info('{}:{} with args {}, {}'.format(phase, name, args, reason))", "response": "Add a new plugin to the config."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the configuration for a plugin.", "response": "def get_plugin_conf(self, phase, name):\n \"\"\"\n Return the configuration for a plugin.\n\n Raises KeyError if there are no plugins of that type.\n Raises IndexError if the named plugin is not listed.\n \"\"\"\n match = [x for x in self.template[phase] if x.get('name') == name]\n return match[0]"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef has_plugin_conf(self, phase, name):\n try:\n self.get_plugin_conf(phase, name)\n return True\n except (KeyError, IndexError):\n return False", "response": "Check whether a plugin is configured."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef adjust_for_scratch(self):\n if self.user_params.scratch.value:\n remove_plugins = [\n (\"prebuild_plugins\", \"koji_parent\"),\n (\"postbuild_plugins\", \"compress\"), # required only to make an archive for Koji\n (\"postbuild_plugins\", \"pulp_pull\"), # required only to make an archive for Koji\n (\"postbuild_plugins\", \"compare_components\"),\n (\"postbuild_plugins\", \"import_image\"),\n (\"exit_plugins\", \"koji_promote\"),\n (\"exit_plugins\", \"koji_tag_build\"),\n (\"exit_plugins\", \"import_image\"),\n (\"prebuild_plugins\", \"check_and_set_rebuild\"),\n (\"prebuild_plugins\", \"stop_autorebuild_if_disabled\")\n ]\n\n if not self.has_tag_suffixes_placeholder():\n remove_plugins.append((\"postbuild_plugins\", \"tag_from_config\"))\n\n for when, which in remove_plugins:\n self.pt.remove_plugin(when, which, 'removed from scratch build request')", "response": "Remove certain plugins in order to handle the scratch build scenario."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nremoving certain plugins in order to handle the isolated build scenario.", "response": "def adjust_for_isolated(self):\n \"\"\"\n Remove certain plugins in order to handle the \"isolated build\"\n scenario.\n \"\"\"\n if self.user_params.isolated.value:\n remove_plugins = [\n (\"prebuild_plugins\", \"check_and_set_rebuild\"),\n (\"prebuild_plugins\", \"stop_autorebuild_if_disabled\")\n ]\n\n for when, which in remove_plugins:\n self.pt.remove_plugin(when, which, 'removed from isolated build request')"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef adjust_for_flatpak(self):\n if self.user_params.flatpak.value:\n remove_plugins = [\n (\"prebuild_plugins\", \"resolve_composes\"),\n # We'll extract the filesystem anyways for a Flatpak instead of exporting\n # the docker image directly, so squash just slows things down.\n (\"prepublish_plugins\", \"squash\"),\n # Pulp can't currently handle Flatpaks, which are OCI images\n (\"postbuild_plugins\", \"pulp_push\"),\n (\"postbuild_plugins\", \"pulp_tag\"),\n (\"postbuild_plugins\", \"pulp_sync\"),\n (\"exit_plugins\", \"pulp_publish\"),\n (\"exit_plugins\", \"pulp_pull\"),\n # delete_from_registry is used for deleting builds from the temporary registry\n # that pulp_sync mirrors from.\n (\"exit_plugins\", \"delete_from_registry\"),\n ]\n for when, which in remove_plugins:\n self.pt.remove_plugin(when, which, 'not needed for flatpak build')", "response": "Adjusts the internal state of the internal state of the internal state of the Flatpaks."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nrenders customizations for site user specified customizations.", "response": "def render_customizations(self):\n \"\"\"\n Customize template for site user specified customizations\n \"\"\"\n disable_plugins = self.pt.customize_conf.get('disable_plugins', [])\n if not disable_plugins:\n logger.debug('No site-user specified plugins to disable')\n else:\n for plugin in disable_plugins:\n try:\n self.pt.remove_plugin(plugin['plugin_type'], plugin['plugin_name'],\n 'disabled at user request')\n except KeyError:\n # Malformed config\n logger.info('Invalid custom configuration found for disable_plugins')\n\n enable_plugins = self.pt.customize_conf.get('enable_plugins', [])\n if not enable_plugins:\n logger.debug('No site-user specified plugins to enable\"')\n else:\n for plugin in enable_plugins:\n try:\n msg = 'enabled at user request'\n self.pt.add_plugin(plugin['plugin_type'], plugin['plugin_name'],\n plugin['plugin_args'], msg)\n except KeyError:\n # Malformed config\n logger.info('Invalid custom configuration found for enable_plugins')"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrender koji if there is yum repo in user params don t pick stuff from koji", "response": "def render_koji(self):\n \"\"\"\n if there is yum repo in user params, don't pick stuff from koji\n \"\"\"\n phase = 'prebuild_plugins'\n plugin = 'koji'\n if not self.pt.has_plugin_conf(phase, plugin):\n return\n\n if self.user_params.yum_repourls.value:\n self.pt.remove_plugin(phase, plugin, 'there is a yum repo user parameter')\n elif not self.pt.set_plugin_arg_valid(phase, plugin, \"target\",\n self.user_params.koji_target.value):\n self.pt.remove_plugin(phase, plugin, 'no koji target supplied in user parameters')"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef render_check_and_set_platforms(self):\n phase = 'prebuild_plugins'\n plugin = 'check_and_set_platforms'\n if not self.pt.has_plugin_conf(phase, plugin):\n return\n\n if self.user_params.koji_target.value:\n self.pt.set_plugin_arg(phase, plugin, \"koji_target\",\n self.user_params.koji_target.value)", "response": "If the check_and_set_platforms plugin is present, configure it"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef render_import_image(self, use_auth=None):\n # import_image is a multi-phase plugin\n if self.user_params.imagestream_name.value is None:\n self.pt.remove_plugin('exit_plugins', 'import_image',\n 'imagestream not in user parameters')\n elif self.pt.has_plugin_conf('exit_plugins', 'import_image'):\n self.pt.set_plugin_arg('exit_plugins', 'import_image', 'imagestream',\n self.user_params.imagestream_name.value)", "response": "Render the import_image plugin"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_user(self, username=\"~\"):\n url = self._build_url(\"users/%s/\" % username, _prepend_namespace=False)\n response = self._get(url)\n check_response(response)\n return response", "response": "get user info about user"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_build(self, build_json):\n url = self._build_url(\"builds/\")\n logger.debug(build_json)\n return self._post(url, data=json.dumps(build_json),\n headers={\"Content-Type\": \"application/json\"})", "response": "Creates a build for the given build_json."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_all_build_configs_by_labels(self, label_selectors):\n labels = ['%s=%s' % (field, value) for field, value in label_selectors]\n labels = ','.join(labels)\n url = self._build_url(\"buildconfigs/\", labelSelector=labels)\n return self._get(url).json()['items']", "response": "Returns all builds matching a given set of label selectors."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_build_config_by_labels(self, label_selectors):\n items = self.get_all_build_configs_by_labels(label_selectors)\n\n if not items:\n raise OsbsException(\n \"Build config not found for labels: %r\" %\n (label_selectors, ))\n if len(items) > 1:\n raise OsbsException(\n \"More than one build config found for labels: %r\" %\n (label_selectors, ))\n\n return items[0]", "response": "Returns a build config matching the given label selectors. This method will raise OsbsException if more than one build config is found."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a build config matching the given label selectors filtering against a predetermined value.", "response": "def get_build_config_by_labels_filtered(self, label_selectors, filter_key, filter_value):\n \"\"\"\n Returns a build config matching the given label selectors, filtering against\n another predetermined value. This method will raise OsbsException\n if not exactly one build config is found after filtering.\n \"\"\"\n items = self.get_all_build_configs_by_labels(label_selectors)\n\n if filter_value is not None:\n build_configs = []\n for build_config in items:\n match_value = graceful_chain_get(build_config, *filter_key.split('.'))\n if filter_value == match_value:\n build_configs.append(build_config)\n items = build_configs\n\n if not items:\n raise OsbsException(\n \"Build config not found for labels: %r\" %\n (label_selectors, ))\n if len(items) > 1:\n raise OsbsException(\n \"More than one build config found for labels: %r\" %\n (label_selectors, ))\n return items[0]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a build config for the specified build_config_json.", "response": "def create_build_config(self, build_config_json):\n \"\"\"\n :return:\n \"\"\"\n url = self._build_url(\"buildconfigs/\")\n return self._post(url, data=build_config_json,\n headers={\"Content-Type\": \"application/json\"})"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstream logs from build_id build_id.", "response": "def stream_logs(self, build_id):\n \"\"\"\n stream logs from build\n\n :param build_id: str\n :return: iterator\n \"\"\"\n kwargs = {'follow': 1}\n\n # If connection is closed within this many seconds, give up:\n min_idle_timeout = 60\n\n # Stream logs, but be careful of the connection closing\n # due to idle timeout. In that case, try again until the\n # call returns more quickly than a reasonable timeout\n # would be set to.\n last_activity = time.time()\n while True:\n buildlogs_url = self._build_url(\"builds/%s/log/\" % build_id,\n **kwargs)\n try:\n response = self._get(buildlogs_url, stream=1,\n headers={'Connection': 'close'})\n check_response(response)\n\n for line in response.iter_lines():\n last_activity = time.time()\n yield line\n # NOTE1: If self._get causes ChunkedEncodingError, ConnectionError,\n # or IncompleteRead to be raised, they'll be wrapped in\n # OsbsNetworkException or OsbsException\n # NOTE2: If iter_lines causes ChunkedEncodingError\n # or IncompleteRead to be raised, it'll simply be silenced.\n # NOTE3: An exception may be raised from\n # check_response(). In this case, exception will be\n # wrapped in OsbsException or OsbsNetworkException,\n # inspect cause to detect ConnectionError.\n except OsbsException as exc:\n if not isinstance(exc.cause, ConnectionError):\n raise\n\n idle = time.time() - last_activity\n logger.debug(\"connection closed after %ds\", idle)\n if idle < min_idle_timeout:\n # Finish output\n return\n\n since = int(idle - 1)\n logger.debug(\"fetching logs starting from %ds ago\", since)\n kwargs['sinceSeconds'] = since"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nstream logs from build", "response": "def logs(self, build_id, follow=False, build_json=None, wait_if_missing=False):\n \"\"\"\n provide logs from build\n\n :param build_id: str\n :param follow: bool, fetch logs as they come?\n :param build_json: dict, to save one get-build query\n :param wait_if_missing: bool, if build doesn't exist, wait\n :return: None, str or iterator\n \"\"\"\n # does build exist?\n try:\n build_json = build_json or self.get_build(build_id).json()\n except OsbsResponseException as ex:\n if ex.status_code == 404:\n if not wait_if_missing:\n raise OsbsException(\"Build '%s' doesn't exist.\" % build_id)\n else:\n raise\n\n if follow or wait_if_missing:\n build_json = self.wait_for_build_to_get_scheduled(build_id)\n\n br = BuildResponse(build_json)\n\n # When build is in new or pending state, openshift responds with 500\n if br.is_pending():\n return\n\n if follow:\n return self.stream_logs(build_id)\n\n buildlogs_url = self._build_url(\"builds/%s/log/\" % build_id)\n response = self._get(buildlogs_url, headers={'Connection': 'close'})\n check_response(response)\n return response.content"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nlist builds for a given Koji Task ID and optionally a given set of labels.", "response": "def list_builds(self, build_config_id=None, koji_task_id=None,\n field_selector=None, labels=None):\n \"\"\"\n List builds matching criteria\n\n :param build_config_id: str, only list builds created from BuildConfig\n :param koji_task_id: str, only list builds for Koji Task ID\n :param field_selector: str, field selector for query\n :return: HttpResponse\n \"\"\"\n query = {}\n selector = '{key}={value}'\n\n label = {}\n if labels is not None:\n label.update(labels)\n\n if build_config_id is not None:\n label['buildconfig'] = build_config_id\n\n if koji_task_id is not None:\n label['koji-task-id'] = str(koji_task_id)\n\n if label:\n query['labelSelector'] = ','.join([selector.format(key=key,\n value=value)\n for key, value in label.items()])\n\n if field_selector is not None:\n query['fieldSelector'] = field_selector\n url = self._build_url(\"builds/\", **query)\n return self._get(url)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a resource quota for a resource.", "response": "def create_resource_quota(self, name, quota_json):\n \"\"\"\n Prevent builds being scheduled and wait for running builds to finish.\n\n :return:\n \"\"\"\n\n url = self._build_k8s_url(\"resourcequotas/\")\n response = self._post(url, data=json.dumps(quota_json),\n headers={\"Content-Type\": \"application/json\"})\n if response.status_code == http_client.CONFLICT:\n url = self._build_k8s_url(\"resourcequotas/%s\" % name)\n response = self._put(url, data=json.dumps(quota_json),\n headers={\"Content-Type\": \"application/json\"})\n\n check_response(response)\n return response"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwaits for a build to finish", "response": "def wait(self, build_id, states):\n \"\"\"\n :param build_id: wait for build to finish\n\n :return:\n \"\"\"\n logger.info(\"watching build '%s'\", build_id)\n for changetype, obj in self.watch_resource(\"builds\", build_id):\n try:\n obj_name = obj[\"metadata\"][\"name\"]\n except KeyError:\n logger.error(\"'object' doesn't have any name\")\n continue\n try:\n obj_status = obj[\"status\"][\"phase\"]\n except KeyError:\n logger.error(\"'object' doesn't have any status\")\n continue\n else:\n obj_status_lower = obj_status.lower()\n logger.info(\"object has changed: '%s', status: '%s', name: '%s'\",\n changetype, obj_status, obj_name)\n if obj_name == build_id:\n logger.info(\"matching build found\")\n logger.debug(\"is %s in %s?\", repr(obj_status_lower), states)\n if obj_status_lower in states:\n logger.debug(\"Yes, build is in the state I'm waiting for.\")\n return obj\n else:\n logger.debug(\"No, build is not in the state I'm \"\n \"waiting for.\")\n else:\n logger.info(\"The build %r isn't me %r\", obj_name, build_id)\n\n # I'm not sure how we can end up here since there are two possible scenarios:\n # 1. our object was found and we are returning in the loop\n # 2. our object was not found and we keep waiting (in the loop)\n # Therefore, let's raise here\n logger.warning(\"build '%s' was not found during wait\", build_id)\n raise OsbsWatchBuildNotFound(\"build '%s' was not found and response stream ended\" %\n build_id)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef adjust_attributes_on_object(self, collection, name, things, values, how):\n url = self._build_url(\"%s/%s\" % (collection, name))\n response = self._get(url)\n logger.debug(\"before modification: %s\", response.content)\n build_json = response.json()\n how(build_json['metadata'], things, values)\n response = self._put(url, data=json.dumps(build_json), use_json=True)\n check_response(response)\n return response", "response": "adjusts labels or annotations on object name"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef update_annotations_on_build(self, build_id, annotations):\n return self.adjust_attributes_on_object('builds', build_id,\n 'annotations', annotations,\n self._update_metadata_things)", "response": "set annotations on build object\n "} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nimports an image into a ImageStream.", "response": "def import_image(self, name, stream_import, tags=None):\n \"\"\"\n Import image tags from a Docker registry into an ImageStream\n\n :return: bool, whether tags were imported\n \"\"\"\n\n # Get the JSON for the ImageStream\n imagestream_json = self.get_image_stream(name).json()\n logger.debug(\"imagestream: %r\", imagestream_json)\n\n if 'dockerImageRepository' in imagestream_json.get('spec', {}):\n logger.debug(\"Removing 'dockerImageRepository' from ImageStream %s\", name)\n source_repo = imagestream_json['spec'].pop('dockerImageRepository')\n imagestream_json['metadata']['annotations'][ANNOTATION_SOURCE_REPO] = source_repo\n imagestream_json = self.update_image_stream(name, imagestream_json).json()\n\n # Note the tags before import\n oldtags = imagestream_json.get('status', {}).get('tags', [])\n logger.debug(\"tags before import: %r\", oldtags)\n\n stream_import['metadata']['name'] = name\n stream_import['spec']['images'] = []\n tags_set = set(tags) if tags else set()\n for tag in imagestream_json.get('spec', {}).get('tags', []):\n if tags_set and tag['name'] not in tags_set:\n continue\n\n image_import = {\n 'from': tag['from'],\n 'to': {'name': tag['name']},\n 'importPolicy': tag.get('importPolicy'),\n 'referencePolicy': tag.get('referencePolicy'),\n }\n stream_import['spec']['images'].append(image_import)\n\n if not stream_import['spec']['images']:\n logger.debug('No tags to import')\n return False\n\n import_url = self._build_url(\"imagestreamimports/\")\n import_response = self._post(import_url, data=json.dumps(stream_import),\n use_json=True)\n self._check_import_image_response(import_response)\n\n new_tags = [\n image['tag']\n for image in import_response.json().get('status', {}).get('images', [])]\n logger.debug(\"tags after import: %r\", new_tags)\n\n return True"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nimport tags from a Docker registry into an ImageStream.", "response": "def import_image_tags(self, name, stream_import, tags, repository, insecure):\n \"\"\"\n Import image tags from a Docker registry into an ImageStream\n\n :return: bool, whether tags were imported\n \"\"\"\n\n # Get the JSON for the ImageStream\n imagestream_json = self.get_image_stream(name).json()\n logger.debug(\"imagestream: %r\", imagestream_json)\n changed = False\n\n # existence of dockerImageRepository is limiting how many tags are updated\n if 'dockerImageRepository' in imagestream_json.get('spec', {}):\n logger.debug(\"Removing 'dockerImageRepository' from ImageStream %s\", name)\n imagestream_json['spec'].pop('dockerImageRepository')\n changed = True\n all_annotations = imagestream_json.get('metadata', {}).get('annotations', {})\n # remove annotations about registry, since method will get them as arguments\n for annotation in ANNOTATION_SOURCE_REPO, ANNOTATION_INSECURE_REPO:\n if annotation in all_annotations:\n imagestream_json['metadata']['annotations'].pop(annotation)\n changed = True\n\n if changed:\n imagestream_json = self.update_image_stream(name, imagestream_json).json()\n\n # Note the tags before import\n oldtags = imagestream_json.get('status', {}).get('tags', [])\n logger.debug(\"tags before import: %r\", oldtags)\n\n stream_import['metadata']['name'] = name\n stream_import['spec']['images'] = []\n tags_set = set(tags) if tags else set()\n\n if not tags_set:\n logger.debug('No tags to import')\n return False\n\n for tag in tags_set:\n image_import = {\n 'from': {\"kind\": \"DockerImage\",\n \"name\": '{}:{}'.format(repository, tag)},\n 'to': {'name': tag},\n 'importPolicy': {'insecure': insecure},\n # referencePolicy will default to \"type: source\"\n # so we don't have to explicitly set it\n }\n stream_import['spec']['images'].append(image_import)\n\n import_url = self._build_url(\"imagestreamimports/\")\n import_response = self._post(import_url, data=json.dumps(stream_import),\n use_json=True)\n self._check_import_image_response(import_response)\n\n new_tags = [\n image['tag']\n for image in import_response.json().get('status', {}).get('images', [])]\n logger.debug(\"tags after import: %r\", new_tags)\n\n return True"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef str_on_2_unicode_on_3(s):\n\n if not PY3:\n return str(s)\n else: # 3+\n if not isinstance(s, str):\n return str(s, encoding=\"utf-8\")\n return s", "response": "str on 2 unicode on 3"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load(self):\n\n formatter = JsonLinesTableFormatter(self.load_dict())\n formatter.accept(self)\n\n return formatter.to_table_data()", "response": "Loads the tabular data from a Line - Delimited JSON file."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nloads the table data from a Google Spreadsheet.", "response": "def load(self):\n \"\"\"\n Load table data from a Google Spreadsheet.\n\n This method consider :py:attr:`.source` as a path to the\n credential JSON file to access Google Sheets API.\n\n The method automatically search the header row start from\n :py:attr:`.start_row`. The condition of the header row is that\n all of the columns have value (except empty columns).\n\n :return:\n Loaded table data. Return one |TableData| for each sheet in\n the workbook. The table name for data will be determined by\n :py:meth:`~.GoogleSheetsTableLoader.make_table_name`.\n :rtype: iterator of |TableData|\n :raises pytablereader.DataError:\n If the header row is not found.\n :raises pytablereader.OpenError:\n If the spread sheet not found.\n \"\"\"\n\n import gspread\n from oauth2client.service_account import ServiceAccountCredentials\n\n self._validate_table_name()\n self._validate_title()\n\n scope = [\"https://spreadsheets.google.com/feeds\", \"https://www.googleapis.com/auth/drive\"]\n credentials = ServiceAccountCredentials.from_json_keyfile_name(self.source, scope)\n\n gc = gspread.authorize(credentials)\n try:\n for worksheet in gc.open(self.title).worksheets():\n self._worksheet = worksheet\n self.__all_values = [row for row in worksheet.get_all_values()]\n\n if self._is_empty_sheet():\n continue\n\n try:\n self.__strip_empty_col()\n except ValueError:\n continue\n\n value_matrix = self.__all_values[self._get_start_row_idx() :]\n try:\n headers = value_matrix[0]\n rows = value_matrix[1:]\n except IndexError:\n continue\n\n self.inc_table_count()\n\n yield TableData(\n self.make_table_name(),\n headers,\n rows,\n dp_extractor=self.dp_extractor,\n type_hints=self._extract_type_hints(headers),\n )\n except gspread.exceptions.SpreadsheetNotFound:\n raise OpenError(\"spreadsheet '{}' not found\".format(self.title))\n except gspread.exceptions.APIError as e:\n raise APIError(e)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the log level of this module.", "response": "def set_log_level(log_level):\n \"\"\"\n Set logging level of this module. Using\n `logbook `__ module for logging.\n\n :param int log_level:\n One of the log level of\n `logbook `__.\n Disabled logging if ``log_level`` is ``logbook.NOTSET``.\n :raises LookupError: If ``log_level`` is an invalid value.\n \"\"\"\n\n if not LOGBOOK_INSTALLED:\n return\n\n # validate log level\n logbook.get_level_name(log_level)\n\n if log_level == logger.level:\n return\n\n if log_level == logbook.NOTSET:\n set_logger(is_enable=False)\n else:\n set_logger(is_enable=True)\n\n logger.level = log_level\n dataproperty.set_log_level(log_level)\n\n try:\n import simplesqlite\n\n simplesqlite.set_log_level(log_level)\n except ImportError:\n pass"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating the values of the buildconfig with values from new BuildConfig.", "response": "def buildconfig_update(orig, new, remove_nonexistent_keys=False):\n \"\"\"Performs update of given `orig` BuildConfig with values from `new` BuildConfig.\n Both BuildConfigs have to be represented as `dict`s.\n\n This function:\n - adds all key/value pairs to `orig` from `new` that are missing\n - replaces values in `orig` for keys that are in both\n - removes key/value pairs from `orig` for keys that are not in `new`,\n but only in dicts nested inside `strategy` key\n (see https://github.com/projectatomic/osbs-client/pull/273#issuecomment-148038314)\n \"\"\"\n if isinstance(orig, dict) and isinstance(new, dict):\n clean_triggers(orig, new)\n if remove_nonexistent_keys:\n missing = set(orig.keys()) - set(new.keys())\n for k in missing:\n orig.pop(k)\n for k, v in new.items():\n if k == 'strategy':\n remove_nonexistent_keys = True\n if isinstance(orig.get(k), dict) and isinstance(v, dict):\n buildconfig_update(orig[k], v, remove_nonexistent_keys)\n else:\n orig[k] = v"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef checkout_git_repo(git_url, target_dir=None, commit=None, retry_times=GIT_MAX_RETRIES,\n branch=None, depth=None):\n \"\"\"\n clone provided git repo to target_dir, optionally checkout provided commit\n yield the ClonedRepoData and delete the repo when finished\n\n :param git_url: str, git repo to clone\n :param target_dir: str, filesystem path where the repo should be cloned\n :param commit: str, commit to checkout, SHA-1 or ref\n :param retry_times: int, number of retries for git clone\n :param branch: str, optional branch of the commit, required if depth is provided\n :param depth: int, optional expected depth\n :return: str, int, commit ID of HEAD\n \"\"\"\n tmpdir = tempfile.mkdtemp()\n target_dir = target_dir or os.path.join(tmpdir, \"repo\")\n try:\n yield clone_git_repo(git_url, target_dir, commit, retry_times, branch, depth)\n finally:\n shutil.rmtree(tmpdir)", "response": "checkout a git repo to target_dir optionally checkout provided commit"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nclone a git repo to target_dir optionally checkout commit.", "response": "def clone_git_repo(git_url, target_dir=None, commit=None, retry_times=GIT_MAX_RETRIES, branch=None,\n depth=None):\n \"\"\"\n clone provided git repo to target_dir, optionally checkout provided commit\n\n :param git_url: str, git repo to clone\n :param target_dir: str, filesystem path where the repo should be cloned\n :param commit: str, commit to checkout, SHA-1 or ref\n :param retry_times: int, number of retries for git clone\n :param branch: str, optional branch of the commit, required if depth is provided\n :param depth: int, optional expected depth\n :return: str, int, commit ID of HEAD\n \"\"\"\n retry_delay = GIT_BACKOFF_FACTOR\n target_dir = target_dir or os.path.join(tempfile.mkdtemp(), \"repo\")\n commit = commit or \"master\"\n logger.info(\"cloning git repo '%s'\", git_url)\n logger.debug(\"url = '%s', dir = '%s', commit = '%s'\",\n git_url, target_dir, commit)\n\n cmd = [\"git\", \"clone\"]\n if branch:\n cmd += [\"-b\", branch, \"--single-branch\"]\n if depth:\n cmd += [\"--depth\", str(depth)]\n elif depth:\n logger.warning(\"branch not provided for %s, depth setting ignored\", git_url)\n depth = None\n\n cmd += [git_url, target_dir]\n\n logger.debug(\"cloning '%s'\", cmd)\n repo_commit = ''\n repo_depth = None\n for counter in range(retry_times + 1):\n try:\n # we are using check_output, even though we aren't using\n # the return value, but we will get 'output' in exception\n subprocess.check_output(cmd, stderr=subprocess.STDOUT)\n repo_commit, repo_depth = reset_git_repo(target_dir, commit, depth)\n break\n except subprocess.CalledProcessError as exc:\n if counter != retry_times:\n logger.info(\"retrying command '%s':\\n '%s'\", cmd, exc.output)\n time.sleep(retry_delay * (2 ** counter))\n else:\n raise OsbsException(\"Unable to clone git repo '%s' \"\n \"branch '%s'\" % (git_url, branch),\n cause=exc, traceback=sys.exc_info()[2])\n\n return ClonedRepoData(target_dir, repo_commit, repo_depth)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_imagestreamtag_from_image(image):\n ret = image\n\n # Remove the registry part\n ret = strip_registry_from_image(image)\n\n # ImageStream names cannot contain '/'\n ret = ret.replace('/', '-')\n\n # If there is no ':' suffix value, add one\n if ret.find(':') == -1:\n ret += \":latest\"\n\n return ret", "response": "Returns ImageStreamTag give a FROM value from the Dockerfile\n "} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_time_from_rfc3339(rfc3339):\n\n try:\n # py 3\n\n dt = dateutil.parser.parse(rfc3339, ignoretz=False)\n return dt.timestamp()\n except NameError:\n # py 2\n\n # Decode the RFC 3339 date with no fractional seconds (the\n # format Origin provides). Note that this will fail to parse\n # valid ISO8601 timestamps not in this exact format.\n time_tuple = strptime(rfc3339, '%Y-%m-%dT%H:%M:%SZ')\n return timegm(time_tuple)", "response": "Returns a time tuple from an ISO 3339 - formatted time string"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsanitizes a string for OpenShift.", "response": "def sanitize_strings_for_openshift(str1, str2='', limit=LABEL_MAX_CHARS, separator='-',\n label=True):\n \"\"\"\n OpenShift requires labels to be no more than 64 characters and forbids any characters other\n than alphanumerics, ., and -. BuildConfig names are similar, but cannot contain /.\n\n Sanitize and concatanate one or two strings to meet OpenShift's requirements. include an\n equal number of characters from both strings if the combined length is more than the limit.\n \"\"\"\n filter_chars = VALID_LABEL_CHARS if label else VALID_BUILD_CONFIG_NAME_CHARS\n str1_san = ''.join(filter(filter_chars.match, list(str1)))\n str2_san = ''.join(filter(filter_chars.match, list(str2)))\n\n str1_chars = []\n str2_chars = []\n groups = ((str1_san, str1_chars), (str2_san, str2_chars))\n\n size = len(separator)\n limit = min(limit, LABEL_MAX_CHARS)\n\n for i in range(max(len(str1_san), len(str2_san))):\n for group, group_chars in groups:\n if i < len(group):\n group_chars.append(group[i])\n size += 1\n if size >= limit:\n break\n else:\n continue\n break\n\n final_str1 = ''.join(str1_chars).strip(separator)\n final_str2 = ''.join(str2_chars).strip(separator)\n return separator.join(filter(None, (final_str1, final_str2)))"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef make_name_from_git(repo, branch, limit=53, separator='-', hash_size=5):\n\n branch = branch or 'unknown'\n full = urlparse(repo).path.lstrip('/') + branch\n repo = git_repo_humanish_part_from_uri(repo)\n shaval = sha256(full.encode('utf-8')).hexdigest()\n hash_str = shaval[:hash_size]\n limit = limit - len(hash_str) - 1\n\n sanitized = sanitize_strings_for_openshift(repo, branch, limit, separator, False)\n return separator.join(filter(None, (sanitized, hash_str)))", "response": "Make a build name from a git repository and branch."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwraps the result of make_name_from_git in a suffix and postfix adding separators for each.", "response": "def wrap_name_from_git(prefix, suffix, *args, **kwargs):\n \"\"\"\n wraps the result of make_name_from_git in a suffix and postfix\n adding separators for each.\n\n see docstring for make_name_from_git for a full list of parameters\n \"\"\"\n # 64 is maximum length allowed by OpenShift\n # 2 is the number of dashes that will be added\n prefix = ''.join(filter(VALID_BUILD_CONFIG_NAME_CHARS.match, list(prefix)))\n suffix = ''.join(filter(VALID_BUILD_CONFIG_NAME_CHARS.match, list(suffix)))\n kwargs['limit'] = kwargs.get('limit', 64) - len(prefix) - len(suffix) - 2\n name_from_git = make_name_from_git(*args, **kwargs)\n return '-'.join([prefix, name_from_git, suffix])"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsanitizes the version of the older setuptools.", "response": "def sanitize_version(version):\n \"\"\"\n Take parse_version() output and standardize output from older\n setuptools' parse_version() to match current setuptools.\n \"\"\"\n if hasattr(version, 'base_version'):\n if version.base_version:\n parts = version.base_version.split('.')\n else:\n parts = []\n else:\n parts = []\n for part in version:\n if part.startswith('*'):\n break\n parts.append(part)\n parts = [int(p) for p in parts]\n\n if len(parts) < 3:\n parts += [0] * (3 - len(parts))\n\n major, minor, micro = parts[:3]\n cleaned_version = '{}.{}.{}'.format(major, minor, micro)\n return cleaned_version"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the most preferred name of a label", "response": "def get_name(self, label_type):\n \"\"\"\n returns the most preferred label name\n if there isn't any correct name in the list\n it will return newest label name\n \"\"\"\n if label_type in self._label_values:\n return self._label_values[label_type][0]\n else:\n return Labels.LABEL_NAMES[label_type][0]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning dictionary indexed by old label name.", "response": "def get_new_names_by_old():\n \"\"\"Return dictionary, new label name indexed by old label name.\"\"\"\n newdict = {}\n\n for label_type, label_names in Labels.LABEL_NAMES.items():\n for oldname in label_names[1:]:\n newdict[oldname] = Labels.LABEL_NAMES[label_type][0]\n return newdict"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_name_and_value(self, label_type):\n if label_type in self._label_values:\n return self._label_values[label_type]\n else:\n return (label_type, self._df_labels[label_type])", "response": "Return tuple of label name and value for label type."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninitializes kerberos credential cache with ticket - granting ticket that is valid for at least a specific hour.", "response": "def kerberos_ccache_init(principal, keytab_file, ccache_file=None):\n \"\"\"\n Checks whether kerberos credential cache has ticket-granting ticket that is valid for at least\n an hour.\n\n Default ccache is used unless ccache_file is provided. In that case, KRB5CCNAME environment\n variable is set to the value of ccache_file if we successfully obtain the ticket.\n \"\"\"\n tgt_valid = False\n env = {\"LC_ALL\": \"C\"} # klist uses locales to format date on RHEL7+\n if ccache_file:\n env[\"KRB5CCNAME\"] = ccache_file\n\n # check if we have tgt that is valid more than one hour\n rc, klist, _ = run([\"klist\"], extraenv=env)\n if rc == 0:\n for line in klist.splitlines():\n m = re.match(KLIST_TGT_RE, line)\n if m:\n year = m.group(\"year\")\n if len(year) == 2:\n year = \"20\" + year\n\n expires = datetime.datetime(\n int(year), int(m.group(\"month\")), int(m.group(\"day\")),\n int(m.group(\"hour\")), int(m.group(\"minute\")), int(m.group(\"second\"))\n )\n\n if expires - datetime.datetime.now() > datetime.timedelta(hours=1):\n logger.debug(\"Valid TGT found, not renewing\")\n tgt_valid = True\n break\n\n if not tgt_valid:\n logger.debug(\"Retrieving kerberos TGT\")\n rc, out, err = run([\"kinit\", \"-k\", \"-t\", keytab_file, principal], extraenv=env)\n if rc != 0:\n raise OsbsException(\"kinit returned %s:\\nstdout: %s\\nstderr: %s\" % (rc, out, err))\n\n if ccache_file:\n os.environ[\"KRB5CCNAME\"] = ccache_file"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload the table data from a SQLite database file.", "response": "def load(self):\n \"\"\"\n Extract tabular data as |TableData| instances from a SQLite database\n file. |load_source_desc_file|\n\n :return:\n Loaded table data iterator.\n |load_table_name_desc|\n\n =================== ==============================================\n Format specifier Value after the replacement\n =================== ==============================================\n ``%(filename)s`` |filename_desc|\n ``%(key)s`` ``%(format_name)s%(format_id)s``\n ``%(format_name)s`` ``\"sqlite\"``\n ``%(format_id)s`` |format_id_desc|\n ``%(global_id)s`` |global_id|\n =================== ==============================================\n :rtype: |TableData| iterator\n :raises pytablereader.DataError:\n If the SQLite database file data is invalid or empty.\n \"\"\"\n\n self._validate()\n\n formatter = SqliteTableFormatter(self.source)\n formatter.accept(self)\n\n return formatter.to_table_data()"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind the data stored in the ConfigMap and return the json of the data that was passed into the ConfigMap on creation", "response": "def get_data(self):\n \"\"\"\n Find the data stored in the config_map\n\n :return: dict, the json of the data data that was passed into the ConfigMap on creation\n \"\"\"\n data = graceful_chain_get(self.json, \"data\")\n if data is None:\n return {}\n\n data_dict = {}\n for key in data:\n if self.is_yaml(key):\n data_dict[key] = yaml.load(data[key])\n else:\n data_dict[key] = json.loads(data[key])\n\n return data_dict"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_data_by_key(self, name):\n data = graceful_chain_get(self.json, \"data\")\n\n if data is None or name not in data:\n return {}\n\n if self.is_yaml(name):\n return yaml.load(data[name]) or {}\n return json.loads(data[name])", "response": "Find the object stored by a JSON string at key name"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nyield a generator that yields TableData objects.", "response": "def to_table_data(self):\n \"\"\"\n :raises ValueError:\n :raises pytablereader.error.ValidationError:\n \"\"\"\n\n self._validate_source_data()\n\n header_list = []\n for json_record in self._buffer:\n for key in json_record:\n if key not in header_list:\n header_list.append(key)\n\n self._loader.inc_table_count()\n\n yield TableData(\n self._make_table_name(),\n header_list,\n self._buffer,\n dp_extractor=self._loader.dp_extractor,\n type_hints=self._extract_type_hints(header_list),\n )"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nlists builds with matching fields", "response": "def list_builds(self, field_selector=None, koji_task_id=None, running=None,\n labels=None):\n \"\"\"\n List builds with matching fields\n\n :param field_selector: str, field selector for Builds\n :param koji_task_id: str, only list builds for Koji Task ID\n :return: BuildResponse list\n \"\"\"\n\n if running:\n running_fs = \",\".join([\"status!={status}\".format(status=status.capitalize())\n for status in BUILD_FINISHED_STATES])\n if not field_selector:\n field_selector = running_fs\n else:\n field_selector = ','.join([field_selector, running_fs])\n response = self.os.list_builds(field_selector=field_selector,\n koji_task_id=koji_task_id, labels=labels)\n serialized_response = response.json()\n build_list = []\n for build in serialized_response[\"items\"]:\n build_list.append(BuildResponse(build, self))\n\n return build_list"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the pod for a build", "response": "def get_pod_for_build(self, build_id):\n \"\"\"\n :return: PodResponse object for pod relating to the build\n \"\"\"\n pods = self.os.list_pods(label='openshift.io/build.name=%s' % build_id)\n serialized_response = pods.json()\n pod_list = [PodResponse(pod) for pod in serialized_response[\"items\"]]\n if not pod_list:\n raise OsbsException(\"No pod for build\")\n elif len(pod_list) != 1:\n raise OsbsException(\"Only one pod expected but %d returned\",\n len(pod_list))\n return pod_list[0]"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_build_request(self, build_type=None, inner_template=None,\n outer_template=None, customize_conf=None,\n arrangement_version=DEFAULT_ARRANGEMENT_VERSION):\n \"\"\"\n return instance of BuildRequest or BuildRequestV2\n\n :param build_type: str, unused\n :param inner_template: str, name of inner template for BuildRequest\n :param outer_template: str, name of outer template for BuildRequest\n :param customize_conf: str, name of customization config for BuildRequest\n :param arrangement_version: int, value of the arrangement version\n\n :return: instance of BuildRequest or BuildRequestV2\n \"\"\"\n if build_type is not None:\n warnings.warn(\"build types are deprecated, do not use the build_type argument\")\n\n validate_arrangement_version(arrangement_version)\n\n if not arrangement_version or arrangement_version < REACTOR_CONFIG_ARRANGEMENT_VERSION:\n build_request = BuildRequest(\n build_json_store=self.os_conf.get_build_json_store(),\n inner_template=inner_template,\n outer_template=outer_template,\n customize_conf=customize_conf)\n else:\n build_request = BuildRequestV2(\n build_json_store=self.os_conf.get_build_json_store(),\n outer_template=outer_template,\n customize_conf=customize_conf)\n\n # Apply configured resource limits.\n cpu_limit = self.build_conf.get_cpu_limit()\n memory_limit = self.build_conf.get_memory_limit()\n storage_limit = self.build_conf.get_storage_limit()\n if (cpu_limit is not None or\n memory_limit is not None or\n storage_limit is not None):\n build_request.set_resource_limits(cpu=cpu_limit,\n memory=memory_limit,\n storage=storage_limit)\n\n return build_request", "response": "Returns an instance of BuildRequest or BuildRequestV2 depending on the build_type and arrangement_version."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef create_build_from_buildrequest(self, build_request):\n build_request.set_openshift_required_version(self.os_conf.get_openshift_required_version())\n build = build_request.render()\n response = self.os.create_build(json.dumps(build))\n build_response = BuildResponse(response.json(), self)\n return build_response", "response": "render provided build_request and submit build from it\n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget an existing build config.", "response": "def _get_existing_build_config(self, build_config):\n \"\"\"\n Uses the given build config to find an existing matching build config.\n Build configs are a match if:\n - metadata.labels.git-repo-name AND metadata.labels.git-branch AND\n metadata.labels.git-full-repo are equal\n OR\n - metadata.labels.git-repo-name AND metadata.labels.git-branch are equal AND\n metadata.spec.source.git.uri are equal\n OR\n - metadata.name are equal\n \"\"\"\n\n bc_labels = build_config['metadata']['labels']\n git_labels = {\n \"label_selectors\": [(key, bc_labels[key]) for key in self._GIT_LABEL_KEYS]\n }\n old_labels_kwargs = {\n \"label_selectors\": [(key, bc_labels[key]) for key in self._OLD_LABEL_KEYS],\n \"filter_key\": FILTER_KEY,\n \"filter_value\": graceful_chain_get(build_config, *FILTER_KEY.split('.'))\n }\n name = {\n \"build_config_id\": build_config['metadata']['name']\n }\n\n queries = (\n (self.os.get_build_config_by_labels, git_labels),\n (self.os.get_build_config_by_labels_filtered, old_labels_kwargs),\n (self.os.get_build_config, name),\n )\n\n existing_bc = None\n for func, kwargs in queries:\n try:\n existing_bc = func(**kwargs)\n # build config found\n break\n except OsbsException as exc:\n # doesn't exist\n logger.info('Build config NOT found via %s: %s',\n func.__name__, str(exc))\n continue\n\n return existing_bc"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_image_stream_info_for_build_request(self, build_request):\n image_stream = None\n image_stream_tag_name = None\n\n if build_request.has_ist_trigger():\n image_stream_tag_id = build_request.trigger_imagestreamtag\n image_stream_id, image_stream_tag_name = image_stream_tag_id.split(':')\n\n try:\n image_stream = self.get_image_stream(image_stream_id).json()\n except OsbsResponseException as x:\n if x.status_code != 404:\n raise\n\n if image_stream:\n try:\n self.get_image_stream_tag(image_stream_tag_id).json()\n except OsbsResponseException as x:\n if x.status_code != 404:\n raise\n\n return image_stream, image_stream_tag_name", "response": "Return ImageStream and ImageStreamTag name for base_image of build_request."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_prod_build(self, *args, **kwargs):\n logger.warning(\"prod (all-in-one) builds are deprecated, \"\n \"please use create_orchestrator_build \"\n \"(support will be removed in version 0.54)\")\n return self._do_create_prod_build(*args, **kwargs)", "response": "Create a production build for the current build."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a worker build with the specified parameters", "response": "def create_worker_build(self, **kwargs):\n \"\"\"\n Create a worker build\n\n Pass through method to create_prod_build with the following\n modifications:\n - platform param is required\n - release param is required\n - arrangement_version param is required, which is used to\n select which worker_inner:n.json template to use\n - inner template set to worker_inner:n.json if not set\n - outer template set to worker.json if not set\n - customize configuration set to worker_customize.json if not set\n\n :return: BuildResponse instance\n \"\"\"\n missing = set()\n for required in ('platform', 'release', 'arrangement_version'):\n if not kwargs.get(required):\n missing.add(required)\n\n if missing:\n raise ValueError(\"Worker build missing required parameters: %s\" %\n missing)\n\n if kwargs.get('platforms'):\n raise ValueError(\"Worker build called with unwanted platforms param\")\n\n arrangement_version = kwargs['arrangement_version']\n kwargs.setdefault('inner_template', WORKER_INNER_TEMPLATE.format(\n arrangement_version=arrangement_version))\n kwargs.setdefault('outer_template', WORKER_OUTER_TEMPLATE)\n kwargs.setdefault('customize_conf', WORKER_CUSTOMIZE_CONF)\n kwargs['build_type'] = BUILD_TYPE_WORKER\n try:\n return self._do_create_prod_build(**kwargs)\n except IOError as ex:\n if os.path.basename(ex.filename) == kwargs['inner_template']:\n raise OsbsValidationException(\"worker invalid arrangement_version %s\" %\n arrangement_version)\n\n raise"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_orchestrator_build(self, **kwargs):\n if not self.can_orchestrate():\n raise OsbsOrchestratorNotEnabled(\"can't create orchestrate build \"\n \"when can_orchestrate isn't enabled\")\n extra = [x for x in ('platform',) if kwargs.get(x)]\n if extra:\n raise ValueError(\"Orchestrator build called with unwanted parameters: %s\" %\n extra)\n\n arrangement_version = kwargs.setdefault('arrangement_version',\n self.build_conf.get_arrangement_version())\n\n if arrangement_version < REACTOR_CONFIG_ARRANGEMENT_VERSION and not kwargs.get('platforms'):\n raise ValueError('Orchestrator build requires platforms param')\n\n kwargs.setdefault('inner_template', ORCHESTRATOR_INNER_TEMPLATE.format(\n arrangement_version=arrangement_version))\n kwargs.setdefault('outer_template', ORCHESTRATOR_OUTER_TEMPLATE)\n kwargs.setdefault('customize_conf', ORCHESTRATOR_CUSTOMIZE_CONF)\n kwargs['build_type'] = BUILD_TYPE_ORCHESTRATOR\n try:\n return self._do_create_prod_build(**kwargs)\n except IOError as ex:\n if os.path.basename(ex.filename) == kwargs['inner_template']:\n raise OsbsValidationException(\"orchestrator invalid arrangement_version %s\" %\n arrangement_version)\n\n raise", "response": "Create an orchestrator build for the current user."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the logs from the build.", "response": "def get_build_logs(self, build_id, follow=False, build_json=None, wait_if_missing=False,\n decode=False):\n \"\"\"\n provide logs from build\n\n NOTE: Since atomic-reactor 1.6.25, logs are always in UTF-8, so if\n asked to decode, we assume that is the encoding in use. Otherwise, we\n return the bytes exactly as they came from the container.\n\n :param build_id: str\n :param follow: bool, fetch logs as they come?\n :param build_json: dict, to save one get-build query\n :param wait_if_missing: bool, if build doesn't exist, wait\n :param decode: bool, whether or not to decode logs as utf-8\n :return: None, bytes, or iterable of bytes\n \"\"\"\n logs = self.os.logs(build_id, follow=follow, build_json=build_json,\n wait_if_missing=wait_if_missing)\n\n if decode and isinstance(logs, GeneratorType):\n return self._decode_build_logs_generator(logs)\n\n # str or None returned from self.os.logs()\n if decode and logs is not None:\n logs = logs.decode(\"utf-8\").rstrip()\n\n return logs"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting orchestrator build logs", "response": "def get_orchestrator_build_logs(self, build_id, follow=False, wait_if_missing=False):\n \"\"\"\n provide logs from orchestrator build\n\n :param build_id: str\n :param follow: bool, fetch logs as they come?\n :param wait_if_missing: bool, if build doesn't exist, wait\n :return: generator yielding objects with attributes 'platform' and 'line'\n \"\"\"\n logs = self.get_build_logs(build_id=build_id, follow=follow,\n wait_if_missing=wait_if_missing, decode=True)\n\n if logs is None:\n return\n if isinstance(logs, GeneratorType):\n for entries in logs:\n for entry in entries.splitlines():\n yield LogEntry(*self._parse_build_log_entry(entry))\n else:\n for entry in logs.splitlines():\n yield LogEntry(*self._parse_build_log_entry(entry))"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nimporting image tags from a Docker registry into an ImageStream", "response": "def import_image(self, name, tags=None):\n \"\"\"\n Import image tags from a Docker registry into an ImageStream\n\n :return: bool, whether tags were imported\n \"\"\"\n stream_import_file = os.path.join(self.os_conf.get_build_json_store(),\n 'image_stream_import.json')\n with open(stream_import_file) as f:\n stream_import = json.load(f)\n return self.os.import_image(name, stream_import, tags=tags)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef import_image_tags(self, name, tags, repository, insecure=False):\n stream_import_file = os.path.join(self.os_conf.get_build_json_store(),\n 'image_stream_import.json')\n with open(stream_import_file) as f:\n stream_import = json.load(f)\n return self.os.import_image_tags(name, stream_import, tags,\n repository, insecure)", "response": "Imports image tags from specified container repository."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nensure the tag is monitored in ImageStreamTag", "response": "def ensure_image_stream_tag(self, stream, tag_name, scheduled=False,\n source_registry=None, organization=None, base_image=None):\n \"\"\"Ensures the tag is monitored in ImageStream\n\n :param stream: dict, ImageStream object\n :param tag_name: str, name of tag to check, without name of\n ImageStream as prefix\n :param scheduled: bool, if True, importPolicy.scheduled will be\n set to True in ImageStreamTag\n :param source_registry: dict, info about source registry\n :param organization: str, oganization for registry\n :param base_image: str, base image\n :return: bool, whether or not modifications were performed\n \"\"\"\n img_stream_tag_file = os.path.join(self.os_conf.get_build_json_store(),\n 'image_stream_tag.json')\n with open(img_stream_tag_file) as f:\n tag_template = json.load(f)\n\n repository = None\n registry = None\n insecure = False\n\n if source_registry:\n registry = RegistryURI(source_registry['url']).docker_uri\n insecure = source_registry.get('insecure', False)\n\n if base_image and registry:\n repository = self._get_enclosed_repo_with_source_registry(base_image,\n registry, organization)\n\n return self.os.ensure_image_stream_tag(stream, tag_name, tag_template,\n scheduled, repository=repository,\n insecure=insecure)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_image_stream(self, name, docker_image_repository,\n insecure_registry=False):\n \"\"\"\n Create an ImageStream object\n\n Raises exception on error\n\n :param name: str, name of ImageStream\n :param docker_image_repository: str, pull spec for docker image\n repository\n :param insecure_registry: bool, whether plain HTTP should be used\n :return: response\n \"\"\"\n img_stream_file = os.path.join(self.os_conf.get_build_json_store(), 'image_stream.json')\n with open(img_stream_file) as f:\n stream = json.load(f)\n stream['metadata']['name'] = name\n stream['metadata'].setdefault('annotations', {})\n stream['metadata']['annotations'][ANNOTATION_SOURCE_REPO] = docker_image_repository\n if insecure_registry:\n stream['metadata']['annotations'][ANNOTATION_INSECURE_REPO] = 'true'\n\n return self.os.create_image_stream(json.dumps(stream))", "response": "Create an ImageStream object with the given name and docker image repository."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_compression_extension(self):\n\n build_request = BuildRequest(build_json_store=self.os_conf.get_build_json_store())\n inner = build_request.inner_template\n postbuild_plugins = inner.get('postbuild_plugins', [])\n for plugin in postbuild_plugins:\n if plugin.get('name') == 'compress':\n args = plugin.get('args', {})\n method = args.get('method', 'gzip')\n if method == 'gzip':\n return '.gz'\n elif method == 'lzma':\n return '.xz'\n raise OsbsValidationException(\"unknown compression method '%s'\"\n % method)\n\n return None", "response": "Find the filename extension for the docker save output which\n may or may not be compressed."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_config_map(self, name, data):\n config_data_file = os.path.join(self.os_conf.get_build_json_store(), 'config_map.json')\n with open(config_data_file) as f:\n config_data = json.load(f)\n config_data['metadata']['name'] = name\n data_dict = {}\n for key, value in data.items():\n data_dict[key] = json.dumps(value)\n config_data['data'] = data_dict\n\n response = self.os.create_config_map(config_data)\n config_map_response = ConfigMapResponse(response.json())\n return config_map_response", "response": "Create a ConfigMap object on the server"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget a ConfigMap object from the server", "response": "def get_config_map(self, name):\n \"\"\"\n Get a ConfigMap object from the server\n\n Raises exception on error\n\n :param name: str, name of configMap to get from the server\n :returns: ConfigMapResponse containing the ConfigMap with the requested name\n \"\"\"\n response = self.os.get_config_map(name)\n config_map_response = ConfigMapResponse(response.json())\n return config_map_response"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwipe the bolt database.", "response": "def wipe(self):\n \"\"\"\n Wipe the bolt database.\n\n Calling this after HoverPy has been instantiated is\n potentially dangerous. This function is mostly used\n internally for unit tests.\n \"\"\"\n try:\n if os.isfile(self._dbpath):\n os.remove(self._dbpath)\n except OSError:\n pass"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget the simulation data.", "response": "def simulation(self, data=None):\n \"\"\"\n Gets / Sets the simulation data.\n\n If no data is passed in, then this method acts as a getter.\n if data is passed in, then this method acts as a setter.\n\n Keyword arguments:\n data -- the simulation data you wish to set (default None)\n \"\"\"\n if data:\n return self._session.put(self.__v2() + \"/simulation\", data=data)\n else:\n return self._session.get(self.__v2() + \"/simulation\").json()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef destination(self, name=\"\"):\n if name:\n return self._session.put(\n self.__v2() + \"/hoverfly/destination\",\n data={\"destination\": name}).json()\n else:\n return self._session.get(\n self.__v2() + \"/hoverfly/destination\").json()", "response": "Gets the destination data."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting / Sets the mode. If no mode is provided, then this method acts as a getter. Keyword arguments: mode -- this should either be 'capture' or 'simulate' (default None)", "response": "def mode(self, mode=None):\n \"\"\"\n Gets / Sets the mode.\n\n If no mode is provided, then this method acts as a getter.\n\n Keyword arguments:\n mode -- this should either be 'capture' or 'simulate' (default None)\n \"\"\"\n if mode:\n logging.debug(\"SWITCHING TO %s\" % mode)\n url = self.__v2() + \"/hoverfly/mode\"\n logging.debug(url)\n return self._session.put(\n url, data=json.dumps({\"mode\": mode})).json()[\"mode\"]\n else:\n return self._session.get(\n self.__v2() + \"/hoverfly/mode\").json()[\"mode\"]"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the metadata of the current user.", "response": "def metadata(self, delete=False):\n \"\"\"\n Gets the metadata.\n \"\"\"\n if delete:\n return self._session.delete(self.__v1() + \"/metadata\").json()\n else:\n return self._session.get(self.__v1() + \"/metadata\").json()"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef records(self, data=None):\n if data:\n return self._session.post(\n self.__v1() + \"/records\", data=data).json()\n else:\n return self._session.get(self.__v1() + \"/records\").json()", "response": "Gets or sets records."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef delays(self, delays=[]):\n if delays:\n return self._session.put(\n self.__v1() + \"/delays\", data=json.dumps(delays)).json()\n else:\n return self._session.get(self.__v1() + \"/delays\").json()", "response": "Gets or sets the delays. \n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef addDelay(self, urlPattern=\"\", delay=0, httpMethod=None):\n print(\"addDelay is deprecated please use delays instead\")\n delay = {\"urlPattern\": urlPattern, \"delay\": delay}\n if httpMethod:\n delay[\"httpMethod\"] = httpMethod\n return self.delays(delays={\"data\": [delay]})", "response": "Adds a delay to the set of delays."} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nenables the hoverfly as a proxy.", "response": "def __enableProxy(self):\n \"\"\"\n Set the required environment variables to enable the use of hoverfly as a proxy.\n \"\"\"\n os.environ[\n \"HTTP_PROXY\"] = self.httpProxy()\n os.environ[\n \"HTTPS_PROXY\"] = self.httpsProxy()\n\n os.environ[\"REQUESTS_CA_BUNDLE\"] = os.path.join(\n os.path.dirname(\n os.path.abspath(__file__)),\n \"cert.pem\")"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites the PID to a temp file.", "response": "def __writepid(self, pid):\n \"\"\"\n HoverFly fails to launch if it's already running on\n the same ports. So we have to keep track of them using\n temp files with the proxy port and admin port, containing\n the processe's PID. \n \"\"\"\n import tempfile\n d = tempfile.gettempdir()\n name = os.path.join(d, \"hoverpy.%i.%i\"%(self._proxyPort, self._adminPort))\n with open(name, 'w') as f:\n f.write(str(pid))\n logging.debug(\"writing to %s\"%name)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nremoves the PID file on shutdown unfortunately unfortunately this may not get called on shutdown", "response": "def __rmpid(self):\n \"\"\"\n Remove the PID file on shutdown, unfortunately\n this may not get called if not given the time to\n shut down.\n \"\"\"\n import tempfile\n d = tempfile.gettempdir()\n name = os.path.join(d, \"hoverpy.%i.%i\"%(self._proxyPort, self._adminPort))\n if os.path.exists(name):\n os.unlink(name)\n logging.debug(\"deleting %s\"%name)"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nkilling HoverFly processes on these ports if they did not shut down correctly", "response": "def __kill_if_not_shut_properly(self):\n \"\"\"\n If the HoverFly process on these given ports\n did not shut down correctly, then kill the pid\n before launching a new instance.\n todo: this will kill existing HoverFly processes\n on those ports indiscriminately\n \"\"\"\n import tempfile\n d = tempfile.gettempdir()\n name = os.path.join(d, \"hoverpy.%i.%i\"%(self._proxyPort, self._adminPort))\n if os.path.exists(name):\n logging.debug(\"pid file exists.. killing it\")\n f = open(name, \"r\")\n pid = int(f.read())\n try:\n import signal\n os.kill(pid, signal.SIGTERM)\n logging.debug(\"killing %i\"%pid)\n except:\n logging.debug(\"nothing to clean up\")\n pass\n finally:\n os.unlink(name)"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting the hoverfly process.", "response": "def __start(self):\n \"\"\"\n Start the hoverfly process.\n\n This function waits until it can make contact\n with the hoverfly API before returning.\n \"\"\"\n logging.debug(\"starting %i\" % id(self))\n self.__kill_if_not_shut_properly()\n self.FNULL = open(os.devnull, 'w')\n flags = self.__flags()\n cmd = [hoverfly] + flags\n if self._showCmd:\n print(cmd)\n self._process = Popen(\n [hoverfly] +\n flags,\n stdin=self.FNULL,\n stdout=self.FNULL,\n stderr=subprocess.STDOUT)\n start = time.time()\n while time.time() - start < 1:\n try:\n url = \"http://%s:%i/api/health\" % (self._host, self._adminPort)\n r = self._session.get(url)\n j = r.json()\n up = \"message\" in j and \"healthy\" in j[\"message\"]\n if up:\n logging.debug(\"has pid %i\" % self._process.pid)\n self.__writepid(self._process.pid)\n return self._process\n else:\n time.sleep(1/100.0)\n except:\n # import traceback\n # traceback.print_exc()\n # wait 10 ms before trying again\n time.sleep(1/100.0)\n pass\n\n logging.error(\"Could not start hoverfly!\")\n raise ValueError(\"Could not start hoverfly!\")"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __stop(self):\n if logging:\n logging.debug(\"stopping\")\n self._process.terminate()\n # communicate means we wait until the process\n # was actually terminated, this removes some\n # warnings in python3\n self._process.communicate()\n self._process = None\n self.FNULL.close()\n self.FNULL = None\n self.__disableProxy()\n # del self._session\n # self._session = None\n self.__rmpid()", "response": "Stop the hoverfly process."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of flags for the current instance of the class.", "response": "def __flags(self):\n \"\"\"\n Internal method. Turns arguments into flags.\n \"\"\"\n flags = []\n if self._capture:\n flags.append(\"-capture\")\n if self._spy:\n flags.append(\"-spy\")\n if self._dbpath:\n flags += [\"-db-path\", self._dbpath]\n flags += [\"-db\", \"boltdb\"]\n else:\n flags += [\"-db\", \"memory\"]\n if self._synthesize:\n assert(self._middleware)\n flags += [\"-synthesize\"]\n if self._simulation:\n flags += [\"-import\", self._simulation]\n if self._proxyPort:\n flags += [\"-pp\", str(self._proxyPort)]\n if self._adminPort:\n flags += [\"-ap\", str(self._adminPort)]\n if self._modify:\n flags += [\"-modify\"]\n if self._verbose:\n flags += [\"-v\"]\n if self._dev:\n flags += [\"-dev\"]\n if self._metrics:\n flags += [\"-metrics\"]\n if self._auth:\n flags += [\"-auth\"]\n if self._middleware:\n flags += [\"-middleware\", self._middleware]\n if self._cert:\n flags += [\"-cert\", self._cert]\n if self._certName:\n flags += [\"-cert-name\", self._certName]\n if self._certOrg:\n flags += [\"-cert-org\", self._certOrg]\n if self._destination:\n flags += [\"-destination\", self._destination]\n if self._key:\n flags += [\"-key\", self._key]\n if self._dest:\n for i in range(len(self._dest)):\n flags += [\"-dest\", self._dest[i]]\n if self._generateCACert:\n flags += [\"-generate-ca-cert\"]\n if not self._tlsVerification:\n flags += [\"-tls-verification\", \"false\"]\n\n logging.debug(\"flags:\" + str(flags))\n return flags"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nextract tabular data as |TableData| instances from a JSON file. |load_source_desc_file| This method can be loading four types of JSON formats: **(1)** Single table data in a file: .. code-block:: json :caption: Acceptable JSON Schema (1): single table { \"type\": \"array\", \"items\": { \"type\": \"object\", \"additionalProperties\": { \"anyOf\": [ {\"type\": \"string\"}, {\"type\": \"number\"}, {\"type\": \"boolean\"}, {\"type\": \"null\"} ] } } } .. code-block:: json :caption: Acceptable JSON example for the JSON schema (1) [ {\"attr_b\": 4, \"attr_c\": \"a\", \"attr_a\": 1}, {\"attr_b\": 2.1, \"attr_c\": \"bb\", \"attr_a\": 2}, {\"attr_b\": 120.9, \"attr_c\": \"ccc\", \"attr_a\": 3} ] The example data will be loaded as the following tabular data: .. table:: +------+------+------+ |attr_a|attr_b|attr_c| +======+======+======+ | 1| 4.0|a | +------+------+------+ | 2| 2.1|bb | +------+------+------+ | 3| 120.9|ccc | +------+------+------+ **(2)** Single table data in a file: .. code-block:: json :caption: Acceptable JSON Schema (2): single table { \"type\": \"object\", \"additionalProperties\": { \"type\": \"array\", \"items\": { \"anyOf\": [ {\"type\": \"string\"}, {\"type\": \"number\"}, {\"type\": \"boolean\"}, {\"type\": \"null\"} ] } } } .. code-block:: json :caption: Acceptable JSON example for the JSON schema (2) { \"attr_a\": [1, 2, 3], \"attr_b\": [4, 2.1, 120.9], \"attr_c\": [\"a\", \"bb\", \"ccc\"] } The example data will be loaded as the following tabular data: .. table:: +------+------+------+ |attr_a|attr_b|attr_c| +======+======+======+ | 1| 4.0|a | +------+------+------+ | 2| 2.1|bb | +------+------+------+ | 3| 120.9|ccc | +------+------+------+ **(3)** Single table data in a file: .. code-block:: json :caption: Acceptable JSON Schema (3): single table { \"type\": \"object\", \"additionalProperties\": { \"anyOf\": [ {\"type\": \"string\"}, {\"type\": \"number\"}, {\"type\": \"boolean\"}, {\"type\": \"null\"} ] } } .. code-block:: json :caption: Acceptable JSON example for the JSON schema (3) { \"num_ratings\": 27, \"support_threads\": 1, \"downloaded\": 925716, \"last_updated\":\"2017-12-01 6:22am GMT\", \"added\":\"2010-01-20\", \"num\": 1.1, \"hoge\": null } The example data will be loaded as the following tabular data: .. table:: +---------------+---------------------+ | key | value | +===============+=====================+ |num_ratings | 27| +---------------+---------------------+ |support_threads| 1| +---------------+---------------------+ |downloaded | 925716| +---------------+---------------------+ |last_updated |2017-12-01 6:22am GMT| +---------------+---------------------+ |added |2010-01-20 | +---------------+---------------------+ |num | 1.1| +---------------+---------------------+ |hoge |None | +---------------+---------------------+ **(4)** Multiple table data in a file: .. code-block:: json :caption: Acceptable JSON Schema (4): multiple tables { \"type\": \"object\", \"additionalProperties\": { \"type\": \"array\", \"items\": { \"type\": \"object\", \"additionalProperties\": { \"anyOf\": [ {\"type\": \"string\"}, {\"type\": \"number\"}, {\"type\": \"boolean\"}, {\"type\": \"null\"} ] } } } } .. code-block:: json :caption: Acceptable JSON example for the JSON schema (4) { \"table_a\" : [ {\"attr_b\": 4, \"attr_c\": \"a\", \"attr_a\": 1}, {\"attr_b\": 2.1, \"attr_c\": \"bb\", \"attr_a\": 2}, {\"attr_b\": 120.9, \"attr_c\": \"ccc\", \"attr_a\": 3} ], \"table_b\" : [ {\"a\": 1, \"b\": 4}, {\"a\": 2 }, {\"a\": 3, \"b\": 120.9} ] } The example data will be loaded as the following tabular data: .. table:: table_a +------+------+------+ |attr_a|attr_b|attr_c| +======+======+======+ | 1| 4.0|a | +------+------+------+ | 2| 2.1|bb | +------+------+------+ | 3| 120.9|ccc | +------+------+------+ .. table:: table_b +-+-----+ |a| b | +=+=====+ |1| 4.0| +-+-----+ |2| None| +-+-----+ |3|120.9| +-+-----+ **(5)** Multiple table data in a file: .. code-block:: json :caption: Acceptable JSON Schema (5): multiple tables { \"type\": \"object\", \"additionalProperties\": { \"type\": \"object\", \"additionalProperties\": { \"type\": \"array\", \"items\": { \"anyOf\": [ {\"type\": \"string\"}, {\"type\": \"number\"}, {\"type\": \"boolean\"}, {\"type\": \"null\"} ] } } } } .. code-block:: json :caption: Acceptable JSON example for the JSON schema (5) { \"table_a\" : { \"attr_a\": [1, 2, 3], \"attr_b\": [4, 2.1, 120.9], \"attr_c\": [\"a\", \"bb\", \"ccc\"] }, \"table_b\" : { \"a\": [1, 3], \"b\": [4, 120.9] } } The example data will be loaded as the following tabular data: .. table:: table_a +------+------+------+ |attr_a|attr_b|attr_c| +======+======+======+ | 1| 4.0|a | +------+------+------+ | 2| 2.1|bb | +------+------+------+ | 3| 120.9|ccc | +------+------+------+ .. table:: table_b +-+-----+ |a| b | +=+=====+ |1| 4.0| +-+-----+ |3|120.9| +-+-----+ **(6)** Multiple table data in a file: .. code-block:: json :caption: Acceptable JSON Schema (6): multiple tables { \"type\": \"object\", \"additionalProperties\": { \"type\": \"object\", \"additionalProperties\": { \"anyOf\": [ {\"type\": \"string\"}, {\"type\": \"number\"}, {\"type\": \"boolean\"}, {\"type\": \"null\"} ] } } } .. code-block:: json :caption: Acceptable JSON example for the JSON schema (6) { \"table_a\": { \"num_ratings\": 27, \"support_threads\": 1, \"downloaded\": 925716, \"last_updated\":\"2017-12-01 6:22am GMT\", \"added\":\"2010-01-20\", \"num\": 1.1, \"hoge\": null }, \"table_b\": { \"a\": 4, \"b\": 120.9 } } The example data will be loaded as the following tabular data: .. table:: table_a +---------------+---------------------+ | key | value | +===============+=====================+ |num_ratings | 27| +---------------+---------------------+ |support_threads| 1| +---------------+---------------------+ |downloaded | 925716| +---------------+---------------------+ |last_updated |2017-12-01 6:22am GMT| +---------------+---------------------+ |added |2010-01-20 | +---------------+---------------------+ |num | 1.1| +---------------+---------------------+ |hoge |None | +---------------+---------------------+ .. table:: table_b +---+-----+ |key|value| +===+=====+ |a | 4.0| +---+-----+ |b |120.9| +---+-----+ :return: Loaded table data iterator. |load_table_name_desc| =================== ============================================== Format specifier Value after the replacement =================== ============================================== ``%(filename)s`` |filename_desc| ``%(key)s`` | This replaced the different value | for each single/multiple JSON tables: | [single JSON table] | ``%(format_name)s%(format_id)s`` | [multiple JSON table] Table data key. ``%(format_name)s`` ``\"json\"`` ``%(format_id)s`` |format_id_desc| ``%(global_id)s`` |global_id| =================== ============================================== :rtype: |TableData| iterator :raises pytablereader.DataError: If the data is invalid JSON. :raises pytablereader.error.ValidationError: If the data is not acceptable JSON format.", "response": "def load(self):\n \"\"\"\n Extract tabular data as |TableData| instances from a JSON file.\n |load_source_desc_file|\n\n This method can be loading four types of JSON formats:\n\n **(1)** Single table data in a file:\n\n .. code-block:: json\n :caption: Acceptable JSON Schema (1): single table\n\n {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"anyOf\": [\n {\"type\": \"string\"},\n {\"type\": \"number\"},\n {\"type\": \"boolean\"},\n {\"type\": \"null\"}\n ]\n }\n }\n }\n\n .. code-block:: json\n :caption: Acceptable JSON example for the JSON schema (1)\n\n [\n {\"attr_b\": 4, \"attr_c\": \"a\", \"attr_a\": 1},\n {\"attr_b\": 2.1, \"attr_c\": \"bb\", \"attr_a\": 2},\n {\"attr_b\": 120.9, \"attr_c\": \"ccc\", \"attr_a\": 3}\n ]\n\n The example data will be loaded as the following tabular data:\n\n .. table::\n\n +------+------+------+\n |attr_a|attr_b|attr_c|\n +======+======+======+\n | 1| 4.0|a |\n +------+------+------+\n | 2| 2.1|bb |\n +------+------+------+\n | 3| 120.9|ccc |\n +------+------+------+\n\n **(2)** Single table data in a file:\n\n .. code-block:: json\n :caption: Acceptable JSON Schema (2): single table\n\n {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"type\": \"array\",\n \"items\": {\n \"anyOf\": [\n {\"type\": \"string\"},\n {\"type\": \"number\"},\n {\"type\": \"boolean\"},\n {\"type\": \"null\"}\n ]\n }\n }\n }\n\n .. code-block:: json\n :caption: Acceptable JSON example for the JSON schema (2)\n\n {\n \"attr_a\": [1, 2, 3],\n \"attr_b\": [4, 2.1, 120.9],\n \"attr_c\": [\"a\", \"bb\", \"ccc\"]\n }\n\n The example data will be loaded as the following tabular data:\n\n .. table::\n\n +------+------+------+\n |attr_a|attr_b|attr_c|\n +======+======+======+\n | 1| 4.0|a |\n +------+------+------+\n | 2| 2.1|bb |\n +------+------+------+\n | 3| 120.9|ccc |\n +------+------+------+\n\n **(3)** Single table data in a file:\n\n .. code-block:: json\n\n :caption: Acceptable JSON Schema (3): single table\n\n {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"anyOf\": [\n {\"type\": \"string\"},\n {\"type\": \"number\"},\n {\"type\": \"boolean\"},\n {\"type\": \"null\"}\n ]\n }\n }\n\n .. code-block:: json\n :caption: Acceptable JSON example for the JSON schema (3)\n\n {\n \"num_ratings\": 27,\n \"support_threads\": 1,\n \"downloaded\": 925716,\n \"last_updated\":\"2017-12-01 6:22am GMT\",\n \"added\":\"2010-01-20\",\n \"num\": 1.1,\n \"hoge\": null\n }\n\n The example data will be loaded as the following tabular data:\n\n .. table::\n\n +---------------+---------------------+\n | key | value |\n +===============+=====================+\n |num_ratings | 27|\n +---------------+---------------------+\n |support_threads| 1|\n +---------------+---------------------+\n |downloaded | 925716|\n +---------------+---------------------+\n |last_updated |2017-12-01 6:22am GMT|\n +---------------+---------------------+\n |added |2010-01-20 |\n +---------------+---------------------+\n |num | 1.1|\n +---------------+---------------------+\n |hoge |None |\n +---------------+---------------------+\n\n **(4)** Multiple table data in a file:\n\n .. code-block:: json\n :caption: Acceptable JSON Schema (4): multiple tables\n\n {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"anyOf\": [\n {\"type\": \"string\"},\n {\"type\": \"number\"},\n {\"type\": \"boolean\"},\n {\"type\": \"null\"}\n ]\n }\n }\n }\n }\n\n .. code-block:: json\n :caption: Acceptable JSON example for the JSON schema (4)\n\n {\n \"table_a\" : [\n {\"attr_b\": 4, \"attr_c\": \"a\", \"attr_a\": 1},\n {\"attr_b\": 2.1, \"attr_c\": \"bb\", \"attr_a\": 2},\n {\"attr_b\": 120.9, \"attr_c\": \"ccc\", \"attr_a\": 3}\n ],\n \"table_b\" : [\n {\"a\": 1, \"b\": 4},\n {\"a\": 2 },\n {\"a\": 3, \"b\": 120.9}\n ]\n }\n\n The example data will be loaded as the following tabular data:\n\n .. table:: table_a\n\n +------+------+------+\n |attr_a|attr_b|attr_c|\n +======+======+======+\n | 1| 4.0|a |\n +------+------+------+\n | 2| 2.1|bb |\n +------+------+------+\n | 3| 120.9|ccc |\n +------+------+------+\n\n .. table:: table_b\n\n +-+-----+\n |a| b |\n +=+=====+\n |1| 4.0|\n +-+-----+\n |2| None|\n +-+-----+\n |3|120.9|\n +-+-----+\n\n **(5)** Multiple table data in a file:\n\n .. code-block:: json\n :caption: Acceptable JSON Schema (5): multiple tables\n\n {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"type\": \"array\",\n \"items\": {\n \"anyOf\": [\n {\"type\": \"string\"},\n {\"type\": \"number\"},\n {\"type\": \"boolean\"},\n {\"type\": \"null\"}\n ]\n }\n }\n }\n }\n\n .. code-block:: json\n :caption: Acceptable JSON example for the JSON schema (5)\n\n {\n \"table_a\" : {\n \"attr_a\": [1, 2, 3],\n \"attr_b\": [4, 2.1, 120.9],\n \"attr_c\": [\"a\", \"bb\", \"ccc\"]\n },\n \"table_b\" : {\n \"a\": [1, 3],\n \"b\": [4, 120.9]\n }\n }\n\n The example data will be loaded as the following tabular data:\n\n .. table:: table_a\n\n +------+------+------+\n |attr_a|attr_b|attr_c|\n +======+======+======+\n | 1| 4.0|a |\n +------+------+------+\n | 2| 2.1|bb |\n +------+------+------+\n | 3| 120.9|ccc |\n +------+------+------+\n\n .. table:: table_b\n\n +-+-----+\n |a| b |\n +=+=====+\n |1| 4.0|\n +-+-----+\n |3|120.9|\n +-+-----+\n\n **(6)** Multiple table data in a file:\n\n .. code-block:: json\n :caption: Acceptable JSON Schema (6): multiple tables\n\n {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"anyOf\": [\n {\"type\": \"string\"},\n {\"type\": \"number\"},\n {\"type\": \"boolean\"},\n {\"type\": \"null\"}\n ]\n }\n }\n }\n\n .. code-block:: json\n :caption: Acceptable JSON example for the JSON schema (6)\n\n {\n \"table_a\": {\n \"num_ratings\": 27,\n \"support_threads\": 1,\n \"downloaded\": 925716,\n \"last_updated\":\"2017-12-01 6:22am GMT\",\n \"added\":\"2010-01-20\",\n \"num\": 1.1,\n \"hoge\": null\n },\n \"table_b\": {\n \"a\": 4,\n \"b\": 120.9\n }\n }\n\n The example data will be loaded as the following tabular data:\n\n .. table:: table_a\n\n +---------------+---------------------+\n | key | value |\n +===============+=====================+\n |num_ratings | 27|\n +---------------+---------------------+\n |support_threads| 1|\n +---------------+---------------------+\n |downloaded | 925716|\n +---------------+---------------------+\n |last_updated |2017-12-01 6:22am GMT|\n +---------------+---------------------+\n |added |2010-01-20 |\n +---------------+---------------------+\n |num | 1.1|\n +---------------+---------------------+\n |hoge |None |\n +---------------+---------------------+\n\n\n .. table:: table_b\n\n +---+-----+\n |key|value|\n +===+=====+\n |a | 4.0|\n +---+-----+\n |b |120.9|\n +---+-----+\n\n :return:\n Loaded table data iterator.\n |load_table_name_desc|\n\n =================== ==============================================\n Format specifier Value after the replacement\n =================== ==============================================\n ``%(filename)s`` |filename_desc|\n ``%(key)s`` | This replaced the different value\n | for each single/multiple JSON tables:\n | [single JSON table]\n | ``%(format_name)s%(format_id)s``\n | [multiple JSON table] Table data key.\n ``%(format_name)s`` ``\"json\"``\n ``%(format_id)s`` |format_id_desc|\n ``%(global_id)s`` |global_id|\n =================== ==============================================\n :rtype: |TableData| iterator\n :raises pytablereader.DataError:\n If the data is invalid JSON.\n :raises pytablereader.error.ValidationError:\n If the data is not acceptable JSON format.\n \"\"\"\n\n formatter = JsonTableFormatter(self.load_dict())\n formatter.accept(self)\n\n return formatter.to_table_data()"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load(self):\n\n self._validate()\n self._logger.logging_load()\n self.encoding = get_file_encoding(self.source, self.encoding)\n\n with io.open(self.source, \"r\", encoding=self.encoding) as fp:\n formatter = MediaWikiTableFormatter(fp.read())\n formatter.accept(self)\n\n return formatter.to_table_data()", "response": "Loads the tabular data from a MediaWiki file."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading the tabular data from a MediaWiki text object.", "response": "def load(self):\n \"\"\"\n Extract tabular data as |TableData| instances from a MediaWiki text\n object.\n |load_source_desc_text|\n\n :return:\n Loaded table data iterator.\n |load_table_name_desc|\n\n =================== ==============================================\n Format specifier Value after the replacement\n =================== ==============================================\n ``%(filename)s`` ``\"\"``\n ``%(key)s`` | This replaced to:\n | **(1)** ``caption`` mark of the table\n | **(2)** ``%(format_name)s%(format_id)s``\n | if ``caption`` mark not included\n | in the table.\n ``%(format_name)s`` ``\"mediawiki\"``\n ``%(format_id)s`` |format_id_desc|\n ``%(global_id)s`` |global_id|\n =================== ==============================================\n :rtype: |TableData| iterator\n :raises pytablereader.DataError:\n If the MediaWiki data is invalid or empty.\n \"\"\"\n\n self._validate()\n self._logger.logging_load()\n\n formatter = MediaWikiTableFormatter(self.source)\n formatter.accept(self)\n\n return formatter.to_table_data()"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn dock json from existing build json", "response": "def get_dock_json(self):\n \"\"\" return dock json from existing build json \"\"\"\n env_json = self.build_json['spec']['strategy']['customStrategy']['env']\n try:\n p = [env for env in env_json if env[\"name\"] == \"ATOMIC_REACTOR_PLUGINS\"]\n except TypeError:\n raise RuntimeError(\"\\\"env\\\" is not iterable\")\n if len(p) <= 0:\n raise RuntimeError(\"\\\"env\\\" misses key ATOMIC_REACTOR_PLUGINS\")\n dock_json_str = p[0]['value']\n dock_json = json.loads(dock_json_str)\n return dock_json"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef dock_json_get_plugin_conf(self, plugin_type, plugin_name):\n match = [x for x in self.dock_json[plugin_type] if x.get('name') == plugin_name]\n return match[0]", "response": "Get the configuration for a plugin."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef remove_plugin(self, plugin_type, plugin_name):\n for p in self.dock_json[plugin_type]:\n if p.get('name') == plugin_name:\n self.dock_json[plugin_type].remove(p)\n break", "response": "Remove a specific plugin from the dock. json file."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd a new plugin to the dock_json.", "response": "def add_plugin(self, plugin_type, plugin_name, args_dict):\n \"\"\"\n if config has plugin, override it, else add it\n \"\"\"\n\n plugin_modified = False\n\n for plugin in self.dock_json[plugin_type]:\n if plugin['name'] == plugin_name:\n plugin['args'] = args_dict\n plugin_modified = True\n\n if not plugin_modified:\n self.dock_json[plugin_type].append({\"name\": plugin_name, \"args\": args_dict})"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking whether a plugin is configured.", "response": "def dock_json_has_plugin_conf(self, plugin_type, plugin_name):\n \"\"\"\n Check whether a plugin is configured.\n \"\"\"\n\n try:\n self.dock_json_get_plugin_conf(plugin_type, plugin_name)\n return True\n except (KeyError, IndexError):\n return False"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_from_path(self):\n\n import requests\n\n url_path = urlparse(self.__url).path\n try:\n url_extension = get_extension(url_path.rstrip(\"/\"))\n except InvalidFilePathError:\n raise UrlError(\"url must include path\")\n\n logger.debug(\"TableUrlLoaderFactory: extension={}\".format(url_extension))\n\n loader_class = self._get_loader_class(self._get_extension_loader_mapping(), url_extension)\n\n try:\n self._fetch_source(loader_class)\n except requests.exceptions.ProxyError as e:\n raise ProxyError(e)\n\n loader = self._create_from_extension(url_extension)\n\n logger.debug(\"TableUrlLoaderFactory: loader={}\".format(loader.format_name))\n\n return loader", "response": "Create a file loader from the file extension to loading file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_from_format_name(self, format_name):\n\n import requests\n\n logger.debug(\"TableUrlLoaderFactory: name={}\".format(format_name))\n\n loader_class = self._get_loader_class(self._get_format_name_loader_mapping(), format_name)\n\n try:\n self._fetch_source(loader_class)\n except requests.exceptions.ProxyError as e:\n raise ProxyError(e)\n\n loader = self._create_from_format_name(format_name)\n\n logger.debug(\"TableUrlLoaderFactory: loader={}\".format(loader.format_name))\n\n return loader", "response": "Create a file loader from a format name."} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a mapping of format - extension and loader class.", "response": "def _get_extension_loader_mapping(self):\n \"\"\"\n :return: Mappings of format-extension and loader class.\n :rtype: dict\n \"\"\"\n\n loader_table = self._get_common_loader_mapping()\n loader_table.update(\n {\n \"asp\": HtmlTableTextLoader,\n \"aspx\": HtmlTableTextLoader,\n \"htm\": HtmlTableTextLoader,\n \"md\": MarkdownTableTextLoader,\n \"sqlite3\": SqliteFileLoader,\n \"xls\": ExcelTableFileLoader,\n \"xlsx\": ExcelTableFileLoader,\n }\n )\n\n return loader_table"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a mapping of format - name and loader class.", "response": "def _get_format_name_loader_mapping(self):\n \"\"\"\n :return: Mappings of format-name and loader class.\n :rtype: dict\n \"\"\"\n\n loader_table = self._get_common_loader_mapping()\n loader_table.update(\n {\n \"excel\": ExcelTableFileLoader,\n \"json_lines\": JsonLinesTableTextLoader,\n \"markdown\": MarkdownTableTextLoader,\n \"mediawiki\": MediaWikiTableTextLoader,\n \"ssv\": CsvTableFileLoader,\n }\n )\n\n return loader_table"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_container_image_ids(self):\n\n statuses = graceful_chain_get(self.json, \"status\", \"containerStatuses\")\n if statuses is None:\n return {}\n\n def remove_prefix(image_id, prefix):\n if image_id.startswith(prefix):\n return image_id[len(prefix):]\n\n return image_id\n\n return {status['image']: remove_prefix(status['imageID'], 'docker://')\n for status in statuses}", "response": "Find the image IDs the containers use."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_failure_reason(self):\n\n reason_key = 'reason'\n cid_key = 'containerID'\n exit_key = 'exitCode'\n\n pod_status = self.json.get('status', {})\n statuses = pod_status.get('containerStatuses', [])\n\n # Find the first non-zero exit code from a container\n # and return its 'message' or 'reason' value\n for status in statuses:\n try:\n terminated = status['state']['terminated']\n exit_code = terminated['exitCode']\n if exit_code != 0:\n reason_dict = {\n exit_key: exit_code,\n }\n\n if 'containerID' in terminated:\n reason_dict[cid_key] = terminated['containerID']\n\n for key in ['message', 'reason']:\n try:\n reason_dict[reason_key] = terminated[key]\n break\n except KeyError:\n continue\n else:\n # Both 'message' and 'reason' are missing\n reason_dict[reason_key] = 'Exit code {code}'.format(\n code=exit_code\n )\n\n return reason_dict\n except KeyError:\n continue\n\n # Failing that, return the 'message' or 'reason' value for the\n # pod\n for key in ['message', 'reason']:\n try:\n return {reason_key: pod_status[key]}\n except KeyError:\n continue\n\n return {reason_key: pod_status['phase']}", "response": "Find the reason a pod failed and return the message or reason value for the next non - zero exit code."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_error_message(self):\n error_reason = self.get_error_reason()\n if error_reason:\n error_message = error_reason.get('pod') or None\n if error_message:\n return \"Error in pod: %s\" % error_message\n plugin = error_reason.get('plugin')[0] or None\n error_message = error_reason.get('plugin')[1] or None\n if error_message:\n # Plugin has non-empty error description\n return \"Error in plugin %s: %s\" % (plugin, error_message)\n else:\n return \"Error in plugin %s\" % plugin", "response": "Return an error message based on atomic - actor s metadata."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a file loader from the file extension.", "response": "def create_from_path(self):\n \"\"\"\n Create a file loader from the file extension to loading file.\n Supported file extensions are as follows:\n\n ========================== =======================================\n Extension Loader\n ========================== =======================================\n ``\"csv\"`` :py:class:`~.CsvTableFileLoader`\n ``\"xls\"``/``\"xlsx\"`` :py:class:`~.ExcelTableFileLoader`\n ``\"htm\"``/``\"html\"`` :py:class:`~.HtmlTableFileLoader`\n ``\"json\"`` :py:class:`~.JsonTableFileLoader`\n ``\"jsonl\"`` :py:class:`~.JsonLinesTableFileLoader`\n ``\"ldjson\"`` :py:class:`~.JsonLinesTableFileLoader`\n ``\"ltsv\"`` :py:class:`~.LtsvTableFileLoader`\n ``\"md\"`` :py:class:`~.MarkdownTableFileLoader`\n ``\"ndjson\"`` :py:class:`~.JsonLinesTableFileLoader`\n ``\"sqlite\"``/``\"sqlite3\"`` :py:class:`~.SqliteFileLoader`\n ``\"tsv\"`` :py:class:`~.TsvTableFileLoader`\n ========================== =======================================\n\n :return:\n Loader that coincides with the file extension of the\n :py:attr:`.file_extension`.\n :raises pytablereader.LoaderNotFoundError:\n |LoaderNotFoundError_desc| loading the file.\n \"\"\"\n\n loader = self._create_from_extension(self.file_extension)\n\n logger.debug(\n \"TableFileLoaderFactory.create_from_path: extension={}, loader={}\".format(\n self.file_extension, loader.format_name\n )\n )\n\n return loader"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_from_format_name(self, format_name):\n\n loader = self._create_from_format_name(format_name)\n\n logger.debug(\n \"TableFileLoaderFactory.create_from_format_name: name={}, loader={}\".format(\n format_name, loader.format_name\n )\n )\n\n return loader", "response": "Create a file loader from a format name."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_extension_loader_mapping(self):\n\n loader_table = self._get_common_loader_mapping()\n loader_table.update(\n {\n \"htm\": HtmlTableFileLoader,\n \"md\": MarkdownTableFileLoader,\n \"sqlite3\": SqliteFileLoader,\n \"xlsx\": ExcelTableFileLoader,\n \"xls\": ExcelTableFileLoader,\n }\n )\n\n return loader_table", "response": "Returns a mapping of format extension and loader class."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_format_name_loader_mapping(self):\n\n loader_table = self._get_common_loader_mapping()\n loader_table.update(\n {\n \"excel\": ExcelTableFileLoader,\n \"json_lines\": JsonLinesTableFileLoader,\n \"markdown\": MarkdownTableFileLoader,\n \"mediawiki\": MediaWikiTableFileLoader,\n \"ssv\": CsvTableFileLoader,\n }\n )\n\n return loader_table", "response": "Returns a mapping of format name and loader class."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nload the table data from an Excel file.", "response": "def load(self):\n \"\"\"\n Extract tabular data as |TableData| instances from an Excel file.\n |spreadsheet_load_desc|\n\n :return:\n Loaded |TableData| iterator.\n |TableData| created for each sheet in the workbook.\n |load_table_name_desc|\n\n =================== ====================================\n Format specifier Value after the replacement\n =================== ====================================\n ``%(filename)s`` Filename of the workbook\n ``%(sheet)s`` Name of the sheet\n ``%(format_name)s`` ``\"spreadsheet\"``\n ``%(format_id)s`` |format_id_desc|\n ``%(global_id)s`` |global_id|\n =================== ====================================\n :rtype: |TableData| iterator\n :raises pytablereader.DataError:\n If the header row is not found.\n :raises pytablereader.error.OpenError:\n If failed to open the source file.\n \"\"\"\n\n import xlrd\n\n self._validate()\n self._logger.logging_load()\n\n try:\n workbook = xlrd.open_workbook(self.source)\n except xlrd.biffh.XLRDError as e:\n raise OpenError(e)\n\n for worksheet in workbook.sheets():\n self._worksheet = worksheet\n\n if self._is_empty_sheet():\n continue\n\n self.__extract_not_empty_col_idx()\n\n try:\n start_row_idx = self._get_start_row_idx()\n except DataError:\n continue\n\n rows = [\n self.__get_row_values(row_idx)\n for row_idx in range(start_row_idx + 1, self._row_count)\n ]\n\n self.inc_table_count()\n headers = self.__get_row_values(start_row_idx)\n\n yield TableData(\n self._make_table_name(),\n headers,\n rows,\n dp_extractor=self.dp_extractor,\n type_hints=self._extract_type_hints(headers),\n )"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load(self):\n\n self._validate()\n self._logger.logging_load()\n self.encoding = get_file_encoding(self.source, self.encoding)\n\n self._ltsv_input_stream = io.open(self.source, \"r\", encoding=self.encoding)\n\n for data_matrix in self._to_data_matrix():\n formatter = SingleJsonTableConverterA(data_matrix)\n formatter.accept(self)\n\n return formatter.to_table_data()", "response": "Loads the tabular data from a LTSV file."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load(self):\n\n self._validate()\n self._logger.logging_load()\n\n self._ltsv_input_stream = self.source.splitlines()\n\n for data_matrix in self._to_data_matrix():\n formatter = SingleJsonTableConverterA(data_matrix)\n formatter.accept(self)\n\n return formatter.to_table_data()", "response": "Loads the tabular data from the LTSV text file."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef totals(self):\n def agg(d):\n keys = ['g','a','p','pm','pn','pim','s','ab','ms','ht','gv','tk','bs']\n res = { k: 0 for k in keys }\n res['fo'] = { 'won': 0, 'total': 0 }\n for _, v in d.items():\n for k in keys:\n res[k] += v[k]\n for fi in res['fo'].keys():\n res['fo'][fi] += v['fo'][fi]\n return res\n \n return self.__apply_to_both(agg)", "response": "Computes and returns a dictionary containing home and away by player shots and face - off by player shots and face - off."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the subset home and away players that satisfy the provided filter function.", "response": "def filter_players(self, pl_filter):\n \"\"\"\n Return the subset home and away players that satisfy the provided filter function.\n \n :param pl_filter: function that takes a by player dictionary and returns bool\n :returns: dict of the form ``{ 'home/away': { by_player_dict } }``. See :py:func:`home_players` and :py:func:`away_players`\n \"\"\"\n def each(d):\n return {\n k: v\n for k, v in d.items()\n if pl_filter(k, v)\n }\n \n return self.__apply_to_both(each)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn all home and away by player info sorted by either the provided key or function.", "response": "def sort_players(self, sort_key=None, sort_func=None, reverse=False):\n \"\"\"\n Return all home and away by player info sorted by either the provided key or function. Must provide\n at least one of the two parameters. Can sort either ascending or descending.\n \n :param sort_key: (def None) dict key to sort on\n :param sort_func: (def None) sorting function\n :param reverse: (optional, def False) if True, sort descending\n :returns: dict of the form ``{ 'home/away': { by_player_dict } }``. See :py:func:`home_players` and :py:func:`away_players`\n \"\"\"\n def each(d):\n t = [ ]\n for num, v in d.items():\n ti = { vk: vv for vk, vv in v.items() }\n ti['num'] = num\n t.append(ti)\n \n if sort_key:\n return sorted(t, key=lambda k: k[sort_key], reverse=reverse)\n else:\n return sorted(t, key=sort_func, reverse=reverse)\n \n return self.__apply_to_both(each)"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef top_by_key(self, sort_key):\n res = self.sort_players(sort_key=sort_key, reverse=True)\n return {\n 'home': res['home'][0],\n 'away': res['away'][0]\n }", "response": "Return home and away by player info for each team in each category."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning home and away by player info for each team who come in first according to the sort_func. Will perform ascending sort.", "response": "def top_by_func(self, sort_func):\n \"\"\"\n Return home/away by player info for the players on each team who come in first according to the\n provided sorting function. Will perform ascending sort.\n \n :param sort_func: function that yields the sorting quantity\n :returns: dict of the form ``{ 'home/away': { by_player_dict } }``. See :py:func:`home_players` and :py:func:`away_players`\n \"\"\"\n res = self.sort_players(sort_func=sort_func, reverse=True)\n return {\n 'home': res['home'][0],\n 'away': res['away'][0]\n }"} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef encode(obj):\n if hasattr(obj, 'json'):\n return obj.json\n if hasattr(obj, '__json__'):\n return obj.__json__()\n return dumps(obj)", "response": "Encode one argument or object to json\n "} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef encode_args(args, extra=False):\n if not args:\n return ''\n\n methodargs = ', '.join([encode(a) for a in args])\n if extra:\n methodargs += ', '\n\n return methodargs", "response": "Encode a list of arguments into a single string."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _send(self, javascript):\n\n # Prepend JS to switch to the proper client context.\n message = \"\"\"\n var _ctx = ctx_switch('%s'),\n browser = _ctx[0],\n ELEMENTS = _ctx[1];\n %s\n \"\"\" % (id(self), javascript)\n\n response = self.connection.send(message)\n\n return self._handle_response(response)", "response": "Sends a javascript string to the zombie. js server and returns the response."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef wait(self, method, *args):\n methodargs = encode_args(args, extra=True)\n js = \"\"\"\n %s(%s wait_callback);\n \"\"\" % (method, methodargs)\n self._send(js)", "response": "Call a method on the zombie. js Browser instance and wait on a callback."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef create_element(self, method, args=None):\n if args is None:\n arguments = ''\n else:\n arguments = \"(%s)\" % encode_args(args)\n js = \"\"\"\n create_element(ELEMENTS, %(method)s%(args)s);\n \"\"\" % {\n 'method': method,\n 'args': arguments\n }\n\n index = self.json(js)\n if index is None:\n return None\n\n return Element(index)", "response": "Evaluate a browser method and CSS selector against the document DOMNode and return a single Element object."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_elements(self, method, args=[]):\n args = encode_args(args)\n\n js = \"\"\"\n create_elements(ELEMENTS, %(method)s(%(args)s))\n \"\"\" % {\n 'method': method,\n 'args': args,\n }\n\n indexes = self.json(js)\n return map(Element, indexes)", "response": "Execute a browser method that will return a list of elements."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfilling a specified form field in the current document.", "response": "def fill(self, field, value):\n \"\"\"\n Fill a specified form field in the current document.\n\n :param field: an instance of :class:`zombie.dom.DOMNode`\n :param value: any string value\n :return: self to allow function chaining.\n \"\"\"\n self.client.nowait('browser.fill', (field, value))\n return self"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nevaluates a CSS selector against the document and return a single DOMNode object.", "response": "def query(self, selector, context=None):\n \"\"\"\n Evaluate a CSS selector against the document (or an optional context\n :class:`zombie.dom.DOMNode`) and return a single\n :class:`zombie.dom.DOMNode` object.\n\n :param selector: a string CSS selector\n (http://zombie.labnotes.org/selectors)\n :param context: an (optional) instance of :class:`zombie.dom.DOMNode`\n \"\"\"\n element = self.client.create_element(\n 'browser.query', (selector, context))\n return DOMNode.factory(element, self)"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef queryAll(self, selector, context=None):\n elements = self.client.create_elements(\n 'browser.queryAll', (selector, context))\n return [DOMNode(e, self) for e in elements]", "response": "Evaluate a CSS selector against the document and return a list of DOMNodes."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind and returns a link element with the given CSS selector.", "response": "def link(self, selector):\n \"\"\"\n Finds and returns a link ```` element (:class:`zombie.dom.DOMNode`).\n You can use a CSS selector or find a link by its text contents (case\n sensitive, but ignores leading/trailing spaces).\n\n :param selector: an optional string CSS selector\n (http://zombie.labnotes.org/selectors) or inner text\n \"\"\"\n element = self.client.create_element('browser.link', (selector,))\n return DOMNode(element, self)"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the value of the current field.", "response": "def value(self, value):\n \"\"\"\n Used to set the ``value`` of form elements.\n \"\"\"\n self.client.nowait(\n 'set_field', (Literal('browser'), self.element, value))"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef fire(self, event):\n self.browser.fire(self.element, event)\n return self", "response": "Fires a specified DOM event on the current node."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef load_all(self):\n try:\n self.toi.load_all()\n self.rosters.load_all()\n #self.summary.load_all()\n self.play_by_play.load_all()\n self.face_off_comp.load_all()\n return self\n except Exception as e:\n print(e)\n return None", "response": "Loads all reports in the system."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the game meta information displayed in report banners including team names away team date and location and attendance.", "response": "def matchup(self):\n \"\"\"\n Return the game meta information displayed in report banners including team names,\n final score, game date, location, and attendance. Data format is\n \n .. code:: python\n \n {\n 'home': home,\n 'away': away,\n 'final': final,\n 'attendance': att,\n 'date': date,\n 'location': loc\n }\n \n :returns: matchup banner info\n :rtype: dict\n \"\"\"\n if self.play_by_play.matchup:\n return self.play_by_play.matchup\n elif self.rosters.matchup:\n return self.rosters.matchup\n elif self.toi.matchup:\n return self.toi.matchup\n else:\n self.face_off_comp.matchup"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _utf8_encode(self, d):\n for k, v in d.items():\n if isinstance(v, str):\n d[k] = v.encode('utf8').lower()\n if isinstance(v, list):\n for index,item in enumerate(v):\n item = item.encode('utf8').lower()\n v[index] = item\n if isinstance(v, dict):\n d[k] = self._utf8_encode(v)\n \n return d", "response": "Ensures all values are encoded in UTF - 8 and converts them to lowercase"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _bool_encode(self, d):\n for k, v in d.items():\n if isinstance(v, bool):\n d[k] = str(v).lower()\n \n return d", "response": "Converts bool values to lowercase strings\n "} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nformats search parameters and values for use with API", "response": "def _options(self, **kwargs):\n \"\"\"\n Formats search parameters/values for use with API\n \n :param \\*\\*kwargs: search parameters/values\n \n \"\"\"\n def _format_fq(d):\n for k,v in d.items():\n if isinstance(v, list):\n d[k] = ' '.join(map(lambda x: '\"' + x + '\"', v))\n else:\n d[k] = '\"' + v + '\"'\n values = []\n for k,v in d.items():\n value = '%s:(%s)' % (k,v)\n values.append(value)\n values = ' AND '.join(values)\n return values\n \n kwargs = self._utf8_encode(kwargs)\n kwargs = self._bool_encode(kwargs)\n \n values = ''\n \n for k, v in kwargs.items():\n if k is 'fq' and isinstance(v, dict):\n v = _format_fq(v)\n elif isinstance(v, list):\n v = ','.join(v)\n values += '%s=%s&' % (k, v)\n \n return values"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalls the API and returns a dictionary of the search results", "response": "def search(self, \n response_format = None, \n key = None, \n **kwargs):\n \"\"\"\n Calls the API and returns a dictionary of the search results\n \n :param response_format: the format that the API uses for its response, \n includes JSON (.json) and JSONP (.jsonp). \n Defaults to '.json'.\n \n :param key: a developer key. Defaults to key given when the articleAPI class was initialized.\n \n \"\"\"\n if response_format is None:\n response_format = self.response_format\n if key is None:\n key = self.key\n \n url = '%s%s?%sapi-key=%s' % (\n API_ROOT, response_format, self._options(**kwargs), key\n )\n \n r = requests.get(url)\n return r.json()"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse(self):\n \n r = super(GameSummRep, self).parse()\n try:\n self.parse_scoring_summary()\n return r and False\n except:\n return False", "response": "Fully parses game summary report."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate an event object corresponding to the given event type.", "response": "def Create(event_type):\n \"\"\"\n Factory method creates objects derived from :py:class`.Event` with class name matching the :py:class`.EventType`.\n \n :param event_type: number for type of event\n :returns: constructed event corresponding to ``event_type``\n :rtype: :py:class:`.Event`\n \"\"\"\n if event_type in EventType.Name:\n # unknown event type gets base class\n if EventType.Name[event_type] == Event.__name__:\n return Event()\n else:\n # instantiate Event subclass with same name as EventType name\n return [t for t in EventFactory.event_list if t.__name__ == EventType.Name[event_type]][0]()\n else:\n raise TypeError(\"EventFactory.Create: Invalid EventType\")"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a shift summary object for the home team", "response": "def home_shift_summ(self):\n \"\"\"\n :returns: :py:class:`.ShiftSummary` by player for the home team\n :rtype: dict ``{ player_num: shift_summary_obj }``\n \"\"\"\n if not self.__wrapped_home:\n self.__wrapped_home = self.__wrap(self._home.by_player)\n \n return self.__wrapped_home"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a dictionary of shift summary objects for the away team", "response": "def away_shift_summ(self):\n \"\"\"\n :returns: :py:class:`.ShiftSummary` by player for the away team\n :rtype: dict ``{ player_num: shift_summary_obj }``\n \"\"\"\n if not self.__wrapped_away:\n self.__wrapped_away = self.__wrap(self._away.by_player)\n \n return self.__wrapped_away"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse(self):\n\n try:\n return super(RosterRep, self).parse() \\\n .parse_rosters() \\\n .parse_scratches() \\\n .parse_coaches() \\\n .parse_officials()\n except:\n return None", "response": "Retreive and parse Play by Play data for the given game key."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse the home and away game rosters and return self on success None on failure", "response": "def parse_rosters(self):\n \"\"\"\n Parse the home and away game rosters\n\n :returns: ``self`` on success, ``None`` otherwise\n \"\"\"\n lx_doc = self.html_doc()\n\n if not self.__blocks:\n self.__pl_blocks(lx_doc)\n\n for t in ['home', 'away']:\n self.rosters[t] = self.__clean_pl_block(self.__blocks[t])\n\n return self if self.rosters else None"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse the home and away healthy scratches", "response": "def parse_scratches(self):\n \"\"\"\n Parse the home and away healthy scratches\n\n :returns: ``self`` on success, ``None`` otherwise\n \"\"\"\n lx_doc = self.html_doc()\n if not self.__blocks:\n self.__pl_blocks(lx_doc)\n\n for t in ['aw_scr', 'h_scr']:\n ix = 'away' if t == 'aw_scr' else 'home'\n self.scratches[ix] = self.__clean_pl_block(self.__blocks[t])\n\n return self if self.scratches else None"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nparse the home and away coaches.", "response": "def parse_coaches(self):\n \"\"\"\n Parse the home and away coaches\n\n :returns: ``self`` on success, ``None`` otherwise\n \"\"\"\n lx_doc = self.html_doc()\n tr = lx_doc.xpath('//tr[@id=\"HeadCoaches\"]')[0]\n\n for i, td in enumerate(tr):\n txt = td.xpath('.//text()')\n txt = ex_junk(txt, ['\\n','\\r'])\n team = 'away' if i == 0 else 'home'\n self.coaches[team] = txt[0]\n\n return self if self.coaches else None"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_officials(self):\n # begin proper body of method\n lx_doc = self.html_doc()\n off_parser = opm(self.game_key.season)\n self.officials = off_parser(lx_doc)\n\n return self if self.officials else None", "response": "Parse the officials tag of the current object."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nvalidating an email address.", "response": "def is_email(address, check_dns=False, diagnose=False):\n \"\"\"Validate an email address.\n\n Keyword arguments:\n address --- the email address as a string\n check_dns --- flag for whether to check the DNS status of the domain\n diagnose --- flag for whether to return True/False or a Diagnosis\n\n \"\"\"\n\n threshold = BaseDiagnosis.CATEGORIES[\"THRESHOLD\"]\n d = ParserValidator().is_email(address, True)\n if check_dns is True and d < BaseDiagnosis.CATEGORIES[\"DNSWARN\"]:\n threshold = BaseDiagnosis.CATEGORIES[\"VALID\"]\n d = max(d, DNSValidator().is_valid(address.split(\"@\")[1], True))\n\n return d if diagnose else d < threshold"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dispatch_loader(scraper, loader_name):\n l = '.'.join([scraper, loader_name])\n def wrapper(f):\n @wraps(f)\n def wrapped(self, *f_args, **f_kwargs):\n if not hasattr(self, '_loaded'):\n self._loaded = { }\n \n already_loaded = self._loaded.setdefault(l, False)\n if not already_loaded:\n attr = getattr(self, scraper)\n self._loaded[l] = getattr(attr, loader_name)() is not None\n return f(self, *f_args, **f_kwargs)\n return wrapped\n return wrapper", "response": "Decorator that dispatches a report loader to the specified scraper."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing shifts from TOI report and return a dictionary of shift - > dict.", "response": "def parse_shifts(self):\n \"\"\"\n Parse shifts from TOI report\n \n :returns: self if successfule else None\n \"\"\"\n \n lx_doc = self.html_doc()\n pl_heads = lx_doc.xpath('//td[contains(@class, \"playerHeading\")]')\n for pl in pl_heads:\n sh_sum = { }\n \n pl_text = pl.xpath('text()')[0]\n num_name = pl_text.replace(',','').split(' ')\n sh_sum['player_num'] = int(num_name[0]) if num_name[0].isdigit() else -1\n sh_sum['player_name'] = { 'first': num_name[2], 'last': num_name[1] }\n \n first_shift = pl.xpath('../following-sibling::tr')[1]\n sh_sum['shifts'], last_shift = self.__player_shifts(first_shift)\n \n while ('Per' not in last_shift.xpath('.//text()')):\n last_shift = last_shift.xpath('following-sibling::tr')[0]\n \n per_summ = last_shift.xpath('.//tr')[0]\n sh_sum['by_period'], last_sum = self.__get_by_per_summ(per_summ)\n \n \n self.by_player[sh_sum['player_num']] = sh_sum\n \n return self if self.by_player else None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck whether a domain is a valid MX or A record.", "response": "def is_valid(self, domain, diagnose=False):\n\n \"\"\"Check whether a domain has a valid MX or A record.\n\n Keyword arguments:\n domain --- the domain to check\n diagnose --- flag to report a diagnosis or a boolean (default False)\n\n \"\"\"\n\n return_status = [ValidDiagnosis()]\n dns_checked = False\n\n # http://tools.ietf.org/html/rfc5321#section-2.3.5\n # Names that can be resolved to MX RRs or address (i.e., A or AAAA)\n # RRs (as discussed in Section 5) are permitted, as are CNAME RRs\n # whose targets can be resolved, in turn, to MX or address RRs.\n #\n # http://tools.ietf.org/html/rfc5321#section-5.1\n # The lookup first attempts to locate an MX record associated with\n # the name. If a CNAME record is found, the resulting name is\n # processed as if it were the initial name. ... If an empty list of\n # MXs is returned, the address is treated as if it was associated\n # with an implicit MX RR, with a preference of 0, pointing to that\n # host.\n #\n # is_email() author's note: We will regard the existence of a CNAME to\n # be sufficient evidence of the domain's existence. For performance\n # reasons we will not repeat the DNS lookup for the CNAME's target, but\n # we will raise a warning because we didn't immediately find an MX\n # record.\n try:\n dns.resolver.query(domain, 'MX')\n dns_checked = True\n except (dns.resolver.NXDOMAIN, dns.name.NameTooLong):\n # Domain can't be found in DNS\n return_status.append(DNSDiagnosis('NO_RECORD'))\n\n # Since dns.resolver gives more information than the PHP analog, we\n # can say that TLDs that throw an NXDOMAIN or NameTooLong error\n # have been checked\n if len(domain.split('.')) == 1:\n dns_checked = True\n except dns.resolver.NoAnswer:\n # MX-record for domain can't be found\n return_status.append(DNSDiagnosis('NO_MX_RECORD'))\n\n try:\n # TODO: See if we can/need to narrow to A / CNAME\n dns.resolver.query(domain)\n except dns.resolver.NoAnswer:\n # No usable records for the domain can be found\n return_status.append(DNSDiagnosis('NO_RECORD'))\n except dns.resolver.NoNameservers:\n return_status.append(DNSDiagnosis('NO_NAMESERVERS'))\n except (dns.exception.Timeout, dns.resolver.Timeout):\n return_status.append(DNSDiagnosis('DNS_TIMEDOUT'))\n\n # Check for TLD addresses\n # -----------------------\n # TLD addresses are specifically allowed in RFC 5321 but they are\n # unusual to say the least. We will allocate a separate\n # status to these addresses on the basis that they are more likely\n # to be typos than genuine addresses (unless we've already\n # established that the domain does have an MX record)\n #\n # http://tools.ietf.org/html/rfc5321#section-2.3.5\n # In the case of a top-level domain used by itself in an address, a\n # single string is used without any dots. This makes the requirement,\n # described in more detail below, that only fully-qualified domain\n # names appear in SMTP transactions on the public Internet,\n # particularly important where top-level domains are involved.\n #\n # TLD format\n # ----------\n # The format of TLDs has changed a number of times. The standards\n # used by IANA have been largely ignored by ICANN, leading to\n # confusion over the standards being followed. These are not defined\n # anywhere, except as a general component of a DNS host name (a label).\n # However, this could potentially lead to 123.123.123.123 being a\n # valid DNS name (rather than an IP address) and thereby creating\n # an ambiguity. The most authoritative statement on TLD formats that\n # the author can find is in a (rejected!) erratum to RFC 1123\n # submitted by John Klensin, the author of RFC 5321:\n #\n # http://www.rfc-editor.org/errata_search.php?rfc=1123&eid=1353\n # However, a valid host name can never have the dotted-decimal\n # form #.#.#.#, since this change does not permit the highest-level\n # component label to start with a digit even if it is not\n # all-numeric.\n if not dns_checked:\n atom_list = domain.split(\".\")\n if len(atom_list) == 1:\n return_status.append(RFC5321Diagnosis('TLD'))\n\n try:\n float(atom_list[len(atom_list)-1][0])\n return_status.append(RFC5321Diagnosis('TLDNUMERIC'))\n except ValueError:\n pass\n\n final_status = max(return_status)\n\n return final_status if diagnose else final_status == ValidDiagnosis()"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef html_doc(self):\n \n if self.__lx_doc is None:\n cn = NHLCn()\n \n if hasattr(cn, self.report_type):\n html = getattr(cn, self.report_type)(self.game_key)\n else:\n raise ValueError('Invalid report type: %s' % self.report_type)\n \n if cn.req_err is None:\n self.__lx_doc = fromstring(html)\n else:\n self.req_err = cn.req_err\n \n return self.__lx_doc", "response": "returns the lxml processed html document"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing the banner matchup meta info for the game.", "response": "def parse_matchup(self):\n \"\"\"\n Parse the banner matchup meta info for the game.\n \n :returns: ``self`` on success or ``None``\n \"\"\"\n lx_doc = self.html_doc()\n try:\n if not self.matchup:\n self.matchup = self._fill_meta(lx_doc)\n return self\n except:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngenerates and yield a stream of parsed plays. Useful for per play processing.", "response": "def parse_plays_stream(self):\n \"\"\"Generate and yield a stream of parsed plays. Useful for per play processing.\"\"\"\n \n lx_doc = self.html_doc()\n if lx_doc is not None:\n parser = PlayParser(self.game_key.season, self.game_key.game_type)\n plays = lx_doc.xpath('//tr[@class = \"evenColor\"]')\n for p in plays:\n p_obj = parser.build_play(p)\n self.plays.append(p_obj)\n \n yield p_obj"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ColMap(season):\n if c.MIN_SEASON <= season <= c.MAX_SEASON:\n return {\n \"play_num\": 0,\n \"per\": 1,\n \"str\": 2,\n \"time\": 3,\n \"event\": 4,\n \"desc\": 5,\n \"vis\": 6,\n \"home\": 7\n }\n else:\n raise ValueError(\"RTSSCol.MAP(season): Invalid season \" + str(season))", "response": "Returns a dictionary mapping the type of information in the RTSS play row to the appropriate column number."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef build_play(self, pbp_row):\n d = pbp_row.findall('./td')\n c = PlayParser.ColMap(self.season)\n \n p = { }\n to_dig = lambda t: int(t) if t.isdigit() else 0\n p['play_num'] = to_int(d[c[\"play_num\"]].text, 0)\n p['period'] = to_int(d[c[\"per\"]].text, 0)\n \n p['strength'] = self.__strength(d[c[\"str\"]].text)\n \n time = d[c[\"time\"]].text.split(\":\")\n p['time'] = { \"min\": int(time[0]), \"sec\": int(time[1]) }\n \n skater_tab = d[c[\"vis\"]].xpath(\"./table\")\n p['vis_on_ice'] = self.__skaters(skater_tab[0][0]) if len(skater_tab) else { }\n \n skater_tab = d[c[\"home\"]].xpath(\"./table\")\n p['home_on_ice'] = self.__skaters(skater_tab[0][0]) if len(skater_tab) else { }\n \n p['event'] = event_type_mapper(\n d[c[\"event\"]].text,\n period=p['period'],\n skater_ct=len(p['vis_on_ice']) + len(p['home_on_ice']),\n game_type=self.game_type\n )\n p['event'].desc = \" \".join([t.encode('ascii', 'replace').decode('utf-8') for t in d[c[\"desc\"]].xpath(\"text()\")])\n\n parse_event_desc(p['event'], season=self.season)\n \n return p", "response": "Parses the RTSS table row and returns a dict of the data."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconstructing a dictionary of players on the ice in the provided table at the time of play.", "response": "def __skaters(self, tab):\n \"\"\"\n Constructs dictionary of players on the ice in the provided table at time of play.\n :param tab: RTSS table of the skaters and goalie on at the time of the play\n :rtype: dictionary, key = player number, value = [position, name]\n \"\"\"\n \n res = { }\n for td in tab.iterchildren():\n if len(td):\n pl_data = td.xpath(\"./table/tr\")\n pl = pl_data[0].xpath(\"./td/font\")\n \n if pl[0].text.isdigit():\n res[int(pl[0].text)] = [s.strip() for s in pl[0].get(\"title\").split(\"-\")][::-1]\n \n s = pl[0].get(\"title\").split(\"-\")\n pos = pl_data[1].getchildren()[0].text\n \n return res"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef exclude_from(l, containing = [], equal_to = []):\n \n cont = lambda li: any(c in li for c in containing)\n eq = lambda li: any(e == li for e in equal_to)\n return [li for li in l if not (cont(li) or eq(li))]", "response": "Exclude elements in list l containing any elements from list l ex.\n "} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef calc_surfdist(surface, labels, annot, reg, origin, target):\n import nibabel as nib\n import numpy as np\n import os\n from surfdist import load, utils, surfdist\n import csv\n\n \"\"\" inputs:\n surface - surface file (e.g. lh.pial, with full path)\n labels - label file (e.g. lh.cortex.label, with full path)\n annot - annot file (e.g. lh.aparc.a2009s.annot, with full path)\n reg - registration file (lh.sphere.reg)\n origin - the label from which we calculate distances\n target - target surface (e.g. fsaverage4)\n \"\"\"\n\n # Load stuff\n surf = nib.freesurfer.read_geometry(surface)\n cort = np.sort(nib.freesurfer.read_label(labels))\n src = load.load_freesurfer_label(annot, origin, cort)\n\n # Calculate distances\n dist = surfdist.dist_calc(surf, cort, src)\n\n # Project distances to target\n trg = nib.freesurfer.read_geometry(target)[0]\n native = nib.freesurfer.read_geometry(reg)[0]\n idx_trg_to_native = utils.find_node_match(trg, native)[0]\n\n # Get indices in trg space \n distt = dist[idx_trg_to_native]\n \n # Write to file and return file handle\n filename = os.path.join(os.getcwd(),'distances.csv')\n distt.tofile(filename,sep=\",\")\n\n return filename", "response": "Calculate distances between a surface and a registration."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef stack_files(files, hemi, source, target):\n import csv\n import os\n import numpy as np\n\n fname = \"sdist_%s_%s_%s.csv\" % (hemi, source, target)\n filename = os.path.join(os.getcwd(),fname)\n\n alldist = []\n\n for dfile in files:\n alldist.append(np.genfromtxt(dfile, delimiter=','))\n\n alldist = np.array(alldist)\n alldist.tofile(filename,\",\")\n\n return filename", "response": "This function takes a list of files as input and vstacks them together."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets short URL of blog post.", "response": "def get_short_url(self, obj):\n \"\"\"\n Get short URL of blog post like '/blog//' using ``get_absolute_url`` if available.\n Removes dependency on reverse URLs of Mezzanine views when deploying Mezzanine only as an API backend.\n \"\"\"\n try:\n url = obj.get_absolute_url()\n except NoReverseMatch:\n url = '/blog/' + obj.slug\n return url"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef head_to_head(self, home_num, away_num):\n if home_num in self.home_fo and away_num in self.home_fo[home_num]['opps']:\n h_fo = self.home_fo[home_num]['opps'][away_num]\n a_fo = self.away_fo[away_num]['opps'][home_num]\n return {\n 'home': { k: h_fo[k] for k in self.__zones },\n 'away': { k: a_fo[k] for k in self.__zones }\n }\n else:\n return { }", "response": "Return the head - to - head face - off outcomes between two players."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the overall faceoff win / total breakdown for home and away as", "response": "def team_totals(self):\n \"\"\"\n Returns the overall faceoff win/total breakdown for home and away as\n \n :returns: dict, ``{ 'home/away': { 'won': won, 'total': total } }``\n \"\"\"\n if self.__team_tots is None:\n self.__team_tots = self.__comp_tot()\n \n return {\n t: self.__team_tots[t]['all']\n for t in [ 'home', 'away' ]\n }"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the faceoff win / total breakdown by zone for home and away as obi.", "response": "def by_zone(self):\n \"\"\"\n Returns the faceoff win/total breakdown by zone for home and away as\n \n .. code:: python\n \n { 'home/away': {\n 'off/def/neut/all': { 'won': won, 'total': total }\n }\n }\n \n :returns: dict\n \"\"\"\n if self.__team_tots is None:\n self.__team_tots = self.__comp_tot()\n \n return {\n t: {\n z: self.__team_tots[t][z]\n for z in self.__zones\n if z != 'all'\n }\n for t in [ 'home', 'away' ]\n }"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fo_pct(self):\n tots = self.team_totals\n return {\n t: tots[t]['won']/(1.0*tots[t]['total']) if tots[t]['total'] else 0.0\n for t in [ 'home', 'away' ]\n }", "response": "Returns the fo percentage of team overall face - off win %.\n \n "} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fo_pct_by_zone(self):\n bz = self.by_zone\n return {\n t: {\n z: bz[t][z]['won']/(1.0*bz[t][z]['total']) if bz[t][z]['total'] else 0.0\n for z in self.__zones\n if z != 'all'\n }\n for t in [ 'home', 'away' ]\n \n }", "response": "Get the fo percent by zone. Format is\n \n "} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nupdates the accumulator with the current play ecords.", "response": "def update(self, play):\n \"\"\"\n Update the accumulator with the current play\n \n :returns: new tally\n :rtype: dict, ``{ 'period': per, 'time': clock, 'team': cumul, 'play': play }``\n \"\"\"\n new_tally = { }\n #if any(isinstance(play.event, te) for te in self.trigger_event_types):\n if self._count_play(play):\n # the team who made the play / triggered the event\n \n team = self._get_team(play)\n\n try:\n self.total[team] += 1\n except:\n self.total[team] = 1\n self.teams.append(team)\n for i in range(len(self.tally)):\n self.tally[i][team] = 0\n \n try:\n new_tally = { k:v for k,v in self.tally[len(self.tally)-1].items() }\n new_tally['period'] = play.period\n new_tally['time'] = play.time\n new_tally[team] += 1\n new_tally['play'] = play\n except:\n new_tally = {\n 'period': play.period,\n 'time': play.time,\n team: 1,\n 'play': play\n }\n \n self.tally.append(new_tally)\n \n return new_tally"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a dictionary of the Cori - share for each team tier tier", "response": "def share(self):\n \"\"\"\n The Cori-share (% of shot attempts) for each team\n \n :returns: dict, ``{ 'home_name': %, 'away_name': % }``\n \"\"\"\n tot = sum(self.total.values())\n return { k: v/float(tot) for k,v in self.total.items() }"} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncomputes the stats defined in self. cum_stats.", "response": "def compute_stats(self):\n \"\"\"\n Compute the stats defined in ``self.cum_stats``.\n \n :returns: collection of all computed :py:class:`.AccumulateStats`\n :rtype: dict\n \"\"\"\n if not self.__have_stats:\n if self.init_cs_teams and self.cum_stats:\n self.__init_cs_teams()\n \n for play in self._rep_reader.parse_plays_stream():\n p = Play(**play)\n self.__wrapped_plays.append(p)\n if self.cum_stats:\n self.__process(p, self.cum_stats, 'update')\n self.__have_stats = True\n \n return self.cum_stats"} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __html_rep(self, game_key, rep_code):\n seas, gt, num = game_key.to_tuple()\n url = [ self.__domain, \"scores/htmlreports/\", str(seas-1), str(seas),\n \"/\", rep_code, \"0\", str(gt), (\"%04i\" % (num)), \".HTM\" ]\n url = ''.join(url)\n\n return self.__open(url)", "response": "Retrieves the nhl html reports for the specified game and report code"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef to_char(token):\n if ord(token) in _range(9216, 9229 + 1):\n token = _unichr(ord(token) - 9216)\n\n return token", "response": "Transforms the ASCII control character symbols to their real char."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck that an address conforms to RFCs 5321 5322 and others.", "response": "def is_email(self, address, diagnose=False):\n \"\"\"Check that an address address conforms to RFCs 5321, 5322 and others.\n\n More specifically, see the follow RFCs:\n * http://tools.ietf.org/html/rfc5321\n * http://tools.ietf.org/html/rfc5322\n * http://tools.ietf.org/html/rfc4291#section-2.2\n * http://tools.ietf.org/html/rfc1123#section-2.1\n * http://tools.ietf.org/html/rfc3696) (guidance only)\n\n Keyword arguments:\n address -- address to check.\n diagnose -- flag to report a diagnosis or a boolean (default False)\n\n \"\"\"\n\n threshold = BaseDiagnosis.CATEGORIES['VALID']\n return_status = [ValidDiagnosis()]\n parse_data = {}\n\n # Parse the address into components, character by character\n raw_length = len(address)\n context = Context.LOCALPART # Where we are\n context_stack = [context] # Where we've been\n context_prior = Context.LOCALPART # Where we just came from\n token = '' # The current character\n token_prior = '' # The previous character\n parse_data[Context.LOCALPART] = '' # The address' components\n parse_data[Context.DOMAIN] = ''\n atom_list = {\n Context.LOCALPART: [''],\n Context.DOMAIN: ['']\n } # The address' dot-atoms\n element_count = 0\n element_len = 0\n hyphen_flag = False # Hyphen cannot occur at the end of a subdomain\n end_or_die = False # CFWS can only appear at the end of an element\n skip = False # Skip flag that simulates i++\n crlf_count = -1 # crlf_count = -1 == !isset(crlf_count)\n\n for i in _range(raw_length):\n\n # Skip simulates the use of ++ operator\n if skip:\n skip = False\n continue\n\n token = address[i]\n token = to_char(token)\n\n # Switch to simulate decrementing; needed for FWS\n repeat = True\n\n while repeat:\n repeat = False\n\n # -------------------------------------------------------\n # Local part\n # -------------------------------------------------------\n if context == Context.LOCALPART:\n # http://tools.ietf.org/html/rfc5322#section-3.4.1\n # local-part = dot-atom / quoted-string /\n # obs-local-part\n #\n # dot-atom = [CFWS] dot-atom-text [CFWS]\n #\n # dot-atom-text = 1*atext *(\".\" 1*atext)\n #\n # quoted-string = [CFWS]\n # DQUOTE *([FWS] qcontent) [FWS] DQUOTE\n # [CFWS]\n #\n # obs-local-part = word *(\".\" word)\n #\n # word = atom / quoted-string\n #\n # atom = [CFWS] 1*atext [CFWS]\n if token == Char.OPENPARENTHESIS:\n if element_len == 0:\n # Comments are OK at the beginning of an element\n if element_count == 0:\n return_status.append(CFWSDiagnosis('COMMENT'))\n else:\n return_status.append(\n DeprecatedDiagnosis('COMMENT'))\n else:\n return_status.append(CFWSDiagnosis('COMMENT'))\n # We can't start a comment in the middle of an\n # element, so this better be the end\n end_or_die = True\n\n context_stack.append(context)\n context = Context.COMMENT\n elif token == Char.DOT:\n if element_len == 0:\n # Another dot, already? Fatal error\n if element_count == 0:\n return_status.append(\n InvalidDiagnosis('DOT_START'))\n else:\n return_status.append(\n InvalidDiagnosis('CONSECUTIVEDOTS'))\n else:\n # The entire local-part can be a quoted string for\n # RFC 5321. If it's just one atom that is quoted\n # then it's an RFC 5322 obsolete form\n if end_or_die:\n return_status.append(\n DeprecatedDiagnosis('LOCALPART'))\n\n # CFWS & quoted strings are OK again now we're at\n # the beginning of an element (although they are\n # obsolete forms)\n end_or_die = False\n element_len = 0\n element_count += 1\n parse_data[Context.LOCALPART] += token\n atom_list[Context.LOCALPART].append('')\n elif token == Char.DQUOTE:\n if element_len == 0:\n # The entire local-part can be a quoted string for\n # RFC 5321. If it's just one atom that is quoted\n # then it's an RFC 5322 obsolete form\n if element_count == 0:\n return_status.append(\n RFC5321Diagnosis('QUOTEDSTRING'))\n else:\n return_status.append(\n DeprecatedDiagnosis('LOCALPART'))\n\n parse_data[Context.LOCALPART] += token\n atom_list[Context.LOCALPART][element_count] += token\n element_len += 1\n end_or_die = True\n context_stack.append(context)\n context = Context.QUOTEDSTRING\n else:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('EXPECTING_ATEXT'))\n # Folding White Space (FWS)\n elif token in [Char.CR, Char.SP, Char.HTAB]:\n # Skip simulates the use of ++ operator if the latter\n # check doesn't short-circuit\n if token == Char.CR:\n skip = True\n\n if (i+1 == raw_length or\n to_char(address[i+1]) != Char.LF):\n return_status.append(\n InvalidDiagnosis('CR_NO_LF'))\n break\n\n if element_len == 0:\n if element_count == 0:\n return_status.append(CFWSDiagnosis('FWS'))\n else:\n return_status.append(\n DeprecatedDiagnosis('FWS'))\n else:\n # We can't start FWS in the middle of an element,\n # so this better be the end\n end_or_die = True\n\n context_stack.append(context)\n context = Context.FWS\n token_prior = token\n # @\n elif token == Char.AT:\n # At this point we should have a valid local-part\n if len(context_stack) != 1: # pragma: no cover\n if diagnose:\n return InvalidDiagnosis('BAD_PARSE')\n else:\n return False\n\n if parse_data[Context.LOCALPART] == '':\n # Fatal error\n return_status.append(\n InvalidDiagnosis('NOLOCALPART'))\n elif element_len == 0:\n # Fatal error\n return_status.append(InvalidDiagnosis('DOT_END'))\n # http://tools.ietf.org/html/rfc5321#section-4.5.3.1.1\n # The maximum total length of a user name or other\n # local-part is 64 octets.\n elif len(parse_data[Context.LOCALPART]) > 64:\n return_status.append(\n RFC5322Diagnosis('LOCAL_TOOLONG'))\n # http://tools.ietf.org/html/rfc5322#section-3.4.1\n # Comments and folding white space\n # SHOULD NOT be used around the \"@\" in the addr-spec.\n #\n # http://tools.ietf.org/html/rfc2119\n # 4. SHOULD NOT This phrase, or the phrase \"NOT\n # RECOMMENDED\" mean that there may exist valid\n # reasons in particular circumstances when the\n # particular behavior is acceptable or even useful,\n # but the full implications should be understood and\n # the case carefully weighed before implementing any\n # behavior described with this label.\n elif context_prior in [Context.COMMENT, Context.FWS]:\n return_status.append(\n DeprecatedDiagnosis('CFWS_NEAR_AT'))\n\n # Clear everything down for the domain parsing\n context = Context.DOMAIN\n context_stack = []\n element_count = 0\n element_len = 0\n # CFWS can only appear at the end of the element\n end_or_die = False\n # atext\n else:\n # http://tools.ietf.org/html/rfc5322#section-3.2.3\n # atext = ALPHA / DIGIT / ; Printable US-ASCII\n # \"!\" / \"#\" / ; characters not\n # \"$\" / \"%\" / ; including specials.\n # \"&\" / \"'\" / ; Used for atoms.\n # \"*\" / \"+\" /\n # \"-\" / \"/\" /\n # \"=\" / \"?\" /\n # \"^\" / \"_\" /\n # \"`\" / \"{\" /\n # \"|\" / \"}\" /\n # \"~\"\n if end_or_die:\n # We have encountered atext where it is no longer\n # valid\n if context_prior in [Context.COMMENT, Context.FWS]:\n return_status.append(\n InvalidDiagnosis('ATEXT_AFTER_CFWS'))\n elif context_prior == Context.QUOTEDSTRING:\n return_status.append(\n InvalidDiagnosis('ATEXT_AFTER_QS'))\n else: # pragma: no cover\n if diagnose:\n return InvalidDiagnosis('BAD_PARSE')\n else:\n return False\n else:\n context_prior = context\n o = ord(token)\n\n if (o < 33 or o > 126 or o == 10 or\n token in Char.SPECIALS):\n return_status.append(\n InvalidDiagnosis('EXPECTING_ATEXT'))\n\n parse_data[Context.LOCALPART] += token\n atom_list[Context.LOCALPART][element_count] += token\n element_len += 1\n # -------------------------------------------------------\n # Domain\n # -------------------------------------------------------\n elif context == Context.DOMAIN:\n # http://tools.ietf.org/html/rfc5322#section-3.4.1\n # domain = dot-atom / domain-literal / obs-domain\n #\n # dot-atom = [CFWS] dot-atom-text [CFWS]\n #\n # dot-atom-text = 1*atext *(\".\" 1*atext)\n #\n # domain-literal = [CFWS]\n # \"[\" *([FWS] dtext) [FWS] \"]\"\n # [CFWS]\n #\n # dtext = %d33-90 / ; Printable US-ASCII\n # %d94-126 / ; characters not\n # obs-dtext ; including [, ], or \\\n #\n # obs-domain = atom *(\".\" atom)\n #\n # atom = [CFWS] 1*atext [CFWS]\n #\n #\n # http://tools.ietf.org/html/rfc5321#section-4.1.2\n # Mailbox = Local-part\n # \"@\"\n # ( Domain / address-literal )\n #\n # Domain = sub-domain *(\".\" sub-domain)\n #\n # address-literal = \"[\" ( IPv4-address-literal /\n # IPv6-address-literal /\n # General-address-literal ) \"]\"\n # ; See Section 4.1.3\n #\n # http://tools.ietf.org/html/rfc5322#section-3.4.1\n # Note: A liberal syntax for the domain portion of\n # addr-spec is given here. However, the domain portion\n # contains addressing information specified by and\n # used in other protocols (e.g., RFC 1034, RFC 1035,\n # RFC 1123, RFC5321). It is therefore incumbent upon\n # implementations to conform to the syntax of\n # addresse for the context in which they are used.\n # is_email() author's note: it's not clear how to interpret\n # this in the context of a general address address\n # validator. The conclusion I have reached is this:\n # \"addressing information\" must comply with RFC 5321 (and\n # in turn RFC 1035), anything that is \"semantically\n # invisible\" must comply only with RFC 5322.\n\n # Comment\n if token == Char.OPENPARENTHESIS:\n if element_len == 0:\n # Comments at the start of the domain are\n # deprecated in the text\n # Comments at the start of a subdomain are\n # obs-domain\n # (http://tools.ietf.org/html/rfc5322#section-3.4.1)\n if element_count == 0:\n return_status.append(\n DeprecatedDiagnosis('CFWS_NEAR_AT'))\n else:\n return_status.append(\n DeprecatedDiagnosis('COMMENT'))\n else:\n return_status.append(CFWSDiagnosis('COMMENT'))\n # We can't start a comment in the middle of an\n # element, so this better be the end\n end_or_die = True\n\n context_stack.append(context)\n context = Context.COMMENT\n # Next dot-atom element\n elif token == Char.DOT:\n if element_len == 0:\n # Another dot, already? Fatal error\n if element_count == 0:\n return_status.append(\n InvalidDiagnosis('DOT_START'))\n else:\n return_status.append(\n InvalidDiagnosis('CONSECUTIVEDOTS'))\n elif hyphen_flag:\n # Previous subdomain ended in a hyphen. Fatal error\n return_status.append(\n InvalidDiagnosis('DOMAINHYPHENEND'))\n else:\n # Nowhere in RFC 5321 does it say explicitly that\n # the domain part of a Mailbox must be a valid\n # domain according to the DNS standards set out in\n # RFC 1035, but this *is* implied in several\n # places. For instance, wherever the idea of host\n # routing is discussed the RFC says that the domain\n # must be looked up in the DNS. This would be\n # nonsense unless the domain was designed to be a\n # valid DNS domain. Hence we must conclude that the\n # RFC 1035 restriction on label length also applies\n # to RFC 5321 domains.\n #\n # http://tools.ietf.org/html/rfc1035#section-2.3.4\n # labels 63 octets or less\n if element_len > 63:\n return_status.append(\n RFC5322Diagnosis('LABEL_TOOLONG'))\n\n # CFWS is OK again now we're at the beginning of an\n # element (although it may be obsolete CFWS)\n end_or_die = False\n element_len = 0\n element_count += 1\n atom_list[Context.DOMAIN].append('')\n parse_data[Context.DOMAIN] += token\n # Domain literal\n elif token == Char.OPENSQBRACKET:\n if parse_data[Context.DOMAIN] == '':\n # Domain literal must be the only component\n end_or_die = True\n element_len += 1\n context_stack.append(context)\n context = Context.LITERAL\n parse_data[Context.DOMAIN] += token\n atom_list[Context.DOMAIN][element_count] += token\n parse_data['literal'] = ''\n else:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('EXPECTING_ATEXT'))\n\n # Folding White Space (FWS)\n elif token in [Char.CR, Char.SP, Char.HTAB]:\n # Skip simulates the use of ++ operator if the latter\n # check doesn't short-circuit\n if token == Char.CR:\n skip = True\n\n if i+1 == raw_length or (to_char(address[i + 1]) !=\n Char.LF):\n # Fatal error\n return_status.append(\n InvalidDiagnosis('CR_NO_LF'))\n break\n\n if element_len == 0:\n if element_count == 0:\n return_status.append(\n DeprecatedDiagnosis('CFWS_NEAR_AT'))\n else:\n return_status.append(\n DeprecatedDiagnosis('FWS'))\n else:\n return_status.append(CFWSDiagnosis('FWS'))\n # We can't start FWS in the middle of an element,\n # so this better be the end\n end_or_die = True\n\n context_stack.append(context)\n context = Context.FWS\n token_prior = token\n # atext\n else:\n # RFC 5322 allows any atext...\n # http://tools.ietf.org/html/rfc5322#section-3.2.3\n # atext = ALPHA / DIGIT / ; Printable US-ASCII\n # \"!\" / \"#\" / ; characters not\n # \"$\" / \"%\" / ; including specials.\n # \"&\" / \"'\" / ; Used for atoms.\n # \"*\" / \"+\" /\n # \"-\" / \"/\" /\n # \"=\" / \"?\" /\n # \"^\" / \"_\" /\n # \"`\" / \"{\" /\n # \"|\" / \"}\" /\n # \"~\"\n\n # But RFC 5321 only allows letter-digit-hyphen to\n # comply with DNS rules (RFCs 1034 & 1123)\n # http://tools.ietf.org/html/rfc5321#section-4.1.2\n # sub-domain = Let-dig [Ldh-str]\n #\n # Let-dig = ALPHA / DIGIT\n #\n # Ldh-str = *( ALPHA / DIGIT / \"-\" ) Let-dig\n #\n if end_or_die:\n # We have encountered atext where it is no longer\n # valid\n if context_prior in [Context.COMMENT, Context.FWS]:\n return_status.append(\n InvalidDiagnosis('ATEXT_AFTER_CFWS'))\n elif context_prior == Context.LITERAL:\n return_status.append(\n InvalidDiagnosis('ATEXT_AFTER_DOMLIT'))\n else: # pragma: no cover\n if diagnose:\n return InvalidDiagnosis('BAD_PARSE')\n else:\n return False\n\n o = ord(token)\n # Assume this token isn't a hyphen unless we discover\n # it is\n hyphen_flag = False\n\n if o < 33 or o > 126 or token in Char.SPECIALS:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('EXPECTING_ATEXT'))\n elif token == Char.HYPHEN:\n if element_len == 0:\n # Hyphens can't be at the beginning of a\n # subdomain\n # Fatal error\n return_status.append(\n InvalidDiagnosis('DOMAINHYPHENSTART'))\n\n hyphen_flag = True\n elif not (47 < o < 58 or 64 < o < 91 or 96 < o < 123):\n # Not an RFC 5321 subdomain, but still OK by RFC\n # 5322\n return_status.append(RFC5322Diagnosis('DOMAIN'))\n\n parse_data[Context.DOMAIN] += token\n atom_list[Context.DOMAIN][element_count] += token\n element_len += 1\n # -------------------------------------------------------\n # Domain literal\n # -------------------------------------------------------\n elif context == Context.LITERAL:\n # http://tools.ietf.org/html/rfc5322#section-3.4.1\n # domain-literal = [CFWS]\n # \"[\" *([FWS] dtext) [FWS] \"]\"\n # [CFWS]\n #\n # dtext = %d33-90 / ; Printable US-ASCII\n # %d94-126 / ; characters not\n # obs-dtext ; including [, ], or \\\n #\n # obs-dtext = obs-NO-WS-CTL / quoted-pair\n\n # End of domain literal\n if token == Char.CLOSESQBRACKET:\n if (max(return_status) <\n BaseDiagnosis.CATEGORIES['DEPREC']):\n # Could be a valid RFC 5321 address literal, so\n # let's check\n #\n # http://tools.ietf.org/html/rfc5321#section-4.1.2\n # address-literal = \"[\" ( IPv4-address-literal /\n # IPv6-address-literal /\n # General-address-literal ) \"]\"\n # ; See Section 4.1.3\n #\n # http://tools.ietf.org/html/rfc5321#section-4.1.3\n # IPv4-address-literal = Snum 3(\".\" Snum)\n #\n # IPv6-address-literal = \"IPv6:\" IPv6-addr\n #\n # General-address-literal = Standardized-tag \":\"\n # 1*dcontent\n #\n # Standardized-tag = Ldh-str\n # ; Standardized-tag MUST be\n # ; specified in a\n # ; Standards-Track RFC and\n # ; registered with IANA\n #\n # dcontent = %d33-90 / ; Printable US-ASCII\n # %d94-126 ; excl. \"[\", \"\\\", \"]\"\n #\n # Snum = 1*3DIGIT\n # ; representing a decimal integer\n # ; value in the range 0-255\n #\n # IPv6-addr = IPv6-full / IPv6-comp /\n # IPv6v4-full / IPv6v4-comp\n #\n # IPv6-hex = 1*4HEXDIG\n #\n # IPv6-full = IPv6-hex 7(\":\" IPv6-hex)\n #\n # IPv6-comp = [IPv6-hex *5(\":\" IPv6-hex)]\n # \"::\"\n # [IPv6-hex *5(\":\" IPv6-hex)]\n # ; The \"::\" represents at least 2\n # ; 16-bit groups of zeros. No more\n # ; than 6 groups in addition to\n # ; the \"::\" may be present.\n #\n # IPv6v4-full = IPv6-hex 5(\":\" IPv6-hex) \":\"\n # IPv4-address-literal\n #\n # IPv6v4-comp = [IPv6-hex *3(\":\" IPv6-hex)]\n # \"::\"\n # [IPv6-hex *3(\":\" IPv6-hex) \":\"]\n # IPv4-address-literal\n # ; The \"::\" represents at least 2\n # ; 16-bit groups of zeros. No more\n # ; than 4 groups in addition to\n # ; the \"::\" and\n # ; IPv4-address-literal may be\n # ; present.\n\n max_groups = 8\n index = False\n address_literal = parse_data['literal']\n\n # Extract IPv4 part from the end of the\n # address-literal (if there is one)\n regex = (\n r\"\\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.)\"\n r\"{3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$\"\n )\n match_ip = re.search(regex, address_literal)\n if match_ip:\n index = address_literal.rfind(\n match_ip.group(0))\n if index != 0:\n # Convert IPv4 part to IPv6 format for\n # further testing\n address_literal = (\n address_literal[0:index] + '0:0')\n\n if index == 0 and index is not False:\n # Nothing there except a valid IPv4 address\n return_status.append(\n RFC5321Diagnosis('ADDRESSLITERAL'))\n elif not address_literal.startswith(Char.IPV6TAG):\n return_status.append(\n RFC5322Diagnosis('DOMAINLITERAL'))\n else:\n ipv6 = address_literal[5:]\n # Revision 2.7: Daniel Marschall's new IPv6\n # testing strategy\n match_ip = ipv6.split(Char.COLON)\n grp_count = len(match_ip)\n index = ipv6.find(Char.DOUBLECOLON)\n\n if index == -1:\n # We need exactly the right number of\n # groups\n if grp_count != max_groups:\n return_status.append(\n RFC5322Diagnosis('IPV6_GRPCOUNT'))\n else:\n if index != ipv6.rfind(Char.DOUBLECOLON):\n return_status.append(\n RFC5322Diagnosis('IPV6_2X2XCOLON'))\n else:\n if index in [0, len(ipv6) - 2]:\n # RFC 4291 allows :: at the start\n # or end of an address with 7 other\n # groups in addition\n max_groups += 1\n\n if grp_count > max_groups:\n return_status.append(\n RFC5322Diagnosis(\n 'IPV6_MAXGRPS'))\n elif grp_count == max_groups:\n # Eliding a single \"::\"\n return_status.append(\n RFC5321Diagnosis(\n 'IPV6DEPRECATED'))\n\n # Revision 2.7: Daniel Marschall's new IPv6\n # testing strategy\n if (ipv6[0] == Char.COLON and\n ipv6[1] != Char.COLON):\n # Address starts with a single colon\n return_status.append(\n RFC5322Diagnosis('IPV6_COLONSTRT'))\n elif (ipv6[-1] == Char.COLON and\n ipv6[-2] != Char.COLON):\n # Address ends with a single colon\n return_status.append(\n RFC5322Diagnosis('IPV6_COLONEND'))\n elif ([re.match(r\"^[0-9A-Fa-f]{0,4}$\", i)\n for i in match_ip].count(None) != 0):\n # Check for unmatched characters\n return_status.append(\n RFC5322Diagnosis('IPV6_BADCHAR'))\n else:\n return_status.append(\n RFC5321Diagnosis('ADDRESSLITERAL'))\n else:\n return_status.append(\n RFC5322Diagnosis('DOMAINLITERAL'))\n\n parse_data[Context.DOMAIN] += token\n atom_list[Context.DOMAIN][element_count] += token\n element_len += 1\n context_prior = context\n context = context_stack.pop()\n elif token == Char.BACKSLASH:\n return_status.append(\n RFC5322Diagnosis('DOMLIT_OBSDTEXT'))\n context_stack.append(context)\n context = Context.QUOTEDPAIR\n # Folding White Space (FWS)\n elif token in [Char.CR, Char.SP, Char.HTAB]:\n # Skip simulates the use of ++ operator if the latter\n # check doesn't short-circuit\n if token == Char.CR:\n skip = True\n\n if (i+1 == raw_length or\n to_char(address[i+1]) != Char.LF):\n return_status.append(\n InvalidDiagnosis('CR_NO_LF'))\n break\n\n return_status.append(CFWSDiagnosis('FWS'))\n\n context_stack.append(context)\n context = Context.FWS\n token_prior = token\n # dtext\n else:\n # http://tools.ietf.org/html/rfc5322#section-3.4.1\n # dtext = %d33-90 / ; Printable US-ASCII\n # %d94-126 / ; characters not\n # obs-dtext ; including [, ], or \\\n #\n # obs-dtext = obs-NO-WS-CTL / quoted-pair\n #\n # obs-NO-WS-CTL = %d1-8 / ; US-ASCII control\n # %d11 / ; characters that do\n # %d12 / ; not include the\n # %d14-31 / ; carriage return, line\n # %d127 ; feed, and white space\n # ; characters\n o = ord(token)\n\n # CR, LF, SP & HTAB have already been parsed above\n if o > 127 or o == 0 or token == Char.OPENSQBRACKET:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('EXPECTING_DTEXT'))\n break\n elif o < 33 or o == 127:\n return_status.append(\n RFC5322Diagnosis('DOMLIT_OBSDTEXT'))\n\n parse_data['literal'] += token\n parse_data[Context.DOMAIN] += token\n atom_list[Context.DOMAIN][element_count] += token\n element_len += 1\n # -------------------------------------------------------\n # Quoted string\n # -------------------------------------------------------\n elif context == Context.QUOTEDSTRING:\n # http://tools.ietf.org/html/rfc5322#section-3.2.4\n # quoted-string = [CFWS]\n # DQUOTE *([FWS] qcontent) [FWS] DQUOTE\n # [CFWS]\n #\n # qcontent = qtext / quoted-pair\n\n # Quoted pair\n if token == Char.BACKSLASH:\n context_stack.append(context)\n context = Context.QUOTEDPAIR\n # Folding White Space (FWS)\n # Inside a quoted string, spaces are allow as regular\n # characters. It's only FWS if we include HTAB or CRLF\n elif token in [Char.CR, Char.HTAB]:\n # Skip simulates the use of ++ operator if the latter\n # check doesn't short-circuit\n if token == Char.CR:\n skip = True\n\n if (i+1 == raw_length or\n to_char(address[i+1]) != Char.LF):\n return_status.append(\n InvalidDiagnosis('CR_NO_LF'))\n break\n\n # http://tools.ietf.org/html/rfc5322#section-3.2.2\n # Runs of FWS, comment, or CFWS that occur between\n # lexical tokens in a structured header field are\n # semantically interpreted as a single space\n # character.\n\n # http://tools.ietf.org/html/rfc5322#section-3.2.4\n # the CRLF in any FWS/CFWS that appears within the\n # quoted string [is] semantically \"invisible\" and\n # therefore not part of the quoted-string\n parse_data[Context.LOCALPART] += Char.SP\n atom_list[Context.LOCALPART][element_count] += Char.SP\n element_len += 1\n\n return_status.append(CFWSDiagnosis('FWS'))\n context_stack.append(context)\n context = Context.FWS\n token_prior = token\n # End of quoted string\n elif token == Char.DQUOTE:\n parse_data[Context.LOCALPART] += token\n atom_list[Context.LOCALPART][element_count] += token\n element_len += 1\n context_prior = context\n context = context_stack.pop()\n # qtext\n else:\n # http://tools.ietf.org/html/rfc5322#section-3.2.4\n # qtext = %d33 / ; Printable US-ASCII\n # %d35-91 / ; characters not\n # %d93-126 / ; including \"\\\" or\n # obs-qtext ; the quote\n # ; character\n #\n # obs-qtext = obs-NO-WS-CTL\n #\n # obs-NO-WS-CTL = %d1-8 / ; US-ASCII control\n # %d11 / ; characters that do\n # %d12 / ; not include the CR,\n # %d14-31 / ; LF, and white space\n # %d127 ; characters\n o = ord(token)\n\n if o > 127 or o == 0 or o == 10:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('EXPECTING_QTEXT'))\n elif o < 32 or o == 127:\n return_status.append(\n DeprecatedDiagnosis('QTEXT'))\n\n parse_data[Context.LOCALPART] += token\n atom_list[Context.LOCALPART][element_count] += token\n element_len += 1\n # -------------------------------------------------------\n # Quoted pair\n # -------------------------------------------------------\n elif context == Context.QUOTEDPAIR:\n # http://tools.ietf.org/html/rfc5322#section-3.2.1\n # quoted-pair = (\"\\\" (VCHAR / WSP)) / obs-qp\n #\n # VCHAR = %d33-126 ; visible (printing)\n # ; characters\n #\n # WSP = SP / HTAB ; white space\n #\n # obs-qp = \"\\\" (%d0 / obs-NO-WS-CTL / LF / CR)\n #\n # obs-NO-WS-CTL = %d1-8 / ; US-ASCII control\n # %d11 / ; characters that do not\n # %d12 / ; include the carriage\n # %d14-31 / ; return, line feed, and\n # %d127 ; white space characters\n #\n # i.e. obs-qp = \"\\\" (%d0-8, %d10-31 / %d127)\n\n o = ord(token)\n\n if o > 127:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('EXPECTING_QPAIR'))\n elif (o < 31 and o != 9) or o == 127:\n # SP & HTAB are allowed\n return_status.append(DeprecatedDiagnosis('QP'))\n\n # At this point we know where this qpair occurred so\n # we could check to see if the character actually\n # needed to be quoted at all.\n # http://tools.ietf.org/html/rfc5321#section-4.1.2\n # the sending system SHOULD transmit the\n # form that uses the minimum quoting possible.\n context_prior = context\n context = context_stack.pop() # End of qpair\n token = Char.BACKSLASH + token\n\n if context == Context.COMMENT:\n pass\n elif context == Context.QUOTEDSTRING:\n parse_data[Context.LOCALPART] += token\n atom_list[Context.LOCALPART][element_count] += token\n # The maximum sizes specified by RFC 5321 are octet\n # counts, so we must include the backslash\n element_len += 2\n elif context == Context.LITERAL:\n parse_data[Context.DOMAIN] += token\n atom_list[Context.DOMAIN][element_count] += token\n # The maximum sizes specified by RFC 5321 are octet\n # counts, so we must include the backslash\n element_len += 2\n else: # pragma: no cover\n if diagnose:\n return InvalidDiagnosis('BAD_PARSE')\n else:\n return False\n # -------------------------------------------------------\n # Comment\n # -------------------------------------------------------\n elif context == Context.COMMENT:\n # http://tools.ietf.org/html/rfc5322#section-3.2.2\n # comment = \"(\" *([FWS] ccontent) [FWS] \")\"\n #\n # ccontent = ctext / quoted-pair / comment\n\n # Nested comment\n if token == Char.OPENPARENTHESIS:\n # Nested comments are OK\n context_stack.append(context)\n context = Context.COMMENT\n # End of comment\n elif token == Char.CLOSEPARENTHESIS:\n context_prior = context\n context = context_stack.pop()\n # Quoted pair\n elif token == Char.BACKSLASH:\n context_stack.append(context)\n context = Context.QUOTEDPAIR\n # Folding White Space (FWS)\n elif token in [Char.CR, Char.SP, Char.HTAB]:\n # Skip simulates the use of ++ operator if the latter\n # check doesn't short-circuit\n if token == Char.CR:\n skip = True\n\n if (i+1 == raw_length or\n to_char(address[i+1]) != Char.LF):\n return_status.append(\n InvalidDiagnosis('CR_NO_LF'))\n break\n\n return_status.append(CFWSDiagnosis('FWS'))\n\n context_stack.append(context)\n context = Context.FWS\n token_prior = token\n # ctext\n else:\n # http://tools.ietf.org/html/rfc5322#section-3.2.3\n # ctext = %d33-39 / ; Printable US-\n # %d42-91 / ; ASCII characters\n # %d93-126 / ; not including\n # obs-ctext ; \"(\", \")\", or \"\\\"\n #\n # obs-ctext = obs-NO-WS-CTL\n #\n # obs-NO-WS-CTL = %d1-8 / ; US-ASCII control\n # %d11 / ; characters that\n # %d12 / ; do not include\n # %d14-31 / ; the CR, LF, and\n # ; white space\n # ; characters\n\n o = ord(token)\n\n if o > 127 or o == 0 or o == 10:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('EXPECTING_CTEXT'))\n break\n elif o < 32 or o == 127:\n return_status.append(DeprecatedDiagnosis('CTEXT'))\n\n # -------------------------------------------------------\n # Folding White Space (FWS)\n # -------------------------------------------------------\n elif context == Context.FWS:\n # http://tools.ietf.org/html/rfc5322#section-3.2.2\n # FWS = ([*WSP CRLF] 1*WSP) / obs-FWS\n # ; Folding white space\n #\n # But note the erratum:\n # http://www.rfc-editor.org/errata_search.php?rfc=5322&eid=1908\n # In the obsolete syntax, any amount of folding white\n # space MAY be inserted where the obs-FWS rule is\n # allowed. This creates the possibility of having two\n # consecutive \"folds\" in a line, and therefore the\n # possibility that a line which makes up a folded header\n # field could be composed entirely of white space.\n #\n # obs-FWS = 1*([CRLF] WSP)\n\n if token_prior == Char.CR:\n if token == Char.CR:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('FWS_CRLF_X2'))\n break\n\n if crlf_count != -1:\n crlf_count += 1\n if crlf_count > 1:\n # Multiple folds = obsolete FWS\n return_status.append(\n DeprecatedDiagnosis('FWS'))\n else:\n crlf_count = 1\n\n # Skip simulates the use of ++ operator if the latter\n # check doesn't short-circuit\n if token == Char.CR:\n skip = True\n\n if (i+1 == raw_length or\n to_char(address[i+1]) != Char.LF):\n return_status.append(InvalidDiagnosis('CR_NO_LF'))\n break\n elif token in [Char.SP, Char.HTAB]:\n pass\n else:\n if token_prior == Char.CR:\n # Fatal error\n return_status.append(\n InvalidDiagnosis('FWS_CRLF_END'))\n break\n\n if crlf_count != -1:\n crlf_count = -1\n\n context_prior = context\n # End of FWS\n context = context_stack.pop()\n\n # Look at this token again in the parent context\n repeat = True\n\n token_prior = token\n\n # -------------------------------------------------------\n # A context we aren't expecting\n # -------------------------------------------------------\n else: # pragma: no cover\n if diagnose:\n return InvalidDiagnosis('BAD_PARSE')\n else:\n return False\n\n # No point in going on if we've got a fatal error\n if max(return_status) > BaseDiagnosis.CATEGORIES['RFC5322']:\n break\n\n # Some simple final tests\n if max(return_status) < BaseDiagnosis.CATEGORIES['RFC5322']:\n if context == Context.QUOTEDSTRING:\n # Fatal error\n return_status.append(InvalidDiagnosis('UNCLOSEDQUOTEDSTR'))\n elif context == Context.QUOTEDPAIR:\n # Fatal error\n return_status.append(InvalidDiagnosis('BACKSLASHEND'))\n elif context == Context.COMMENT:\n # Fatal error\n return_status.append(InvalidDiagnosis('UNCLOSEDCOMMENT'))\n elif context == Context.LITERAL:\n # Fatal error\n return_status.append(InvalidDiagnosis('UNCLOSEDDOMLIT'))\n elif token == Char.CR:\n # Fatal error\n return_status.append(InvalidDiagnosis('FWS_CRLF_END'))\n elif parse_data[Context.DOMAIN] == '':\n # Fatal error\n return_status.append(InvalidDiagnosis('NODOMAIN'))\n elif element_len == 0:\n # Fatal error\n return_status.append(InvalidDiagnosis('DOT_END'))\n elif hyphen_flag:\n # Fatal error\n return_status.append(InvalidDiagnosis('DOMAINHYPHENEND'))\n # http://tools.ietf.org/html/rfc5321#section-4.5.3.1.2\n # The maximum total length of a domain name or number is 255 octets\n elif len(parse_data[Context.DOMAIN]) > 255:\n return_status.append(RFC5322Diagnosis('DOMAIN_TOOLONG'))\n # http://tools.ietf.org/html/rfc5321#section-4.1.2\n # Forward-path = Path\n #\n # Path = \"<\" [ A-d-l \":\" ] Mailbox \">\"\n #\n # http://tools.ietf.org/html/rfc5321#section-4.5.3.1.3\n # The maximum total length of a reverse-path or forward-path is\n # 256 octets (including the punctuation and element separators).\n #\n # Thus, even without (obsolete) routing information, the Mailbox\n # can only be 254 characters long. This is confirmed by this\n # verified erratum to RFC 3696:\n #\n # http://www.rfc-editor.org/errata_search.php?rfc=3696&eid=1690\n # However, there is a restriction in RFC 2821 on the length of an\n # address in MAIL and RCPT commands of 254 characters. Since\n # addresses that do not fit in those fields are not normally\n # useful, the upper limit on address lengths should normally be\n # considered to be 254.\n elif len(parse_data[Context.LOCALPART] + Char.AT +\n parse_data[Context.DOMAIN]) > 254:\n return_status.append(RFC5322Diagnosis('TOOLONG'))\n # http://tools.ietf.org/html/rfc1035#section-2.3.4\n # labels 63 octets or less\n elif element_len > 63:\n return_status.append(RFC5322Diagnosis('LABEL_TOOLONG'))\n\n return_status = list(set(return_status))\n final_status = max(return_status)\n\n if len(return_status) != 1:\n # Remove redundant ValidDiagnosis\n return_status.pop(0)\n\n parse_data['status'] = return_status\n\n if final_status < threshold:\n final_status = ValidDiagnosis()\n\n if diagnose:\n return final_status\n else:\n return final_status < BaseDiagnosis.CATEGORIES['THRESHOLD']"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading a list of source nodes for a specified freesurfer label.", "response": "def load_freesurfer_label(annot_input, label_name, cortex=None):\n \"\"\"\n Get source node list for a specified freesurfer label.\n\n Inputs\n -------\n annot_input : freesurfer annotation label file\n label_name : freesurfer label name\n cortex : not used\n \"\"\"\n\n if cortex is not None:\n print(\"Warning: cortex is not used to load the freesurfer label\")\n\n labels, color_table, names = nib.freesurfer.read_annot(annot_input)\n names = [i.decode('utf-8') for i in names]\n label_value = names.index(label_name)\n label_nodes = np.array(np.where(np.in1d(labels, label_value)), dtype=np.int32)\n\n return label_nodes"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget freesurfer label names from an annot file.", "response": "def get_freesurfer_label(annot_input, verbose = True):\n \"\"\"\n Print freesurfer label names.\n \"\"\"\n labels, color_table, names = nib.freesurfer.read_annot(annot_input)\n if verbose:\n print(names)\n return names"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef viz(coords, faces, stat_map=None,\n elev=0, azim=0, cmap='coolwarm',\n threshold=None, alpha='auto',\n bg_map=None, bg_on_stat=False,\n figsize=None,\n **kwargs):\n\n ''' Visualize results on cortical surface using matplotlib.\n\n Inputs\n -------\n coords : numpy array of shape (n_nodes,3), each row specifying the x,y,z\n coordinates of one node of surface mesh\n faces : numpy array of shape (n_faces, 3), each row specifying the indices\n of the three nodes building one node of the surface mesh\n stat_map : numpy array of shape (n_nodes,) containing the values to be\n visualized for each node.\n elev, azim : integers, elevation and azimuth parameters specifying the view\n on the 3D plot. For Freesurfer surfaces elev=0, azim=0 will\n give a lateral view for the right and a medial view for the\n left hemisphere, elev=0, azim=180 will give a medial view for\n the right and lateral view for the left hemisphere.\n cmap : Matplotlib colormap, the color range will me forced to be symmetric.\n Colormaps can be specified as string or colormap object.\n threshold : float, threshold to be applied to the map, will be applied in\n positive and negative direction, i.e. values < -abs(threshold)\n and > abs(threshold) will be shown.\n alpha : float, determines the opacity of the background mesh, in'auto' mode\n alpha defaults to .5 when no background map is given, to 1 otherwise.\n bg_map : numpy array of shape (n_nodes,) to be plotted underneath the\n statistical map. Specifying a sulcal depth map as bg_map results\n in realistic shadowing of the surface.\n bg_on_stat : boolean, specifies whether the statistical map should be\n multiplied with the background map for shadowing. Otherwise,\n only areas that are not covered by the statsitical map after\n thresholding will show shadows.\n figsize : tuple of intergers, dimensions of the figure that is produced.\n\n\n Output\n ------\n Matplotlib figure object\n '''\n\n import numpy as np\n import matplotlib.pyplot as plt\n import matplotlib.tri as tri\n from mpl_toolkits.mplot3d import Axes3D\n\n # load mesh and derive axes limits\n faces = np.array(faces, dtype=int)\n limits = [coords.min(), coords.max()]\n\n # set alpha if in auto mode\n if alpha == 'auto':\n if bg_map is None:\n alpha = .5\n else:\n alpha = 1\n\n # if cmap is given as string, translate to matplotlib cmap\n if type(cmap) == str:\n cmap = plt.cm.get_cmap(cmap)\n\n # initiate figure and 3d axes\n if figsize is not None:\n fig = plt.figure(figsize=figsize)\n else:\n fig = plt.figure()\n ax = fig.add_subplot(111, projection='3d', xlim=limits, ylim=limits)\n ax.view_init(elev=elev, azim=azim)\n ax.set_axis_off()\n\n # plot mesh without data\n p3dcollec = ax.plot_trisurf(coords[:, 0], coords[:, 1], coords[:, 2],\n triangles=faces, linewidth=0.,\n antialiased=False,\n color='white')\n\n # If depth_map and/or stat_map are provided, map these onto the surface\n # set_facecolors function of Poly3DCollection is used as passing the\n # facecolors argument to plot_trisurf does not seem to work\n if bg_map is not None or stat_map is not None:\n\n face_colors = np.ones((faces.shape[0], 4))\n face_colors[:, :3] = .5*face_colors[:, :3]\n\n if bg_map is not None:\n bg_data = bg_map\n if bg_data.shape[0] != coords.shape[0]:\n raise ValueError('The bg_map does not have the same number '\n 'of vertices as the mesh.')\n bg_faces = np.mean(bg_data[faces], axis=1)\n bg_faces = bg_faces - bg_faces.min()\n bg_faces = bg_faces / bg_faces.max()\n face_colors = plt.cm.gray_r(bg_faces)\n\n # modify alpha values of background\n face_colors[:, 3] = alpha*face_colors[:, 3]\n\n if stat_map is not None:\n stat_map_data = stat_map\n stat_map_faces = np.mean(stat_map_data[faces], axis=1)\n\n # Ensure symmetric colour range, based on Nilearn helper function:\n # https://github.com/nilearn/nilearn/blob/master/nilearn/plotting/img_plotting.py#L52\n vmax = max(-np.nanmin(stat_map_faces), np.nanmax(stat_map_faces))\n vmin = -vmax\n\n if threshold is not None:\n kept_indices = np.where(abs(stat_map_faces) >= threshold)[0]\n stat_map_faces = stat_map_faces - vmin\n stat_map_faces = stat_map_faces / (vmax-vmin)\n if bg_on_stat:\n face_colors[kept_indices] = cmap(stat_map_faces[kept_indices]) * face_colors[kept_indices]\n else:\n face_colors[kept_indices] = cmap(stat_map_faces[kept_indices])\n else:\n stat_map_faces = stat_map_faces - vmin\n stat_map_faces = stat_map_faces / (vmax-vmin)\n if bg_on_stat:\n face_colors = cmap(stat_map_faces) * face_colors\n else:\n face_colors = cmap(stat_map_faces)\n\n p3dcollec.set_facecolors(face_colors)\n\n return fig, ax", "response": "Visualize the results of a cortical surface using matplotlib."} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef surf_keep_cortex(surf, cortex):\n\n # split surface into vertices and triangles\n vertices, triangles = surf\n\n # keep only the vertices within the cortex label\n cortex_vertices = np.array(vertices[cortex], dtype=np.float64)\n\n # keep only the triangles within the cortex label\n cortex_triangles = triangles_keep_cortex(triangles, cortex)\n\n return cortex_vertices, cortex_triangles", "response": "Remove medial wall from cortical surface to ensure that shortest paths are only calculated through the cortex."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nremoving triangles with nodes not contained in the cortex label array", "response": "def triangles_keep_cortex(triangles, cortex):\n \"\"\"\n Remove triangles with nodes not contained in the cortex label array\n \"\"\"\n\n # for or each face/triangle keep only those that only contain nodes within the list of cortex nodes\n input_shape = triangles.shape\n triangle_is_in_cortex = np.all(np.reshape(np.in1d(triangles.ravel(), cortex), input_shape), axis=1)\n\n cortex_triangles_old = np.array(triangles[triangle_is_in_cortex], dtype=np.int32)\n\n # reassign node index before outputting triangles\n new_index = np.digitize(cortex_triangles_old.ravel(), cortex, right=True)\n cortex_triangles = np.array(np.arange(len(cortex))[new_index].reshape(cortex_triangles_old.shape), dtype=np.int32)\n\n return cortex_triangles"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntranslate source nodes to new surface.", "response": "def translate_src(src, cortex):\n \"\"\"\n Convert source nodes to new surface (without medial wall).\n \"\"\"\n src_new = np.array(np.where(np.in1d(cortex, src))[0], dtype=np.int32)\n\n return src_new"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn data values to space of full cortex (including medial wall), with medial wall equal to zero.", "response": "def recort(input_data, surf, cortex):\n \"\"\"\n Return data values to space of full cortex (including medial wall), with medial wall equal to zero.\n \"\"\"\n data = np.zeros(len(surf[0]))\n data[cortex] = input_data\n return data"} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef find_node_match(simple_vertices, complex_vertices):\n\n import scipy.spatial\n\n # make array for writing in final voronoi seed indices\n voronoi_seed_idx = np.zeros((simple_vertices.shape[0],), dtype='int64')-1\n missing = np.where(voronoi_seed_idx == -1)[0].shape[0]\n mapping_single = np.zeros_like(voronoi_seed_idx)\n\n neighbours = 0\n col = 0\n\n while missing != 0:\n\n neighbours += 100\n # find nearest neighbours\n inaccuracy, mapping = scipy.spatial.KDTree(\n complex_vertices).query(simple_vertices, k=neighbours)\n # go through columns of nearest neighbours until unique mapping is\n # achieved, if not before end of neighbours, extend number of\n # neighbours\n while col < neighbours:\n # find all missing voronoi seed indices\n missing_idx = np.where(voronoi_seed_idx == -1)[0]\n missing = missing_idx.shape[0]\n if missing == 0:\n break\n else:\n # for missing entries fill in next neighbour\n mapping_single[missing_idx] = np.copy(\n mapping[missing_idx, col])\n # find unique values in mapping_single\n unique, double_idx = np.unique(\n mapping_single, return_inverse=True)\n # empty voronoi seed index\n voronoi_seed_idx = np.zeros(\n (simple_vertices.shape[0],), dtype='int64')-1\n # fill voronoi seed idx with unique values\n for u in range(unique.shape[0]):\n # find the indices of this value in mapping\n entries = np.where(double_idx == u)[0]\n # set the first entry to the value\n voronoi_seed_idx[entries[0]] = unique[u]\n # go to next column\n col += 1\n\n return voronoi_seed_idx, inaccuracy", "response": "Find the best match for a node in a given complex mesh."} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing shot info for home team.", "response": "def parse_home_shots(self):\n \"\"\"\n Parse shot info for home team.\n \n :returns: ``self`` on success, ``None`` otherwise\n \"\"\"\n try:\n self.__set_shot_tables()\n self.shots['home'] = self.__parse_shot_tables(\n self.__home_top,\n self.__home_bot\n )\n return self\n except:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing shot info for away team.", "response": "def parse_away_shots(self):\n \"\"\"\n Parse shot info for away team.\n \n :returns: ``self`` on success, ``None`` otherwise\n \"\"\"\n try:\n self.__set_shot_tables()\n self.shots['away'] = self.__parse_shot_tables(\n self.__aw_top,\n self.__aw_bot\n )\n return self\n except:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse face - off info for home team.", "response": "def parse_home_fo(self):\n \"\"\"\n Parse face-off info for home team.\n \n :returns: ``self`` on success, ``None`` otherwise\n \"\"\"\n try:\n self.__set_fo_tables()\n self.face_offs['home'] = self.__parse_fo_table(self.__home_fo)\n return self\n except:\n return None"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef parse_away_fo(self):\n try:\n self.__set_fo_tables()\n self.face_offs['away'] = self.__parse_fo_table(self.__away_fo)\n return self\n except:\n return None", "response": "Parses face - off info for away team."} {"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dist_calc(surf, cortex, source_nodes):\n\n \"\"\"\n Calculate exact geodesic distance along cortical surface from set of source nodes.\n \"dist_type\" specifies whether to calculate \"min\", \"mean\", \"median\", or \"max\" distance values\n from a region-of-interest. If running only on single node, defaults to \"min\".\n \"\"\"\n\n cortex_vertices, cortex_triangles = surf_keep_cortex(surf, cortex)\n translated_source_nodes = translate_src(source_nodes, cortex)\n data = gdist.compute_gdist(cortex_vertices, cortex_triangles, source_indices = translated_source_nodes)\n dist = recort(data, surf, cortex)\n del data\n\n return dist", "response": "Calculate exact geodesic distance along cortical surface from set of source nodes."} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate closest nodes to each source node using exact geodesic distance along the cortical surface.", "response": "def zone_calc(surf, cortex, src):\n \"\"\"\n Calculate closest nodes to each source node using exact geodesic distance along the cortical surface.\n \"\"\"\n\n cortex_vertices, cortex_triangles = surf_keep_cortex(surf, cortex)\n\n dist_vals = np.zeros((len(source_nodes), len(cortex_vertices)))\n\n for x in range(len(source_nodes)):\n\n translated_source_nodes = translate_src(source_nodes[x], cortex)\n dist_vals[x, :] = gdist.compute_gdist(cortex_vertices, cortex_triangles, source_indices = translated_source_nodes)\n\n data = np.argsort(dist_vals, axis=0)[0, :] + 1\n\n zone = recort(data, surf, cortex)\n\n del data\n\n return zone"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalculates exact geodesic distance along cortical surface from set of source nodes.", "response": "def dist_calc_matrix(surf, cortex, labels, exceptions = ['Unknown', 'Medial_wall'], verbose = True):\n \"\"\"\n Calculate exact geodesic distance along cortical surface from set of source nodes.\n \"labels\" specifies the freesurfer label file to use. All values will be used other than those\n specified in \"exceptions\" (default: 'Unknown' and 'Medial_Wall').\n\n returns:\n dist_mat: symmetrical nxn matrix of minimum distance between pairs of labels\n rois: label names in order of n\n \"\"\"\n\n cortex_vertices, cortex_triangles = surf_keep_cortex(surf, cortex)\n\n # remove exceptions from label list:\n label_list = sd.load.get_freesurfer_label(labels, verbose = False)\n rs = np.where([a not in exceptions for a in label_list])[0]\n rois = [label_list[r] for r in rs]\n if verbose:\n print(\"# of regions: \" + str(len(rois)))\n\n # calculate distance from each region to all nodes:\n dist_roi = []\n for roi in rois:\n source_nodes = sd.load.load_freesurfer_label(labels, roi)\n translated_source_nodes = translate_src(source_nodes, cortex)\n dist_roi.append(gdist.compute_gdist(cortex_vertices, cortex_triangles,\n source_indices = translated_source_nodes))\n if verbose:\n print(roi)\n dist_roi = np.array(dist_roi)\n\n # Calculate min distance per region:\n dist_mat = []\n for roi in rois:\n source_nodes = sd.load.load_freesurfer_label(labels, roi)\n translated_source_nodes = translate_src(source_nodes, cortex)\n dist_mat.append(np.min(dist_roi[:,translated_source_nodes], axis = 1))\n dist_mat = np.array(dist_mat)\n\n return dist_mat, rois"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nparses only the home faceoffs and set the attributes to self. face_offs. home", "response": "def parse_home_face_offs(self):\n \"\"\"\n Parse only the home faceoffs\n \n :returns: ``self`` on success, ``None`` otherwise\n \"\"\"\n self.__set_team_docs()\n self.face_offs['home'] = FaceOffRep.__read_team_doc(self.__home_doc)\n return self"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse_away_face_offs(self):\n self.__set_team_docs()\n self.face_offs['away'] = FaceOffRep.__read_team_doc(self.__vis_doc)\n return self", "response": "Parse only the away faceoffs\n \n "} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nload a module by filename", "response": "def load_module(filename):\n \"\"\"\n Loads a module by filename\n \"\"\"\n basename = os.path.basename(filename)\n path = os.path.dirname(filename)\n sys.path.append(path)\n # TODO(tlan) need to figure out how to handle errors thrown here\n return __import__(os.path.splitext(basename)[0])"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_machine_mapping(machine_list):\n if machine_list is None:\n return {}\n else:\n mapping = {}\n for pair in machine_list:\n if (constants.MACHINE_SEPARATOR not in pair) or (pair.count(constants.MACHINE_SEPARATOR) != 1):\n raise ValueError(\"machine pairs must be passed as two strings separted by a %s\", constants.MACHINE_SEPARATOR)\n (logical, physical) = pair.split(constants.MACHINE_SEPARATOR)\n # add checks for reachability\n mapping[logical] = physical\n return mapping", "response": "Convert the machine list argument from a list of names to a mapping of logical names to physical hosts."} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef parse_config_list(config_list):\n if config_list is None:\n return {}\n else:\n mapping = {}\n for pair in config_list:\n if (constants.CONFIG_SEPARATOR not in pair) or (pair.count(constants.CONFIG_SEPARATOR) != 1):\n raise ValueError(\"configs must be passed as two strings separted by a %s\", constants.CONFIG_SEPARATOR)\n (config, value) = pair.split(constants.CONFIG_SEPARATOR)\n mapping[config] = value\n return mapping", "response": "Parse a list of configuration properties separated by ="} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef parse_config_file(config_file_path):\n extension = os.path.splitext(config_file_path)[1]\n if extension == '.pyc':\n raise ValueError(\"Skipping .pyc file as config\")\n if extension == '.json':\n with open(config_file_path) as config_file:\n try:\n mapping = json.load(config_file)\n except ValueError as e:\n logger.error(\"Did not load json configs\", e)\n raise SyntaxError('Unable to parse config file:%s due to malformed JSON. Aborting' %(config_file_path))\n elif extension == '.py':\n mapping = {}\n file_dict = load_module(config_file_path)\n for attr_name in dir(file_dict):\n if not (attr_name.startswith('_') or attr_name.startswith('__')):\n attr = getattr(file_dict, attr_name)\n if type(attr) is dict:\n mapping.update(attr)\n else:\n with open(config_file_path) as config_file:\n lines = [line.rstrip() for line in config_file if line.rstrip() != \"\" and not line.startswith(\"#\")]\n mapping = parse_config_list(lines)\n\n return mapping", "response": "Parse a configuration file. Currently only supports. json. py and. properties separated by =\n "} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef exec_with_env(ssh, command, msg='', env={}, **kwargs):\r\n bash_profile_command = \"source .bash_profile > /dev/null 2> /dev/null;\"\r\n env_command = build_os_environment_string(env)\r\n new_command = bash_profile_command + env_command + command\r\n if kwargs.get('sync', True):\r\n return better_exec_command(ssh, new_command, msg)\r\n else:\r\n return ssh.exec_command(new_command)", "response": "Execute a command with the given environment"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nusing paramiko to execute a command but handles failure by raising a ParamikoException", "response": "def better_exec_command(ssh, command, msg):\r\n \"\"\"Uses paramiko to execute a command but handles failure by raising a ParamikoError if the command fails.\r\n Note that unlike paramiko.SSHClient.exec_command this is not asynchronous because we wait until the exit status is known\r\n\r\n :Parameter ssh: a paramiko SSH Client\r\n :Parameter command: the command to execute\r\n :Parameter msg: message to print on failure\r\n\r\n :Returns (paramiko.Channel)\r\n the underlying channel so that the caller can extract stdout or send to stdin\r\n\r\n :Raises SSHException: if paramiko would raise an SSHException\r\n :Raises ParamikoError: if the command produces output to stderr\r\n \"\"\"\r\n chan = ssh.get_transport().open_session()\r\n chan.exec_command(command)\r\n exit_status = chan.recv_exit_status()\r\n if exit_status != 0:\r\n msg_str = chan.recv_stderr(1024)\r\n err_msgs = []\r\n while len(msg_str) > 0:\r\n err_msgs.append(msg_str)\r\n msg_str = chan.recv_stderr(1024)\r\n err_msg = ''.join(err_msgs)\r\n logger.error(err_msg)\r\n raise ParamikoError(msg, err_msg)\r\n return chan"} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef log_output(chan):\r\n if hasattr(chan, \"recv\"):\r\n str = chan.recv(1024)\r\n msgs = []\r\n while len(str) > 0:\r\n msgs.append(str)\r\n str = chan.recv(1024)\r\n msg = ''.join(msgs).strip()\r\n if len(msg) > 0:\r\n logger.info(msg)", "response": "Logs the output of a remote command\r\n"} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef open_remote_file(hostname, filename, mode='r', bufsize=-1, username=None, password=None):\r\n with get_ssh_client(hostname, username=username, password=password) as ssh:\r\n sftp = None\r\n f = None\r\n try:\r\n sftp = ssh.open_sftp()\r\n f = sftp.open(filename, mode, bufsize)\r\n yield f\r\n finally:\r\n if f is not None:\r\n f.close()\r\n if sftp is not None:\r\n sftp.close()", "response": "A generator that opens a remote file on the remote machine."} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef deploy(self, unique_id, configs=None):\n self.install(unique_id, configs)\n self.start(unique_id, configs)", "response": "Deploys the service to the host."} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform a soft bounce for the specified process .", "response": "def soft_bounce(self, unique_id, configs=None):\n \"\"\" Performs a soft bounce (stop and start) for the specified process\n\n :Parameter unique_id: the name of the process\n \"\"\"\n self.stop(unique_id, configs)\n self.start(unique_id, configs)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nperforms a hard bounce for the specified process .", "response": "def hard_bounce(self, unique_id, configs=None):\n \"\"\" Performs a hard bounce (kill and start) for the specified process\n\n :Parameter unique_id: the name of the process\n \"\"\"\n self.kill(unique_id, configs)\n self.start(unique_id, configs)"} {"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sleep(self, unique_id, delay, configs=None):\n self.pause(unique_id, configs)\n time.sleep(delay)\n self.resume(unique_id, configs)", "response": "Pauses the process for the specified delay and then resumes it."} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nissue a sigstop for the specified process", "response": "def pause(self, unique_id, configs=None):\n \"\"\" Issues a sigstop for the specified process\n\n :Parameter unique_id: the name of the process\n \"\"\"\n pids = self.get_pid(unique_id, configs)\n if pids != constants.PROCESS_NOT_RUNNING_PID:\n pid_str = ' '.join(str(pid) for pid in pids)\n hostname = self.processes[unique_id].hostname\n with get_ssh_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ssh:\n better_exec_command(ssh, \"kill -SIGSTOP {0}\".format(pid_str), \"PAUSING PROCESS {0}\".format(unique_id))"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _send_signal(self, unique_id, signalno, configs):\n pids = self.get_pid(unique_id, configs)\n if pids != constants.PROCESS_NOT_RUNNING_PID:\n pid_str = ' '.join(str(pid) for pid in pids)\n hostname = self.processes[unique_id].hostname\n msg= Deployer._signalnames.get(signalno,\"SENDING SIGNAL %s TO\"%signalno)\n with get_ssh_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ssh:\n better_exec_command(ssh, \"kill -{0} {1}\".format(signalno, pid_str), \"{0} PROCESS {1}\".format(msg, unique_id))", "response": "Issues a signal to the specified process"} {"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef resume(self, unique_id, configs=None):\n self._send_signal(unique_id, signal.SIGCONT,configs)", "response": "Resumes the process with the specified unique_id."} {"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef kill(self, unique_id, configs=None):\n self._send_signal(unique_id, signal.SIGKILL, configs)", "response": "Issues a kill - 9 to the specified process\n "} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef terminate(self, unique_id, configs=None):\n self._send_signal(unique_id, signal.SIGTERM, configs)", "response": "Issues a kill - 15 to the specified process\n "} {"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nissuing a signal to hangup the specified process", "response": "def hangup(self, unique_id, configs=None):\n \"\"\"\n Issue a signal to hangup the specified process\n\n :Parameter unique_id: the name of the process\n \"\"\"\n self._send_signal(unique_id, signal.SIGHUP, configs)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndeprecate name for fetch_logs", "response": "def get_logs(self, unique_id, logs, directory, pattern=constants.FILTER_NAME_ALLOW_NONE):\n \"\"\"deprecated name for fetch_logs\"\"\"\n self.fetch_logs(unique_id, logs, directory, pattern)"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef fetch_logs(self, unique_id, logs, directory, pattern=constants.FILTER_NAME_ALLOW_NONE):\n hostname = self.processes[unique_id].hostname\n install_path = self.processes[unique_id].install_path\n self.fetch_logs_from_host(hostname, install_path, unique_id, logs, directory, pattern)", "response": "Fetches the logs from the remote host to the local directory"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef fetch_logs_from_host(hostname, install_path, prefix, logs, directory, pattern):\n if hostname is not None:\n with get_sftp_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ftp:\n for f in logs:\n try:\n mode = ftp.stat(f).st_mode\n except IOError, e:\n if e.errno == errno.ENOENT:\n logger.error(\"Log file \" + f + \" does not exist on \" + hostname)\n pass\n else:\n copy_dir(ftp, f, directory, prefix)\n if install_path is not None:\n copy_dir(ftp, install_path, directory, prefix, pattern)", "response": "Static method Copies logs from specified host on the specified install path and stores them in the specified directory."} {"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninstalls the executable on the remote machine under the given unique_id.", "response": "def install(self, unique_id, configs=None):\n \"\"\"\n Copies the executable to the remote machine under install path. Inspects the configs for the possible keys\n 'hostname': the host to install on\n 'install_path': the location on the remote host\n 'executable': the executable to copy\n 'no_copy': if this config is passed in and true then this method will not copy the executable assuming that it is\n already installed\n 'post_install_cmds': an optional list of commands that should be executed on the remote machine after the\n executable has been installed. If no_copy is set to true, then the post install commands will not be run.\n\n If the unique_id is already installed on a different host, this will perform the cleanup action first.\n If either 'install_path' or 'executable' are provided the new value will become the default.\n\n :param unique_id:\n :param configs:\n :return:\n \"\"\"\n\n # the following is necessay to set the configs for this function as the combination of the\n # default configurations and the parameter with the parameter superceding the defaults but\n # not modifying the defaults\n if configs is None:\n configs = {}\n tmp = self.default_configs.copy()\n tmp.update(configs)\n configs = tmp\n\n hostname = None\n is_tarfile = False\n is_zipfile = False\n if unique_id in self.processes and 'hostname' in configs:\n self.uninstall(unique_id, configs)\n hostname = configs['hostname']\n elif 'hostname' in configs:\n hostname = configs['hostname']\n elif unique_id not in self.processes:\n # we have not installed this unique_id before and no hostname is provided in the configs so raise an error\n raise DeploymentError(\"hostname was not provided for unique_id: \" + unique_id)\n\n env = configs.get(\"env\", {})\n install_path = configs.get('install_path') or self.default_configs.get('install_path')\n pid_file = configs.get('pid_file') or self.default_configs.get('pid_file')\n if install_path is None:\n logger.error(\"install_path was not provided for unique_id: \" + unique_id)\n raise DeploymentError(\"install_path was not provided for unique_id: \" + unique_id)\n if not configs.get('no_copy', False):\n with get_ssh_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ssh:\n log_output(better_exec_command(ssh, \"mkdir -p {0}\".format(install_path),\n \"Failed to create path {0}\".format(install_path)))\n log_output(better_exec_command(ssh, \"chmod 755 {0}\".format(install_path),\n \"Failed to make path {0} writeable\".format(install_path)))\n executable = configs.get('executable') or self.default_configs.get('executable')\n if executable is None:\n logger.error(\"executable was not provided for unique_id: \" + unique_id)\n raise DeploymentError(\"executable was not provided for unique_id: \" + unique_id)\n\n #if the executable is in remote location copy to local machine\n copy_from_remote_location = False;\n\n if (\":\" in executable):\n copy_from_remote_location = True\n\n if (\"http\" not in executable):\n remote_location_server = executable.split(\":\")[0]\n remote_file_path = executable.split(\":\")[1] \n remote_file_name = os.path.basename(remote_file_path)\n\n local_temp_file_name = os.path.join(configs.get(\"tmp_dir\",\"/tmp\"),remote_file_name)\n \n if not os.path.exists(local_temp_file_name):\n with get_sftp_client(remote_location_server,username=runtime.get_username(), password=runtime.get_password()) as ftp:\n try:\n ftp.get(remote_file_path, local_temp_file_name)\n executable = local_temp_file_name\n except:\n raise DeploymentError(\"Unable to load file from remote server \" + executable)\n #use urllib for http copy\n else: \n remote_file_name = executable.split(\"/\")[-1]\n local_temp_file_name = os.path.join(configs.get(\"tmp_dir\",\"/tmp\"),remote_file_name)\n if not os.path.exists(local_temp_file_name):\n try:\n urllib.urlretrieve (executable, local_temp_file_name)\n except:\n raise DeploymentError(\"Unable to load file from remote server \" + executable)\n executable = local_temp_file_name \n\n try: \n exec_name = os.path.basename(executable)\n install_location = os.path.join(install_path, exec_name)\n with get_sftp_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ftp:\n ftp.put(executable, install_location)\n except:\n raise DeploymentError(\"Unable to copy executable to install_location:\" + install_location)\n finally:\n #Track if its a tarfile or zipfile before deleting it in case the copy to remote location fails\n is_tarfile = tarfile.is_tarfile(executable)\n is_zipfile = zipfile.is_zipfile(executable)\n if (copy_from_remote_location and not configs.get('cache',False)):\n os.remove(executable) \n\n # only supports tar and zip (because those modules are provided by Python's standard library)\n if configs.get('extract', False) or self.default_configs.get('extract', False):\n if is_tarfile:\n log_output(better_exec_command(ssh, \"tar -xf {0} -C {1}\".format(install_location, install_path),\n \"Failed to extract tarfile {0}\".format(exec_name)))\n elif is_zipfile:\n log_output(better_exec_command(ssh, \"unzip -o {0} -d {1}\".format(install_location, install_path),\n \"Failed to extract zipfile {0}\".format(exec_name)))\n else:\n logger.error(executable + \" is not a supported filetype for extracting\")\n raise DeploymentError(executable + \" is not a supported filetype for extracting\")\n post_install_cmds = configs.get('post_install_cmds', False) or self.default_configs.get('post_install_cmds', [])\n for cmd in post_install_cmds:\n relative_cmd = \"cd {0}; {1}\".format(install_path, cmd)\n log_output(exec_with_env(ssh, relative_cmd,\n msg=\"Failed to execute post install command: {0}\".format(relative_cmd), env=env))\n self.processes[unique_id] = Process(unique_id, self.service_name, hostname, install_path)\n self.processes[unique_id].pid_file = pid_file"} {"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef start(self, unique_id, configs=None):\n # the following is necessay to set the configs for this function as the combination of the\n # default configurations and the parameter with the parameter superceding the defaults but\n # not modifying the defaults\n if configs is None:\n configs = {}\n tmp = self.default_configs.copy()\n tmp.update(configs)\n configs = tmp\n\n logger.debug(\"starting \" + unique_id)\n\n # do not start if already started\n if self.get_pid(unique_id, configs) is not constants.PROCESS_NOT_RUNNING_PID:\n return None\n\n if unique_id not in self.processes:\n self.install(unique_id, configs)\n\n hostname = self.processes[unique_id].hostname\n install_path = self.processes[unique_id].install_path\n\n # order of precedence for start_command and args from highest to lowest:\n # 1. configs\n # 2. from Process\n # 3. from Deployer\n start_command = configs.get('start_command') or self.processes[unique_id].start_command or self.default_configs.get('start_command')\n pid_file = configs.get('pid_file') or self.default_configs.get('pid_file')\n if start_command is None:\n logger.error(\"start_command was not provided for unique_id: \" + unique_id)\n raise DeploymentError(\"start_command was not provided for unique_id: \" + unique_id)\n args = configs.get('args') or self.processes[unique_id].args or self.default_configs.get('args')\n if args is not None:\n full_start_command = \"{0} {1}\".format(start_command, ' '.join(args))\n else:\n full_start_command = start_command\n command = \"cd {0}; {1}\".format(install_path, full_start_command)\n env = configs.get(\"env\", {})\n with get_ssh_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ssh:\n exec_with_env(ssh, command, msg=\"Failed to start\", env=env, sync=configs.get('sync', False))\n\n self.processes[unique_id].start_command = start_command\n self.processes[unique_id].args = args\n # For cases where user pases it with start command\n if self.processes[unique_id].pid_file is None:\n self.processes[unique_id].pid_file = pid_file\n\n if 'delay' in configs:\n time.sleep(configs['delay'])", "response": "Start the service on the given unique_id."} {"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef stop(self, unique_id, configs=None):\n # the following is necessay to set the configs for this function as the combination of the\n # default configurations and the parameter with the parameter superceding the defaults but\n # not modifying the defaults\n if configs is None:\n configs = {}\n tmp = self.default_configs.copy()\n tmp.update(configs)\n configs = tmp\n\n logger.debug(\"stopping \" + unique_id)\n\n if unique_id in self.processes:\n hostname = self.processes[unique_id].hostname\n else:\n logger.error(\"Can't stop {0}: process not known\".format(unique_id))\n raise DeploymentError(\"Can't stop {0}: process not known\".format(unique_id))\n\n if configs.get('terminate_only', False):\n self.terminate(unique_id, configs)\n else:\n stop_command = configs.get('stop_command') or self.default_configs.get('stop_command')\n env = configs.get(\"env\", {})\n if stop_command is not None:\n install_path = self.processes[unique_id].install_path\n with get_ssh_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ssh:\n log_output(exec_with_env(ssh, \"cd {0}; {1}\".format(install_path, stop_command),\n msg=\"Failed to stop {0}\".format(unique_id), env=env))\n else:\n self.terminate(unique_id, configs)\n\n if 'delay' in configs:\n time.sleep(configs['delay'])", "response": "Stop the service with the given unique_id."} {"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_pid(self, unique_id, configs=None):\n RECV_BLOCK_SIZE = 16\n # the following is necessay to set the configs for this function as the combination of the\n # default configurations and the parameter with the parameter superceding the defaults but\n # not modifying the defaults\n if configs is None:\n configs = {}\n tmp = self.default_configs.copy()\n tmp.update(configs)\n configs = tmp\n\n if unique_id in self.processes:\n hostname = self.processes[unique_id].hostname\n else:\n return constants.PROCESS_NOT_RUNNING_PID\n\n if self.processes[unique_id].start_command is None:\n return constants.PROCESS_NOT_RUNNING_PID\n\n if self.processes[unique_id].pid_file is not None:\n with open_remote_file(hostname, self.processes[unique_id].pid_file,\n username=runtime.get_username(), password=runtime.get_password()) as pid_file:\n full_output = pid_file.read()\n elif 'pid_file' in configs.keys():\n with open_remote_file(hostname, configs['pid_file'],\n username=runtime.get_username(), password=runtime.get_password()) as pid_file:\n full_output = pid_file.read()\n else:\n pid_keyword = self.processes[unique_id].start_command\n if self.processes[unique_id].args is not None:\n pid_keyword = \"{0} {1}\".format(pid_keyword, ' '.join(self.processes[unique_id].args))\n pid_keyword = configs.get('pid_keyword', pid_keyword)\n # TODO(jehrlich): come up with a simpler approach to this\n pid_command = \"ps aux | grep '{0}' | grep -v grep | tr -s ' ' | cut -d ' ' -f 2 | grep -Eo '[0-9]+'\".format(pid_keyword)\n pid_command = configs.get('pid_command', pid_command)\n non_failing_command = \"{0}; if [ $? -le 1 ]; then true; else false; fi;\".format(pid_command)\n env = configs.get(\"env\", {})\n with get_ssh_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ssh:\n chan = exec_with_env(ssh, non_failing_command, msg=\"Failed to get PID\", env=env)\n output = chan.recv(RECV_BLOCK_SIZE)\n full_output = output\n while len(output) > 0:\n output = chan.recv(RECV_BLOCK_SIZE)\n full_output += output\n if len(full_output) > 0:\n pids = [int(pid_str) for pid_str in full_output.split('\\n') if pid_str.isdigit()]\n if len(pids) > 0:\n return pids\n\n return constants.PROCESS_NOT_RUNNING_PID", "response": "Gets the pid of the process with unique_id."} {"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_host(self, unique_id):\n if unique_id in self.processes:\n return self.processes[unique_id].hostname\n logger.error(\"{0} not a known process\".format(unique_id))\n raise NameError(\"{0} not a known process\".format(unique_id))", "response": "Gets the host of the process with unique_id."} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nterminates all the running processes.", "response": "def kill_all_process(self):\n \"\"\" Terminates all the running processes. By default it is set to false.\n Users can set to true in config once the method to get_pid is done deterministically\n either using pid_file or an accurate keyword\n\n \"\"\"\n if (runtime.get_active_config(\"cleanup_pending_process\",False)):\n for process in self.get_processes():\n self.terminate(process.unique_id)"} {"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting a string to the corresponding log level", "response": "def string_to_level(log_level):\n \"\"\"\n Converts a string to the corresponding log level\n \"\"\"\n if (log_level.strip().upper() == \"DEBUG\"):\n return logging.DEBUG\n if (log_level.strip().upper() == \"INFO\"):\n return logging.INFO\n if (log_level.strip().upper() == \"WARNING\"):\n return logging.WARNING\n if (log_level.strip().upper() == \"ERROR\"):\n return logging.ERROR"} {"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing command line arguments and then run the test suite", "response": "def main():\n \"\"\"\n Parse command line arguments and then run the test suite\n \"\"\"\n parser = argparse.ArgumentParser(description='A distributed test framework')\n parser.add_argument('testfile',\n help='The file that is used to determine the test suite run')\n parser.add_argument('--test-only',\n nargs='*',\n dest='test_list',\n help='run only the named tests to help debug broken tests')\n parser.add_argument('--machine-list',\n nargs='*',\n dest='machine_list',\n help='''mapping of logical host names to physical names allowing the same\n test suite to run on different hardware, each argument is a pair\n of logical name and physical name separated by a =''')\n parser.add_argument('--config-overrides',\n nargs='*',\n dest='config_overrides',\n help='''config overrides at execution time, each argument is a config with\n its value separated by a =. This has the highest priority of all\n configs''')\n parser.add_argument('-d', '--output-dir',\n dest='output_dir',\n help='''Directory to write output files and logs. Defaults to the current\n directory.''')\n parser.add_argument(\"--log-level\", dest=\"log_level\", help=\"Log level (default INFO)\", default=\"INFO\")\n parser.add_argument(\"--console-log-level\", dest=\"console_level\", help=\"Console Log level (default ERROR)\",\n default=\"ERROR\")\n parser.add_argument(\"--nopassword\", action='store_true', dest=\"nopassword\", help=\"Disable password prompt\")\n parser.add_argument(\"--user\", dest=\"user\", help=\"user to run the test as (defaults to current user)\")\n args = parser.parse_args()\n try:\n call_main(args)\n except ValueError:\n #We only sys.exit here, as call_main is used as part of a unit test\n #and should not exit the system\n sys.exit(1)"} {"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclear all the relevant globals to start fresh \u2013", "response": "def reset_all():\n \"\"\"\n Clear relevant globals to start fresh\n :return:\n \"\"\"\n global _username\n global _password\n global _active_config\n global _active_tests\n global _machine_names\n global _deployers\n reset_deployers()\n reset_collector()\n _username = None\n _password = None\n _active_config = None\n _active_tests = {}\n _machine_names = defaultdict()"} {"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the config value associated with the config_option", "response": "def get_active_config(config_option, default=None):\n \"\"\"\n gets the config value associated with the config_option or returns an empty string if the config is not found\n :param config_option:\n :param default: if not None, will be used\n :return: value of config. If key is not in config, then default will be used if default is not set to None. \n Otherwise, KeyError is thrown.\n \"\"\"\n return _active_config.mapping[config_option] if default is None else _active_config.mapping.get(config_option, default)"} {"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate the report based on the configuration and test names.", "response": "def generate(self):\n \"\"\"\n Generates the report\n \"\"\"\n self._setup()\n\n header_html = self._generate_header()\n footer_html = self._generate_footer()\n results_topbar_html = self._generate_topbar(\"results\")\n summary_topbar_html = self._generate_topbar(\"summary\")\n logs_topbar_html = self._generate_topbar(\"logs\")\n diff_topbar_html = self._generate_topbar(\"diff\")\n\n summary_body_html = self._generate_summary_body()\n diff_body_html = self._generate_diff_body()\n summary_html = header_html + summary_topbar_html + summary_body_html + footer_html\n diff_html = header_html + diff_topbar_html + diff_body_html+ footer_html\n Reporter._make_file(summary_html, self.report_info.home_page)\n Reporter._make_file(diff_html,self.report_info.diff_page)\n\n log_body_html = self._generate_log_body()\n log_html = header_html + logs_topbar_html + log_body_html+footer_html\n Reporter._make_file(log_html, self.report_info.log_page)\n\n for config_name in self.report_info.config_to_test_names_map.keys():\n config_dir = os.path.join(self.report_info.resource_dir, config_name)\n utils.makedirs(config_dir)\n\n config_body_html = self._generate_config_body(config_name)\n config_html = header_html + results_topbar_html + config_body_html + footer_html\n config_file = os.path.join(config_dir, config_name + self.report_info.report_file_sfx)\n Reporter._make_file(config_html, config_file)\n\n for test_name in self.data_source.get_test_names(config_name):\n test_body_html = self._generate_test_body(config_name, test_name)\n test_html = header_html + results_topbar_html + test_body_html + footer_html\n test_file = os.path.join(config_dir, test_name + self.report_info.report_file_sfx)\n Reporter._make_file(test_html, test_file)"} {"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets a specific file from DB.", "response": "def execute ( self, conn, dataset, dataset_access_type, transaction=False ):\n \"\"\"\n for a given file\n \"\"\"\n if not conn:\n dbsExceptionHandler(\"dbsException-failed-connect2host\", \"Oracle/Dataset/UpdateType. Expects db connection from upper layer.\", self.logger.exception)\n binds = { \"dataset\" : dataset , \"dataset_access_type\" : dataset_access_type ,\"myuser\": dbsUtils().getCreateBy(), \"mydate\": dbsUtils().getTime() }\n result = self.dbi.processData(self.sql, binds, conn, transaction)"} {"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks all the input for GET API.", "response": "def inputChecks(**_params_):\n \"\"\"\n This is a function to check all the input for GET APIs.\n \"\"\"\n def checkTypes(_func_, _params_ = _params_):\n log = clog.error_log\n @wraps(_func_)\n def wrapped(*args, **kw):\n arg_names = _func_.__code__.co_varnames[:_func_.__code__.co_argcount]\n ka = {}\n ka.update(list(zip(arg_names, args)))\n ka.update(kw)\n #print ka\n for name, value in ka.iteritems():\n #In fact the framework removes all the input variables that is not in the args list of _addMethod.\n #So DBS list API will never see these variables. For example, if one has\n #http://hostname/cms_dbs/DBS/datatiers?name=abc, the API will get a request to list all the datatiers because\n #\"name=abc\" is removed by the framework since name is not a key work for the api.\n if name !='self':\n types = _params_[name]\n #if name =='lumi_list': value = cjson.decode(value)\n if not isinstance(value, types):\n serverlog = \"Expected '%s' to be %s; was %s.\" % (name, types, type(value))\n #raise TypeError, \"Expected '%s' to be %s; was %s.\" % (name, types, type(value))\n dbsExceptionHandler(\"dbsException-invalid-input2\", message=\"Invalid Input DataType %s for %s...\" %(type(value), name[:10]),\\\n logger=log.error, serverError=serverlog)\n else:\n try:\n if isinstance(value, basestring):\n try:\n value = str(value)\n except:\n dbsExceptionHandler(\"dbsException-invalid-input\", \"invalid value for %s\" %name)\n if name == 'dataset':\n if '*' in value:\n searchdataset(value)\n else:\n reading_dataset_check(value)\n elif name =='lumi_list': value = cjson.decode(value)\n elif name =='validFileOnly': \n\t\t\t\t try:\n\t\t\t\t int(value)\n\t\t\t\t except Exception as e:\n\t\t\t\t dbsExceptionHandler(\"dbsException-invalid-input2\", message=\"invalid value for %s\" %name, serverError=\"invalid value %s for %s\" %(value, name), logger=log.error)\t\n elif name =='sumOverLumi':\n try:\n int(value)\n except Exception as e:\n dbsExceptionHandler(\"dbsException-invalid-input2\", message=\"invalid value for %s\" %name, serverError=\"invalid value %s for %s\" %(value, name), logger=log.error) \n elif name =='block_name':\n if '*' in value:\n searchblock(value)\n else:\n reading_block_check(value)\n elif name =='primary_ds_name':\n if '*' in value: searchstr(value)\n else: primdataset(value)\n elif name =='processed_ds_name':\n if '*' in value:\n searchstr(value)\n else:\n reading_procds_check(value)\n elif name=='logical_file_name':\n if '*' in value:\n searchstr(value)\n else:\n reading_lfn_check(value)\n elif name=='processing_version':\n procversion(value)\n elif name=='global_tag':\n if '*' in value: searchstr(value)\n else: globalTag(value)\n elif name == 'create_by':\n DBSUser(value)\n elif name == 'last_modified_by':\n DBSUser(value)\n else:\n searchstr(value)\n elif type(value) == list:\n if name == 'logical_file_name':\n for f in value:\n if '*' in f:\n searchstr(f)\n else:\n reading_lfn_check(f)\n elif name == 'block_names':\n for block_name in value:\n reading_block_check(block_name)\n elif name == 'run_num':\n for run_num in value:\n try: \n int(run_num)\n except Exception:\n try:\n min_run, max_run = run_num.split('-', 1)\n int(min_run)\n int(max_run)\n except Exception as e:\n serverLog = str(e) + \"\\n run_num=%s is an invalid run number.\" %run_num \n dbsExceptionHandler(\"dbsException-invalid-input2\", message=\"Invalid input data %s...: invalid run number.\" %run_num[:10],\\\n serverError=serverLog, logger=log.error)\n\t\t\t\telif name == 'dataset_id':\n\t\t\t\t for id in value:\n\t\t\t\t\ttry:\n\t\t\t\t\t int(id)\n\t\t\t\t\texcept Exception :\n\t\t\t\t\t try: \n\t\t\t\t\t\tmin_id, max_id = id.split('-', 1)\n\t\t\t\t\t\tint(min_id)\n\t\t\t\t\t\tint(max_id)\n\t\t\t\t\t except Exception as e :\t\t\n\t\t\t\t\t\tserverLog = str(e) + \"\\n dataset_id=%s is an invalid oracle id.\" %dataset_id\n\t\t\t\t\t\tdbsExceptionHandler(\"dbsException-invalid-input2\", message=\"Invalid input data %s...: invalid dataset_id.\" %id[:10], \\\n\t\t\t\t\t\t\tserverError=serverLog, logger=log.error)\t\t\n except AssertionError as ae:\n serverLog = str(ae) + \" key-value pair (%s, %s) cannot pass input checking\" %(name, value)\n #print ae\n dbsExceptionHandler(\"dbsException-invalid-input2\", message=\"Invalid Input Data %s...: Not Match Required Format\" %value[:10],\\\n serverError=serverLog, logger=log.error)\n\t\t\texcept Exception as e1:\n\t\t\t raise\t\n\t\t\t\n return _func_(*args, **kw)\n return wrapped\n return checkTypes"}