diff --git a/src/xpra/codecs/codec_constants.py b/src/xpra/codecs/codec_constants.py index 64c94568c2..4140b0c630 100644 --- a/src/xpra/codecs/codec_constants.py +++ b/src/xpra/codecs/codec_constants.py @@ -44,21 +44,30 @@ def get_avutil_enum_from_colorspace(pixfmt): class codec_spec(object): - def __init__(self, codec_class, quality, speed, setup_cost, cpu_cost, gpu_cost, latency, max_w, max_h, max_pixels, can_scale=False): + def __init__(self, codec_class, quality=100, speed=100, + setup_cost=50, cpu_cost=100, gpu_cost=0, + min_w=1, min_h=1, max_w=4*1024, max_h=4*1024, max_pixels=4*1024*4*1024, + can_scale=False, + width_mask=0xFFFF, height_mask=0xFFFF): self.codec_class = codec_class self.quality = quality self.speed = speed self.setup_cost = setup_cost self.cpu_cost = cpu_cost self.gpu_cost = gpu_cost - self.latency = latency + self.min_w = min_w + self.min_h = min_h self.max_w = max_w self.max_h = max_h self.max_pixels = max_pixels + self.width_mask = width_mask + self.height_mask = height_mask self.can_scale = can_scale def can_handle(self, width, height): - return self.max_w>=width and self.max_h>=height and self.max_pixels>(width*height) + return self.max_w>=width and self.max_h>=height \ + and self.min_w<=width and self.min_h<=height \ + and self.max_pixels>(width*height) def __str__(self): return "codec_spec(%s)" % self.__dict__ diff --git a/src/xpra/codecs/csc_nvcuda/colorspace_converter.py b/src/xpra/codecs/csc_nvcuda/colorspace_converter.py index 4561a0e28d..91ef472afe 100644 --- a/src/xpra/codecs/csc_nvcuda/colorspace_converter.py +++ b/src/xpra/codecs/csc_nvcuda/colorspace_converter.py @@ -17,7 +17,7 @@ def get_spec(in_colorspace, out_colorspace): assert in_colorspace in COLORSPACES, "invalid input colorspace: %s (must be one of %s)" % (in_colorspace, COLORSPACES) assert out_colorspace in COLORSPACES, "invalid output colorspace: %s (must be one of %s)" % (out_colorspace, COLORSPACES) #ratings: quality, speed, setup cost, cpu cost, gpu cost, latency, max_w, max_h, max_pixels - return codec_spec(ColorspaceConverter, 100, 100, 10, 100, 0, 50, 4096, 4096, 4096*4096, False) + return codec_spec(ColorspaceConverter, setup_cost=10) class ColorspaceConverter(object): diff --git a/src/xpra/codecs/csc_swscale/colorspace_converter.pyx b/src/xpra/codecs/csc_swscale/colorspace_converter.pyx index e922143350..93aa21a7f8 100644 --- a/src/xpra/codecs/csc_swscale/colorspace_converter.pyx +++ b/src/xpra/codecs/csc_swscale/colorspace_converter.pyx @@ -57,10 +57,10 @@ def get_output_colorspaces(input_colorspace): def get_spec(in_colorspace, out_colorspace): assert in_colorspace in COLORSPACES, "invalid input colorspace: %s (must be one of %s)" % (in_colorspace, COLORSPACES) assert out_colorspace in COLORSPACES, "invalid output colorspace: %s (must be one of %s)" % (out_colorspace, COLORSPACES) - #ratings: quality, speed, setup cost, cpu cost, gpu cost, latency, max_w, max_h, max_pixels - #we can handle high quality and full speed #setup cost is very low (usually less than 1ms!) - return codec_spec(ColorspaceConverter, 100, 100, 20, 100, 0, 0, 4096, 4096, 4096*4096, True) + #there are restrictions on dimensions (8x2 minimum!) + #swscale can be used to scale (obviously) + return codec_spec(ColorspaceConverter, setup_cost=20, min_w=8, min_h=2, can_scale=True) cdef class CSCImage: diff --git a/src/xpra/codecs/enc_x264/encoder.pyx b/src/xpra/codecs/enc_x264/encoder.pyx index 6dbe35c4d7..300483c7e9 100644 --- a/src/xpra/codecs/enc_x264/encoder.pyx +++ b/src/xpra/codecs/enc_x264/encoder.pyx @@ -67,7 +67,7 @@ def get_spec(colorspace): #ratings: quality, speed, setup cost, cpu cost, gpu cost, latency, max_w, max_h, max_pixels #we can handle high quality and any speed #setup cost is moderate (about 10ms) - return codec_spec(Encoder, 100, 100, 70, 100, 0, 40, 4096, 4096, 4096*4096) + return codec_spec(Encoder, setup_cost=70, width_mask=0xFFFE, height_mask=0xFFFE) cdef class Encoder: diff --git a/src/xpra/codecs/nvenc/encoder.py b/src/xpra/codecs/nvenc/encoder.py index 0525429ea3..853584c963 100644 --- a/src/xpra/codecs/nvenc/encoder.py +++ b/src/xpra/codecs/nvenc/encoder.py @@ -12,7 +12,7 @@ def get_colorspaces(): def get_spec(colorspace): assert colorspace in COLORSPACES, "invalid colorspace: %s (must be one of %s)" % (colorspace, COLORSPACES) #ratings: quality, speed, setup cost, cpu cost, gpu cost, latency, max_w, max_h, max_pixels - return codec_spec(Encoder, 60, 100, 80, 10, 100, 80, 4096, 4096, 4096*4096) + return codec_spec(Encoder, quality=60, setup_cost=100, cpu_cost=10, gpu_cost=100) class Encoder(object): diff --git a/src/xpra/codecs/video_enc_pipeline.py b/src/xpra/codecs/video_enc_pipeline.py index cff72610ef..599578ea7b 100644 --- a/src/xpra/codecs/video_enc_pipeline.py +++ b/src/xpra/codecs/video_enc_pipeline.py @@ -82,45 +82,3 @@ def init_csc_option(self, csc_module): spec = csc_module.get_spec(in_csc, out_csc) item = out_csc, spec csc_specs.append(item) - - - def check_pipeline(self, csc_encoder, video_encoder, encoding, width, height, src_format): - if video_encoder is None: - return False - - if csc_encoder: - if csc_encoder.get_src_format()!=src_format: - debug("check_pipeline csc: switching source format from %s to %s", - csc_encoder.get_src_format(), src_format) - return False - elif csc_encoder.get_src_width()!=width or csc_encoder.get_src_height()!=height: - debug("check_pipeline csc: window dimensions have changed from %sx%s to %sx%s", - csc_encoder.get_src_width(), csc_encoder.get_src_height(), width, height) - return False - elif csc_encoder.get_dst_format()!=video_encoder.get_src_format(): - log.warn("check_pipeline csc: intermediate format mismatch: %s vs %s", - csc_encoder.get_dst_format(), video_encoder.get_src_format()) - return False - - encoder_src_format = csc_encoder.get_dst_format() - encoder_src_width = csc_encoder.get_dst_width() - encoder_src_height = csc_encoder.get_dst_height() - else: - #direct to video encoder without csc: - encoder_src_format = src_format - encoder_src_width = width - encoder_src_height = height - - if video_encoder.get_src_format()!=encoder_src_format: - debug("check_pipeline video: invalid source format %s, expected %s", - video_encoder.get_src_format(), encoder_src_format) - return False - elif video_encoder.get_type()!=encoding: - debug("check_pipeline video: invalid encoding %s, expected %s", - video_encoder.get_type(), encoding) - return False - elif video_encoder.get_width()!=encoder_src_width or video_encoder.get_height()!=encoder_src_height: - debug("check_pipeline video: window dimensions have changed from %sx%s to %sx%s", - video_encoder.get_width(), video_encoder.get_height(), encoder_src_width, encoder_src_height) - return False - return True diff --git a/src/xpra/codecs/vpx/encoder.pyx b/src/xpra/codecs/vpx/encoder.pyx index 2f16408cf4..e594efb6f4 100644 --- a/src/xpra/codecs/vpx/encoder.pyx +++ b/src/xpra/codecs/vpx/encoder.pyx @@ -52,7 +52,7 @@ def get_spec(colorspace): #ratings: quality, speed, setup cost, cpu cost, gpu cost, latency, max_w, max_h, max_pixels #quality: we only handle YUV420P but this is already accounted for by get_colorspaces() based score calculations #setup cost is reasonable (usually about 5ms) - return codec_spec(Encoder, 100, 100, 40, 100, 0, 30, 4096, 4096, 4096*4096) + return codec_spec(Encoder, setup_cost=40) cdef class Encoder: diff --git a/src/xpra/server/window_video_source.py b/src/xpra/server/window_video_source.py index 787e16b62f..5602f80ea0 100644 --- a/src/xpra/server/window_video_source.py +++ b/src/xpra/server/window_video_source.py @@ -39,6 +39,9 @@ def __init__(self, *args): if x in self.SERVER_CORE_ENCODINGS: self._encoders[x] = self.video_encode + self.width_mask = 0xFFFF + self.height_mask = 0xFFFF + self._csc_encoder = None self._video_encoder = None self._lock = Lock() #to ensure we serialize access to the encoder and its internals @@ -101,13 +104,16 @@ def cancel_damage(self): def process_damage_region(self, damage_time, window, x, y, w, h, coding, options): WindowSource.process_damage_region(self, damage_time, window, x, y, w, h, coding, options) - if coding in ("vpx", "x264") and (w%2==1 or h%2==1): - if w%2==1: - lossless = self.find_common_lossless_encoder(window.has_alpha(), coding, 1*h) - WindowSource.process_damage_region(self, damage_time, window, x+w-1, y, 1, h, lossless, options) - if h%2==1: - lossless = self.find_common_lossless_encoder(window.has_alpha(), coding, w*1) - WindowSource.process_damage_region(self, damage_time, window, x, y+h-1, x+w, 1, lossless, options) + #now figure out if we need to send edges separately: + dw = w - (w & self.width_mask) + dh = h - (h & self.height_mask) + if coding in ("vpx", "x264") and (dw>0 or dh>0): + if dw>0: + lossless = self.find_common_lossless_encoder(window.has_alpha(), coding, dw*h) + WindowSource.process_damage_region(self, damage_time, window, x+w-dw, y, dw, h, lossless, options) + if dh>0: + lossless = self.find_common_lossless_encoder(window.has_alpha(), coding, w*dh) + WindowSource.process_damage_region(self, damage_time, window, x, y+h-dh, x+w, dh, lossless, options) def reconfigure(self, force_reload=False): @@ -224,9 +230,9 @@ def get_score(self, csc_format, csc_spec, encoder_spec, width, height): """ #first discard if we cannot handle this size: if csc_spec and not csc_spec.can_handle(width, height): - return -1, "" + return -1 if not encoder_spec.can_handle(width, height): - return -1, "" + return -1 #debug("get_score%s", (csc_format, csc_spec, encoder_spec, # width, height, min_quality, target_quality, min_speed, target_speed)) def clamp(v): @@ -261,12 +267,22 @@ def clamp(v): #score for "edge resistance": ecsc_score = 100 if csc_spec: + #OR the masks so we have a chance of making it work + width_mask = csc_spec.width_mask & encoder_spec.width_mask + height_mask = csc_spec.height_mask & encoder_spec.height_mask + csc_width = width & width_mask + csc_height = height & height_mask if self._csc_encoder is None or self._csc_encoder.get_dst_format()!=csc_format or \ type(self._csc_encoder)!=csc_spec.codec_class or \ - self._csc_encoder.get_src_width()!=width or self._csc_encoder.get_src_height()!=height: + self._csc_encoder.get_src_width()!=csc_width or self._csc_encoder.get_src_height()!=csc_height: #if we have to change csc, account for new csc setup cost: ecsc_score = 100 - csc_spec.setup_cost - enc_width, enc_height = self.get_encoder_dimensions(csc_spec, width, height) + enc_width, enc_height = self.get_encoder_dimensions(csc_spec, encoder_spec, csc_width, csc_height) + else: + width_mask = encoder_spec.width_mask + height_mask = encoder_spec.height_mask + enc_width = width & width_mask + enc_height = height & height_mask ee_score = 100 if self._video_encoder is None or type(self._video_encoder)!=encoder_spec.codec_class or \ self._video_encoder.get_src_format()!=csc_format or \ @@ -279,22 +295,22 @@ def clamp(v): width, height), int(qscore), int(sscore), int(er_score)) return int((qscore+sscore+er_score)/3.0) - def get_encoder_dimensions(self, csc_spec, width, height): + def get_encoder_dimensions(self, csc_spec, encoder_spec, width, height): """ - Given a csc spec and dimensions, we calculate + Given a csc and encoder specs and dimensions, we calculate the dimensions that we would use as output. Taking into account: * applications can require scaling (see "scaling" attribute) * we scale fullscreen and maximize windows when at high speed and low quality. * we do not bother scaling small dimensions + * the encoder may not support all dimensions + (see width and height masks) """ - if not csc_spec or not self.video_scaling or width<=32 or height<=16: - return width, height - #FIXME: take screensize into account, - #we want to scale more when speed is high and min-quality is low - #also framerate? + #TODO: framerate is relevant, probably scaling = self.scaling + if not self.video_scaling: + scaling = None if scaling is None: quality = self.get_current_quality() speed = self.get_current_speed() @@ -311,8 +327,10 @@ def get_encoder_dimensions(self, csc_spec, width, height): return width, height if float(v)/float(u)<0.1: #don't downscale more than 10 times! (for each dimension - that's 100 times!) v, u = 1, 10 - enc_width = int(width * v / u) - enc_height = int(height * v / u) + enc_width = int(width * v / u) & encoder_spec.width_mask + enc_height = int(height * v / u) & encoder_spec.height_mask + if not encoder_spec.can_handle(enc_width, enc_height): + return width, height return enc_width, enc_height @@ -321,9 +339,11 @@ def check_pipeline(self, encoding, width, height, src_format): Checks that the current pipeline is still valid for the given input. If not, close it and make a new one. """ + debug("check_pipeline%s", (encoding, width, height, src_format)) #must be called with video lock held! - if self._video_pipeline_helper.check_pipeline(self._csc_encoder, self._video_encoder, encoding, width, height, src_format): + if self.do_check_pipeline(encoding, width, height, src_format): return True #OK! + #cleanup existing one if needed: if self._csc_encoder: self.do_csc_encoder_cleanup() @@ -333,6 +353,56 @@ def check_pipeline(self, encoding, width, height, src_format): scores = self.get_video_pipeline_options(encoding, width, height, src_format) return self.setup_pipeline(scores, width, height, src_format) + def do_check_pipeline(self, encoding, width, height, src_format): + """ + Checks that the current pipeline is still valid + for the given input. If not, close it and make a new one. + """ + debug("do_check_pipeline%s", (encoding, width, height, src_format)) + #must be called with video lock held! + if self._video_encoder is None: + return False + + if self._csc_encoder: + csc_width = width & self.width_mask + csc_height = height & self.height_mask + if self._csc_encoder.get_src_format()!=src_format: + debug("check_pipeline csc: switching source format from %s to %s", + self._csc_encoder.get_src_format(), src_format) + return False + elif self._csc_encoder.get_src_width()!=csc_width or self._csc_encoder.get_src_height()!=csc_height: + debug("check_pipeline csc: window dimensions have changed from %sx%s to %sx%s, csc info=%s", + self._csc_encoder.get_src_width(), self._csc_encoder.get_src_height(), csc_width, csc_height, self._csc_encoder.get_info()) + return False + elif self._csc_encoder.get_dst_format()!=self._video_encoder.get_src_format(): + log.warn("check_pipeline csc: intermediate format mismatch: %s vs %s, csc info=%s", + self._csc_encoder.get_dst_format(), self._video_encoder.get_src_format(), self._csc_encoder.get_info()) + return False + + encoder_src_format = self._csc_encoder.get_dst_format() + encoder_src_width = self._csc_encoder.get_dst_width() + encoder_src_height = self._csc_encoder.get_dst_height() + else: + #direct to video encoder without csc: + encoder_src_format = src_format + encoder_src_width = width & self.width_mask + encoder_src_height = height & self.height_mask + + if self._video_encoder.get_src_format()!=encoder_src_format: + debug("check_pipeline video: invalid source format %s, expected %s", + self._video_encoder.get_src_format(), encoder_src_format) + return False + elif self._video_encoder.get_type()!=encoding: + debug("check_pipeline video: invalid encoding %s, expected %s", + self._video_encoder.get_type(), encoding) + return False + elif self._video_encoder.get_width()!=encoder_src_width or self._video_encoder.get_height()!=encoder_src_height: + debug("check_pipeline video: window dimensions have changed from %sx%s to %sx%s", + self._video_encoder.get_width(), self._video_encoder.get_height(), encoder_src_width, encoder_src_height) + return False + return True + + def setup_pipeline(self, scores, width, height, src_format): """ Given a list of pipeline options ordered by their score @@ -349,20 +419,28 @@ def setup_pipeline(self, scores, width, height, src_format): speed = self.get_current_speed() quality = self.get_current_quality() if csc_spec: - enc_width, enc_height = self.get_encoder_dimensions(csc_spec, width, height) + #TODO: no need to OR encoder mask if we are scaling... + self.width_mask = csc_spec.width_mask & encoder_spec.width_mask + self.height_mask = csc_spec.height_mask & encoder_spec.height_mask + csc_width = width & self.width_mask + csc_height = height & self.height_mask + enc_width, enc_height = self.get_encoder_dimensions(csc_spec, encoder_spec, csc_width, csc_height) #csc speed is not very important compared to encoding speed, #so make sure it never degrades quality csc_speed = min(speed, 100-quality/2.0) csc_start = time.time() self._csc_encoder = csc_spec.codec_class() - self._csc_encoder.init_context(width, height, src_format, + self._csc_encoder.init_context(csc_width, csc_height, src_format, enc_width, enc_height, enc_in_format, csc_speed) csc_end = time.time() debug("setup_pipeline: csc=%s, info=%s, setup took %.2fms", self._csc_encoder, self._csc_encoder.get_info(), (csc_end-csc_start)*1000.0) else: - enc_width = width - enc_height = height + #use the encoder's mask directly since that's all we have to worry about! + self.width_mask = encoder_spec.width_mask + self.height_mask = encoder_spec.height_mask + enc_width = width & self.width_mask + enc_height = height & self.height_mask enc_start = time.time() self._video_encoder = encoder_spec.codec_class() self._video_encoder.init_context(enc_width, enc_height, enc_in_format, quality, speed, self.encoding_options) @@ -388,15 +466,18 @@ def video_encode(self, encoding, image, options): """ debug("video_encode%s", (encoding, image, options)) x, y, w, h = image.get_geometry()[:4] - width = w & 0xFFFE - height = h & 0xFFFE assert x==0 and y==0, "invalid position: %s,%s" % (x,y) src_format = image.get_pixel_format() try: self._lock.acquire() - if not self.check_pipeline(encoding, width, height, src_format): + if not self.check_pipeline(encoding, w, h, src_format): raise Exception("failed to setup a pipeline for %s encoding!" % encoding) + #dw and dh are the edges we don't handle here + width = w & self.width_mask + height = h & self.height_mask + debug("video_encode%s w-h=%s-%s, width-height=%s-%s", (encoding, image, options), w, h, width, height) + csc_image, csc, enc_width, enc_height = self.csc_image(image, width, height) start = time.time()